diff --git a/.gitattributes b/.gitattributes index 8302eb59e3..477d2ce0e0 100644 --- a/.gitattributes +++ b/.gitattributes @@ -23,8 +23,16 @@ # # scripts/check_append_only_ledgers.sh holds that distinction as a case table. docs/audits/*.jsonl merge=union -# Append-only ledgers one level down. `docs/audits/*.jsonl` does not cross a `/`, -# so these two resolved no merge driver (check_append_only_ledgers, after the -# main merge into #4431). Named file by file: the scope rule above still holds. -docs/audits/review-corpus/corpus-v1.jsonl merge=union -docs/audits/rex-001/rex-04-admission.jsonl merge=union +# rex/001 (#4354): the admission files are appended one row per cell, so they are +# ledgers. `docs/audits/*.jsonl` does not cross a `/`, so they need their own line +# (main named rex-04-admission.jsonl alone, after the main merge into #4431; this +# glob covers it). +docs/audits/rex-001/*.jsonl merge=union +# The corpus (written whole by `rex corpus`) and the #4026 oracle captures (written +# whole by c4026_live.sh) are single-writer goldens, not ledgers: a union merge of +# two regenerations would interleave both versions' rows. Declared `audit-golden`, +# so a two-sided change conflicts and check_append_only_ledgers.sh accepts them. +docs/audits/review-corpus/*.jsonl audit-golden +# PRM-S1 v2 (#4447): the frozen set and each build's replay rows are written whole +# by `replay build` / `replay run`, so they are single-writer goldens, not ledgers. +docs/audits/prm-001/s1-v2/**/*.jsonl audit-golden diff --git a/Cargo.lock b/Cargo.lock index 12091a73b2..e1a8ef240e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1378,10 +1378,14 @@ dependencies = [ name = "aprender-review-experiment" version = "0.71.0" dependencies = [ + "half", + "regex", "serde", "serde_json", "sha2 0.10.9", + "toml 0.8.23", "ureq 2.12.1", + "zstd", ] [[package]] diff --git a/Makefile b/Makefile index 31d2e727aa..914de1af3a 100644 --- a/Makefile +++ b/Makefile @@ -1565,6 +1565,21 @@ oracle-check: oracle @git diff --exit-code tests/oracle/differential.json \ || { echo "FAIL: tests/oracle/differential.json differs from a fresh run — commit it"; exit 1; } +# PRA-001 §1 G14 (T0): read-only, one row per fleet host; UNREACHABLE fails the target. +transcript-retention-audit: + @bash scripts/transcript_retention_audit.sh +.PHONY: transcript-retention-audit + +# PRM-C13 (was PRA T13), lane-independence-v1: κ_err scoreboard of the shadow lane vs every +# counted lane on matured gold SPLIT rows. With MANIFEST and CANDIDATE it gates (exit 12 = S-14). +SPLIT ?= val +MIN_N ?= 30 +lane-kappa: + @test -n "$(ROWS)" || { echo "usage: make lane-kappa ROWS=trace.jsonl [SPLIT=val] [MIN_N=30 | MANIFEST=m.json CANDIDATE=c.jsonl]"; exit 2; } + cargo run -q -p aprender-review-experiment --example rex -- lane-kappa "$(ROWS)" --split "$(SPLIT)" \ + $(if $(MANIFEST),--manifest "$(MANIFEST)" --candidate "$(CANDIDATE)",--min-n "$(MIN_N)") +.PHONY: lane-kappa + # ONT-001 §3.8 / ONT-2c — the OWL oracle (release gate only, R-13; never per PR). Three arms: # horned-owl re-parses the fixture's written .ofn and must equal the HAND-WRITTEN axiom list; every live # axiom must be a told-closure-admitted kind; ELK 0.4.3 (pinned by sha256, needs a JVM) must agree with diff --git a/README.md b/README.md index 237de95dae..63e9de0014 100644 --- a/README.md +++ b/README.md @@ -114,7 +114,7 @@ publishing — all backed by YAML provable contracts that fail CI on drift. | Metric | Count | Source of truth | |-------:|------:|---| | Workspace crates | **81** workspace crates | `cargo metadata --no-deps` (NOT `ls crates/` — 4 are `exclude`d, 1 has no Cargo.toml) | -| Provable contracts | **1903** provable contracts | `contracts/census.json` `.n_files` — the set `pv lint` walks (`pv census`, ONT-001 ONT-1; regenerated by `make contracts`, written by `make readme-sync`, guarded by `scripts/check_readme_claims.sh`) | +| Provable contracts | **1922** provable contracts | `contracts/census.json` `.n_files` — the set `pv lint` walks (`pv census`, ONT-001 ONT-1; regenerated by `make contracts`, written by `make readme-sync`, guarded by `scripts/check_readme_claims.sh`) | | CLI commands | **113** CLI commands | `contracts/apr-cli-commands-v1.yaml` §`commands` (parse the list; `apr --help` also lists `help` itself, and `grep -c '^ - name:'` gives 119 — other same-indent `name:` keys exist in the file) | | Book CLI chapters | **114** chapters | `ls book/src/cli/*.md` | | Book lib chapters | **73** chapters | `ls book/src/lib/*.md` (parity with `pub mod`) | @@ -403,7 +403,7 @@ falsification_tests: prediction: apr validate bad-model.apr exits non-zero ``` -The tree carries 1903 contracts across inference, training, quantization, attention, FFN, +The tree carries 1922 contracts across inference, training, quantization, attention, FFN, tokenization, model formats, CLI safety — and this README itself. ## Migration from old crates diff --git a/contracts/agent-trace-terms-v1.yaml b/contracts/agent-trace-terms-v1.yaml new file mode 100644 index 0000000000..7c6d58f929 --- /dev/null +++ b/contracts/agent-trace-terms-v1.yaml @@ -0,0 +1,48 @@ +metadata: + version: 1.0.0 + kind: schema + description: | + Provider-terms tagging for agent-trace-v1 rows (PRA-001 T12, spec §2.10, §9 P-TERMS; + agent-trace ruling 2026-09-25 items 1 and 7). Every row carries provider + {anthropic, google, local}, an exact model_id (never a family alias; a local model is + its weights sha256), access_channel {anthropic-api, claude-code-cli, antigravity, + gemini-api, vertex, apr-serve} consistent with the provider, and terms_ref {https url, + YYYY-MM-DD effective_date, RFC 3339 UTC fetched_at}. Any pool can be rebuilt without + a provider or channel through the first-class --exclude-provider / --exclude-channel + arguments. A row with a missing or inconsistent tag is REFUSED by every rebuild, since + an untagged row cannot be shown not to come from an excluded source. The public-release + default keeps local rows only (§2.11). This records the governing terms. It is not + legal advice; S-6 governs. Implementation: crates/aprender-review-experiment/src/terms.rs. + references: + - "docs/specifications/research-design-pr-agent-qwen-3.5.md" + - "contracts/training-pool-admission-v1.yaml" + +equations: + rebuild: + formula: "kept(R, X) = { r in R : gaps(r) = {} and provider(r) not in X.providers and channel(r) not in X.channels }" + +falsification_tests: + - id: FALSIFY-TERMS-001 + name: pool_rebuild_excluding_a_provider + prediction: "P-TERMS: a rebuild of a six-channel batch with --exclude-provider google succeeds, keeps 0 google rows, and passes the rest through byte-identical" + test_harness: "cargo test -p aprender-review-experiment --lib falsify_terms_001" + expected_output: "exit 0" + if_fails: "A pool cannot be rebuilt without a provider, so provider terms cannot be honoured later." + - id: FALSIFY-TERMS-002 + name: channel_exclusion_is_narrower_than_provider + prediction: "--exclude-channel antigravity drops only antigravity rows; gemini-api and vertex rows of the same provider stay" + test_harness: "cargo test -p aprender-review-experiment --lib falsify_terms_002" + expected_output: "exit 0" + if_fails: "Terms that differ by channel (App. A.7) cannot be applied separately." + - id: FALSIFY-TERMS-003 + name: identity_case_table + prediction: "rows with a family alias, unknown or empty model_id, unknown provider or channel, a channel of another provider, a non-sha local model, or a missing/non-https/ill-dated terms_ref all have gaps; well-formed rows do not" + test_harness: "cargo test -p aprender-review-experiment --lib falsify_terms_003" + expected_output: "exit 0" + if_fails: "A row with unknown or aliased identity (G-ID) is accepted as tagged." + - id: FALSIFY-TERMS-004 + name: untagged_row_is_refused_never_kept + prediction: "a row with gaps, or a non-JSON line, is refused by every rebuild, including one that excludes nothing, and is never in kept" + test_harness: "cargo test -p aprender-review-experiment --lib falsify_terms_004" + expected_output: "exit 0" + if_fails: "An untagged row slips past an exclusion filter it cannot be checked against." diff --git a/contracts/contracts.nt b/contracts/contracts.nt index f09e24b2be..49fcde1949 100644 --- a/contracts/contracts.nt +++ b/contracts/contracts.nt @@ -4665,6 +4665,11 @@ . "contracts/duende/agent-orchestration-v1.yaml"^^ . "agent-orchestration-v1"^^ . + . + . + "contracts/agent-trace-terms-v1.yaml"^^ . + "agent-trace-terms-v1"^^ . + "schema"^^ . . . . @@ -11951,6 +11956,11 @@ . "contracts/kv-cache-sizing-v1.yaml"^^ . "kv-cache-sizing-v1"^^ . + . + . + "contracts/lane-independence-v1.yaml"^^ . + "lane-independence-v1"^^ . + "schema"^^ . . . . @@ -13149,6 +13159,16 @@ . "contracts/trueno-rag/retrieval-quality-v1.yaml"^^ . "retrieval-quality-v1"^^ . + . + . + "contracts/review-b2-loop-v1.yaml"^^ . + "review-b2-loop-v1"^^ . + "schema"^^ . + . + . + "contracts/review-champion-challenger-v1.yaml"^^ . + "review-champion-challenger-v1"^^ . + "schema"^^ . . . "contracts/review-corpus-contamination-v1.yaml"^^ . @@ -13164,16 +13184,51 @@ "contracts/review-experiment-receipt-v1.yaml"^^ . "review-experiment-receipt-v1"^^ . "schema"^^ . + . + . + "contracts/review-lane-perf-ratchet-v1.yaml"^^ . + "review-lane-perf-ratchet-v1"^^ . + "schema"^^ . + . + . + "contracts/review-ledger-v1.yaml"^^ . + "review-ledger-v1"^^ . + "schema"^^ . + . + . + "contracts/review-ledger-v2.yaml"^^ . + "review-ledger-v2"^^ . + "schema"^^ . + . + . + "contracts/review-replay-v1.yaml"^^ . + "review-replay-v1"^^ . + "schema"^^ . . . "contracts/rex-cell-admission-v1.yaml"^^ . "rex-cell-admission-v1"^^ . "schema"^^ . + . + . + "contracts/rex-pilot-projection-v1.yaml"^^ . + "rex-pilot-projection-v1"^^ . + "schema"^^ . . . "contracts/rex-prereg-v1.yaml"^^ . "rex-prereg-v1"^^ . "schema"^^ . + . + . + "contracts/rex-promotion-ladder-v1.yaml"^^ . + "rex-promotion-ladder-v1"^^ . + "schema"^^ . + . + . + "contracts/rex-promotion-ladder-v2.yaml"^^ . + "rex-promotion-ladder-v2"^^ . + "schema"^^ . . . . @@ -13363,6 +13418,11 @@ . "contracts/entrenar/sovereign-tensor-v1.yaml"^^ . "sovereign-tensor-v1"^^ . + . + . + "contracts/sparse-logits-v1.yaml"^^ . + "sparse-logits-v1"^^ . + "schema"^^ . . . . @@ -13534,11 +13594,31 @@ . "contracts/tokenizer-vocab-v1.yaml"^^ . "tokenizer-vocab-v1"^^ . + . + . + "contracts/trace-admission-prov-v1.yaml"^^ . + "trace-admission-prov-v1"^^ . + "schema"^^ . + . + . + "contracts/trace-admission-secret-v1.yaml"^^ . + "trace-admission-secret-v1"^^ . + "schema"^^ . . . "contracts/trace-attn-sub-stages-v1.yaml"^^ . "trace-attn-sub-stages-v1"^^ . "pattern"^^ . + . + . + "contracts/trace-datacard-v1.yaml"^^ . + "trace-datacard-v1"^^ . + "schema"^^ . + . + . + "contracts/trace-dedup-v1.yaml"^^ . + "trace-dedup-v1"^^ . + "schema"^^ . . . "contracts/trace-ffn-sub-block-gguf-v1.yaml"^^ . @@ -13560,6 +13640,16 @@ "contracts/trace-moe-gpu-sub-stages-v1.yaml"^^ . "trace-moe-gpu-sub-stages-v1"^^ . "pattern"^^ . + . + . + "contracts/trace-outcome-join-v1.yaml"^^ . + "trace-outcome-join-v1"^^ . + "schema"^^ . + . + . + "contracts/trace-split-guard-v1.yaml"^^ . + "trace-split-guard-v1"^^ . + "schema"^^ . . . . @@ -13732,6 +13822,11 @@ "contracts/work-dbc-v1.yaml"^^ . "work-dbc-v1"^^ . "pattern"^^ . + . + . + "contracts/workload-profile-v1.yaml"^^ . + "workload-profile-v1"^^ . + "schema"^^ . . . . @@ -16714,13 +16809,24 @@ "true"^^ . "signing_demo"^^ . "false"^^ . + . + . + "aprender-review-experiment"^^ . + "crates/aprender-review-experiment/examples/replay.rs"^^ . + "true"^^ . + "replay"^^ . + "llama"^^ . + "qwen3.5"^^ . + "true"^^ . . . "aprender-review-experiment"^^ . "crates/aprender-review-experiment/examples/rex.rs"^^ . "true"^^ . "rex"^^ . - "false"^^ . + "llama"^^ . + "qwen3.5"^^ . + "true"^^ . . . "aprender-serve"^^ . diff --git a/contracts/lane-independence-v1.yaml b/contracts/lane-independence-v1.yaml new file mode 100644 index 0000000000..6b6a0c768d --- /dev/null +++ b/contracts/lane-independence-v1.yaml @@ -0,0 +1,53 @@ +metadata: + version: 2.0.0 + kind: schema + description: | + PRM-C13 (was PRA-001 T13) (PRM-001 v3 "PROMETHEUS" H7 §3, S-14; row 2 of §7, was research-design-pr-agent-qwen-3.5 §5 M4). + `make lane-kappa SPLIT=val ROWS=F [MANIFEST=M CANDIDATE=F2]` = `rex lane-kappa ROWS --split val (--min-n N | --manifest M --candidate ROWS2)` (crates/aprender-review-experiment: kappa_probe.rs, examples/rex.rs) + measures κ_err, Cohen's κ on binary per-item error indicators against GOLD, for every + (qwen, counted-lane) pair on matured gold val rows, reusing stats::error_kappa (PRM-001 §3 error + overlap). An error is a parsed verdict that disagrees with the gold outcome; unparsed items and + silver, pending or quarantined rows are excluded, never counted as errors. A pair with fewer than + `min_n` shared items, or with κ undefined, reports `insufficient` (S-14), never a number. + 2.0.0: the gate is v3's δ-free H7, decided by the prereg-locked stats::h7_holds. On I, the + items the shadow lane and EVERY counted lane (voter) scored, κ_err(qwen, V) must be ≤ the + largest voter–voter κ_err for each voter V. v2's baseline + δ rule is gone, and a manifest that + still carries `delta` is refused, since nothing would read it. The voters and min_n come only from + the run manifest, registered before training. Refusal is an andon (non-zero exit), not a warning. + references: + - "docs/specifications/PRM-001-prometheus.md" + - "docs/specifications/review-experiment-protocol.md" + +equations: + error_indicator: + formula: "err(lane, i) = parsed(lane, i) ∧ verdict(lane, i) ≠ gold(i)" + kappa_err: + formula: "κ_err(a, b) = (p_o − p_e) / (1 − p_e) over items i with gold(i) ∧ matured(i) ∧ split(i) = val ∧ parsed(a, i) ∧ parsed(b, i)" + gate: + formula: "pass ⇔ |I| ≥ min_n ∧ h7_holds([κ_err_I(qwen, V) : V ∈ voters], [κ_err_I(V_i, V_j) : i < j]) = Some(true), i.e. ∀ V: κ_err_I(qwen, V) ≤ max_{i voter–voter max) is refused naming that voter; with identical voters (ceiling 1) the same copy passes, since the bound is inclusive" + test_harness: "cargo test -p aprender-review-experiment --lib falsify_lind_003" + expected_output: "exit 0" + if_fails: "A silver-trained Qwen that copies the voters' blind spots could occupy the dissent lane (R13, PRA-H1)." + - id: FALSIFY-LIND-004 + name: voters_and_min_n_are_pre_registered_and_delta_is_gone + prediction: "a manifest with no block, min_n 0, no voters, a registration at or after training start, a non-Z timestamp, or a v2 `delta` is refused" + test_harness: "cargo test -p aprender-review-experiment --lib falsify_lind_004" + expected_output: "exit 0" + if_fails: "The runner could pick the voters after seeing κ, or believe a δ that the δ-free gate never reads." diff --git a/contracts/review-b2-loop-v1.yaml b/contracts/review-b2-loop-v1.yaml new file mode 100644 index 0000000000..6046539ce9 --- /dev/null +++ b/contracts/review-b2-loop-v1.yaml @@ -0,0 +1,47 @@ +metadata: + version: 1.0.0 + kind: schema + description: | + PROMETHEUS (PRM-001, was REX-001) REX-12 B2 loop, spec §5.3 B2a..B2d. Each B2 tier waits + on one apr verb accepting qwen35 (B2a distill, B2b finetune, B2c merge, B2d quantize). + `rex b2 status` reads the in-tree refusal ledger (evidence/verbs/refusals.json, + refusal-receipt-v1) and writes one row per tier: ready only on a positive acceptance + receipt, verb_refused{removed_by} on a ledger row, and unledgered when neither exists. + A missing refusal row is not acceptance, so an unmeasured verb is NotRun and never + silently ready. It exits 11 while any row is NotRun. Ready tiers feed the REX-11 §5.4 + gate (review-champion-challenger-v1). `rex b2 teacher-receipt` receipts the 27B + teacher's top-k logits at m = 1 over the train pool (every non-test item): sha, count, + k, the size of the sealed index, and 0 test hits. Any test item, unknown item, + duplicate row, wrong-k or non-finite position, sealed diff sha or hunk in any string + field, or an empty sealed index refuses the whole file. At HEAD all four rows are + unledgered; the qwen35 fine-tune/distill verbs are EPIC #4002 (0.75.0). + references: + - "docs/specifications/review-experiment-protocol.md" + - "contracts/refusal-receipt-v1.yaml" + - "https://github.com/paiml/aprender/issues/4367" + +equations: + row: + formula: "ready(v) ⇔ accepted(v) ∧ ¬refused(v); accepted(v) ∧ refused(v) ⇒ error; ¬accepted(v) ∧ ¬refused(v) ⇒ unledgered" + teacher: + formula: "receipt ⇔ |sealed| > 0 ∧ ∀ row: split(id) = dev ∧ once(id) ∧ ∀ pos: |pos| = k ∧ finite ∧ scan(file) = ∅" + +falsification_tests: + - id: FALSIFY-RXB-001 + name: no_b2_row_is_ready_without_an_acceptance_receipt + prediction: "on the in-tree ledger no B2 row is ready; a ledger row reads verb_refused with its removed_by; no row plus no acceptance reads unledgered; accepted and refused at once is an error" + test_harness: "cargo test -p aprender-review-experiment --lib falsify_rxb_001" + expected_output: "exit 0" + if_fails: "A B2 tier whose verb nobody measured could read as ready and enter §5.4." + - id: FALSIFY-RXB-002 + name: the_ledger_is_refusal_receipt_v1_or_rejected + prediction: "removed_by accepts v., never and unscheduled; tbd, a bare or patch version, a sha and empty are rejected; duplicate verbs, exit 0 and a foreign schema reject the file" + test_harness: "cargo test -p aprender-review-experiment --lib falsify_rxb_002" + expected_output: "exit 0" + if_fails: "A decorative removed_by would read as a validated refusal." + - id: FALSIFY-RXB-003 + name: the_teacher_dataset_is_receipted_with_zero_test_hashes + prediction: "two dev rows at k = 4 are receipted with sha, count 2, m 1 and 0 test hits; a test item, a duplicate, an unknown id, wrong k, NaN, a sealed diff in a string field, a sealed diff sha, no rows and an empty index are each refused" + test_harness: "cargo test -p aprender-review-experiment --lib falsify_rxb_003" + expected_output: "exit 0" + if_fails: "Distillation could train the student on the sealed test split." diff --git a/contracts/review-champion-challenger-v1.yaml b/contracts/review-champion-challenger-v1.yaml new file mode 100644 index 0000000000..3af8cac185 --- /dev/null +++ b/contracts/review-champion-challenger-v1.yaml @@ -0,0 +1,65 @@ +metadata: + version: 1.0.0 + kind: schema + description: | + PROMETHEUS (PRM-001, was REX-001) REX-11 champion/challenger, spec §5.3–§5.4. A + challenger (B1a prompt version, B1b retrieval few-shot, a B2 checkpoint) replaces the + serving champion only if all five §5.4 gates hold on the current sealed test version: + (1) McNemar exact one-sided p < ALPHA on per-item correctness, (2) precision point + estimate not lower, (3) H1 and H2 hold, (4) 0 contamination hits, (5) p95 ≤ the §4 + queue budget. Every gate fails closed: unmeasured is failed. Each decision spends one + of the version's 20 evaluations (§2.2); refusals (budget spent, no executed row (R-6), + no champion rows, challenger == champion) spend nothing. Promotion is a pin bump of + (weights, prompt, adapter) shas; the rollback target is the row's champion_before. + Demotion: two consecutive gold escapes where the champion said PASS and a voter said + FAIL. B1b retrieves from non-test gold only (R-2), never the query itself, and every + challenger prompt passes the contamination guard, which scans each fenced block alone + because a gold `- defect at …` line after a fence joins the last hunk and hides it + from a whole-text scan. + references: + - "docs/specifications/review-experiment-protocol.md" + - "https://github.com/paiml/aprender/issues/4366" + +equations: + promote: + formula: "promote ⇔ used < 20 ∧ ran ∧ p_McNemar(b,c) < α ∧ Δprecision ≥ 0 ∧ H1 ∧ H2 ∧ hits = 0 ∧ p95 ≤ budget" + demote: + formula: "demote ⇔ ∃ consecutive gold escapes e_i, e_{i+1}: champion_passed ∧ a_voter_failed" + +falsification_tests: + - id: FALSIFY-RXC-001 + name: all_five_gates_promote_and_each_alone_rejects + prediction: "b=20, c=0 with every gate measured green promotes; flipping any single gate (or leaving it unmeasured) rejects with exactly that gate's reason; more correct but less precise is rejected on gate 2" + test_harness: "cargo test -p aprender-review-experiment --lib falsify_rxc_001" + expected_output: "exit 0" + if_fails: "A challenger could be promoted with an unmeasured or failing gate." + - id: FALSIFY-RXC-002 + name: the_evaluation_budget_counts_and_refusals_spend_nothing + prediction: "the 21st decision on one test version is refused and spends nothing; a rotated version has its own budget; a challenger that never ran and the champion itself are refused; promotion moves the pin and records the rollback sha" + test_harness: "cargo test -p aprender-review-experiment --lib falsify_rxc_002" + expected_output: "exit 0" + if_fails: "The sealed test set could be over-fitted by unbounded challengers, or a never-run challenger counted." + - id: FALSIFY-RXC-003 + name: two_consecutive_gold_escapes_demote + prediction: "two consecutive gold champion-PASS/voter-FAIL escapes demote; a silver row neither breaks nor extends the run; a caught gold escape breaks it" + test_harness: "cargo test -p aprender-review-experiment --lib falsify_rxc_003" + expected_output: "exit 0" + if_fails: "A champion that keeps missing gold defects would keep serving." + - id: FALSIFY-RXC-004 + name: a_test_item_in_the_retrieval_pool_is_an_error + prediction: "select() on a pool containing a Split::Test item errors naming the item and R-2" + test_harness: "cargo test -p aprender-review-experiment --lib falsify_rxc_004" + expected_output: "exit 0" + if_fails: "Few-shot retrieval could show the model a sealed test diff." + - id: FALSIFY-RXC-005 + name: retrieval_ranks_by_similarity_and_never_returns_the_query + prediction: "the most similar dev item ranks first; a dev query retrieves neither its own id nor a byte-identical diff" + test_harness: "cargo test -p aprender-review-experiment --lib falsify_rxc_005" + expected_output: "exit 0" + if_fails: "A dev evaluation becomes open-book." + - id: FALSIFY-RXC-006 + name: a_rendered_prompt_carrying_a_test_hunk_is_refused + prediction: "a clean rendered prompt passes; the same prompt with a sealed item as an example is refused naming it, although the whole-text scan alone misses it; an empty index is refused" + test_harness: "cargo test -p aprender-review-experiment --lib falsify_rxc_006" + expected_output: "exit 0" + if_fails: "A B1 challenger prompt could carry a sealed test diff past the contamination guard." diff --git a/contracts/review-corpus-contamination-v1.yaml b/contracts/review-corpus-contamination-v1.yaml index 790babea85..9fd8ad0388 100644 --- a/contracts/review-corpus-contamination-v1.yaml +++ b/contracts/review-corpus-contamination-v1.yaml @@ -1,5 +1,5 @@ metadata: - version: 1.0.0 + version: 1.1.0 kind: schema description: | PROMETHEUS (PRM-001) sealed-test contamination check, spec §2.2 and rule R-2. A sealed @@ -8,14 +8,28 @@ metadata: JSON string values, so a diff stored in a JSONL field is seen unescaped. Training roots: docs/audits/review-corpus/train/ (contamination::TRAIN_ROOTS). Any hit is a hard FAIL (S-3). + v1.1 (PRA-001 T7, gate G-CON, spec §2.9.4): the exact checks miss a sealed diff that + was re-indented or had identifiers renamed, so a sealed item is also hit by CLUSTER. + Tokens drop all whitespace and blank lines, keep a +/- line marker, keep Rust + keywords and literals, and replace every identifier with $ (MOSS normalisation). + Diff header lines are skipped. A sealed item's sketch is the 64 smallest 12-token + shingle hashes (bottom-k MinHash; items under 32 shingles have none). A text holds + the item when >= 0.7 of that sketch is among its own shingles (containment, so a + diff inside a longer prompt counts). Sketches live in test-sketch-v1.txt, written by + `rex corpus-build` beside the prereg-locked manifest, which is unchanged. + False-positive measurement, 2026-09-25: 391 real .rs commit diffs from origin/main, + all 152,490 ordered pairs. At k=5, 469 pairs cluster; at k=12, 7 do (0.005%), all + inside one family of templated "beat" test files that are copies with renamed + identifiers, so over-refusing them fails safe. k=16 gives the same 7. references: - "docs/specifications/review-experiment-protocol.md" - "docs/audits/review-corpus/test-manifest-v1.txt" - "https://github.com/paiml/aprender/issues/4357" + - "https://github.com/paiml/aprender/issues/4354" equations: leak: - formula: "leak(f) = exists t in sealed: t.diff_sha256 in text(f) or hunks(t) ∩ hunks(text(f) ∪ json_strings(f)) != {}" + formula: "leak(f) = exists t in sealed: t.diff_sha256 in text(f) or hunks(t) ∩ hunks(text(f) ∪ json_strings(f)) != {} or |sketch(t) ∩ shingles(s)| / |sketch(t)| >= 0.7 for some s in text(f) ∪ json_strings(f)" falsification_tests: - id: FALSIFY-RCC-001 @@ -36,3 +50,21 @@ falsification_tests: test_harness: "cargo test -p aprender-review-experiment --lib falsify_rcc_003" expected_output: "exit 0" if_fails: "S-3: a sealed test item is in training data. STOP; rotate the leaked items out." + - id: FALSIFY-RCC-004 + name: perturbed_sealed_item_is_a_cluster_hit + prediction: "a sealed diff re-indented with tabs, re-spaced, with blank lines added, a partial identifier rename and a re-based header passes the exact checks, and with sketches is a cluster hit and a Refused pool row (also inside a JSONL prompt field, and with one line added as in a backport)" + test_harness: "cargo test -p aprender-review-experiment --lib falsify_rcc_004" + expected_output: "exit 0" + if_fails: "G-CON: a whitespace or rename perturbation smuggles a sealed test item into a training pool." + - id: FALSIFY-RCC-005 + name: unrelated_diffs_are_not_clustered + prediction: "unrelated diffs, including one with the sealed item's loop skeleton, stay below 0.7 containment; a diff under 32 shingles has no sketch" + test_harness: "cargo test -p aprender-review-experiment --lib falsify_rcc_005" + expected_output: "exit 0" + if_fails: "The cluster check refuses ordinary training rows, so the pool shrinks for no reason." + - id: FALSIFY-RCC-006 + name: sketch_file_round_trips_and_fails_closed + prediction: "render/parse round-trips; a wrong scheme line, bad hex or an empty sketch parses to None" + test_harness: "cargo test -p aprender-review-experiment --lib falsify_rcc_006" + expected_output: "exit 0" + if_fails: "A corrupt sketch file is read as sealing nothing." diff --git a/contracts/review-lane-perf-ratchet-v1.yaml b/contracts/review-lane-perf-ratchet-v1.yaml new file mode 100644 index 0000000000..b0435b2830 --- /dev/null +++ b/contracts/review-lane-perf-ratchet-v1.yaml @@ -0,0 +1,45 @@ +metadata: + version: 1.0.0 + kind: schema + description: | + PROMETHEUS (PRM-001, was REX-001) REX-10 perf ratchet, spec §5.1: review p95 on the + primary cell may never rise after 3 recorded tags. Each tag reruns the perf arm on the + primary cell over one fixed item set (full dev split + 30 fixed test items, timing + only) with the new apr tag and with llama.cpp. `rex ratchet record` turns the tag's + warm, non-rerun, admissible receipts into one `review-lane-perf-ratchet-v1` entry + (per-item wall ms, nearest-rank p95 and its bootstrap CI, llama.cpp p95, receipts + sha); `rex ratchet check` compares every tag after the third with the best earlier + tag by a paired bootstrap of the Harrell–Davis p95 ratio and exits 10 on RED. + Unpaired CI overlap (§4 tie rule) and a paired nearest-rank p95 both miss a +10 % + rise at the pilot's n; the only level used is the prereg's ALPHA (R-7). A tag with + no timings did not run and is not a recorded tag (R-6). A growing gap to llama.cpp + is a note to file (§9.1), not an andon. + references: + - "docs/specifications/review-experiment-protocol.md" + - "https://github.com/paiml/aprender/issues/4365" + +equations: + ratchet: + formula: "RED ⇔ ∃ i ≥ 3: lo_{1-α/2}( boot_paired( HD95(t_i) / HD95(t_best( 1 ∨ violations ≠ ∅" + state: + formula: "|tags| ≤ 3 ⇒ arming; |tags| > 3 ∧ ¬RED ⇒ green" + +falsification_tests: + - id: FALSIFY-RXR-001 + name: a_planted_ten_percent_p95_regression_is_red + prediction: "3 tags arm; an unchanged 4th tag is green; the same tag × 1.10 is RED in 10/10 independent runs with ≤ 1/10 false alarms; giving back a gain to the best tag is RED" + test_harness: "cargo test -p aprender-review-experiment --lib falsify_rxr_001" + expected_output: "exit 0" + if_fails: "A release could slow review p95 by 10 % and the ratchet would stay green." + - id: FALSIFY-RXR-002 + name: the_unpaired_overlap_rule_misses_ten_percent + prediction: "the two unpaired p95 CIs of a +10 % tag overlap at n = 130, so the overlap rule cannot be the ratchet's rule" + test_harness: "cargo test -p aprender-review-experiment --lib falsify_rxr_002" + expected_output: "exit 0" + if_fails: "The finding that forced the paired rule no longer holds; re-derive the rule." + - id: FALSIFY-RXR-003 + name: the_ratchet_fails_closed + prediction: "a duplicate tag, a foreign cell, or no common items is RED; record refuses an empty tag, two apr binaries under one tag, and a non-finite timing; a growing llama.cpp gap is noted" + test_harness: "cargo test -p aprender-review-experiment --lib falsify_rxr_003" + expected_output: "exit 0" + if_fails: "A malformed ratchet file or a tag that never ran would read as green." diff --git a/contracts/review-ledger-v1.yaml b/contracts/review-ledger-v1.yaml new file mode 100644 index 0000000000..94ce4ee907 --- /dev/null +++ b/contracts/review-ledger-v1.yaml @@ -0,0 +1,45 @@ +metadata: + version: 1.0.0 + kind: schema + description: | + PROMETHEUS (PRM-001, was REX-001) REX-07 day-0 shadow lane, spec §5.2/§7. The 4B + lane runs inside the quorum rail as a non-voting advisory lane (paiml-implement + advisory-lane.sh, #420), which records `.advisory_lane` in every quorum receipt. + `rex ledger REPO OUT RECEIPT...` (crates/aprender-review-experiment: ledger.rs) turns + receipts into `review-ledger-v1` rows (repo, ticket, head, PR, diff sha, counted lanes + with model ids, verdicts and finding counts, the shadow result, apr tag, weights sha, + outcome=pending until the ASL escape join) and measures shadow coverage. A receipt + with no advisory_lane, or with the lane `off`, is missing; a lane that did not answer + is Unknown{LaneUnavailable}, never a verdict; a shadow not recorded as uncounted, or a + width other than the counted lanes, is a violation. Exit 10 unless coverage holds. + superseded_by: "contracts/review-ledger-v2.yaml (writing); v1 rows stay readable" + references: + - "docs/specifications/review-experiment-protocol.md" + - "https://github.com/paiml/aprender/issues/4362" + - "https://github.com/paiml/paiml-implement/pull/420" + +equations: + coverage: + formula: "holds ⇔ |Q| > 0 ∧ ∀ q ∈ Q: shadow(q) ∈ {Answered, Unknown} ∧ counts(q) = false ∧ width(q) = |counted_lanes(q)|" + unknown: + formula: "shadow(q) ∈ {Verdict, NotRun, Refused} (the paiml-implement#436 wire form since 89d994bc1); the 402aff870 form state ≠ answered ⇒ Unknown{LaneUnavailable} is read-only, see review-ledger-v2" + +falsification_tests: + - id: FALSIFY-RXL-001 + name: a_quorum_without_a_shadow_row_is_missing + prediction: "a receipt with no advisory_lane, or with the lane off, is listed as missing and coverage fails; an empty or unparsable set never holds" + test_harness: "cargo test -p aprender-review-experiment --lib falsify_rxl_001" + expected_output: "exit 0" + if_fails: "Quorums could run without the shadow lane and REX-07's 100 % would still read green." + - id: FALSIFY-RXL-002 + name: gx10_down_is_unknown_and_keeps_the_width + prediction: "an unavailable lane, or an answered state with no verdict, is Unknown{LaneUnavailable}; the width stays 3" + test_harness: "cargo test -p aprender-review-experiment --lib falsify_rxl_002" + expected_output: "exit 0" + if_fails: "A gx10 outage would enter the ledger as a verdict and contaminate H4/H5 agreement." + - id: FALSIFY-RXL-003 + name: a_counted_or_width_changing_shadow_is_a_violation + prediction: "counts=true, a missing counts flag, or a width other than the counted lanes fails coverage" + test_harness: "cargo test -p aprender-review-experiment --lib falsify_rxl_003" + expected_output: "exit 0" + if_fails: "The non-voting lane could start voting before REX-09's promotion gates." diff --git a/contracts/review-ledger-v2.yaml b/contracts/review-ledger-v2.yaml new file mode 100644 index 0000000000..858db39be9 --- /dev/null +++ b/contracts/review-ledger-v2.yaml @@ -0,0 +1,82 @@ +metadata: + version: 1.0.0 + kind: schema + description: | + PRA-001 T2 (spec docs/specifications/research-design-pr-agent-qwen-3.5.md §7), on top of + PRM-001 REX-07. `rex ledger` (crates/aprender-review-experiment: ledger.rs, ledger_v1.rs) + writes `review-ledger-v2` rows. v1 kept findings as a count, so each quorum lost its + (input, output, findings) triple. v2 keeps each counted lane's findings text verbatim + plus its agent-trace-v1 input/output shas. Text never captured is None, never an empty + list. The apr shadow row is the typed paiml-implement#436 wire form (Verdict | NotRun | + Refused). Outcomes are appended as OutcomeJoin lines and never rewrite a row. v1 rows + (both shapes ever written under that name) still parse through read_row, and a + typed-shadow v1 row migrates with its count kept and its text None. Supersedes + review-ledger-v1 for writing; v1 stays for reading. + references: + - "docs/specifications/research-design-pr-agent-qwen-3.5.md" + - "docs/specifications/review-experiment-protocol.md" + - "contracts/review-ledger-v1.yaml" + - "https://github.com/paiml/aprender/issues/4354" + - "https://github.com/paiml/paiml-implement/issues/436" + +equations: + coverage: + formula: "holds ⇔ |Q| > 0 ∧ missing = ∅ ∧ violations = ∅ ∧ identity_gaps = 0 ∧ trace_gaps = 0" + shadow: + formula: "shadow(q) ∈ {Verdict{verdict ∈ {PASS,FAIL}, cell, backend}, NotRun{NoExecutor|Busy|Timeout|ContextOverflow|TrainActive}, Refused{cell, removed_by ∈ {ladder,gpu-proof,parse}}}; any other value ⇒ violation" + trace_gap: + formula: "gap(l) ⇔ findings(l) = None ∨ input_sha256(l) = None ∨ output_sha256(l) = None" + outcome: + formula: "outcome(r, J) = at-max { j ∈ J : j.repo = r.repo ∧ j.head = r.head }.outcome, else r.outcome; r is never rewritten" + migration: + formula: "migrate(v1) = v2 with findings_count = v1.findings ∧ findings = None ∧ shas = None; shadow(v1) legacy ⇒ Err" + +falsification_tests: + - id: FALSIFY-RXL-001 + name: a_quorum_without_a_shadow_row_is_missing + prediction: "no advisory_lane, or the lane off, is missing; an empty set never holds" + test_harness: "cargo test -p aprender-review-experiment --lib falsify_rxl_001" + expected_output: "exit 0" + if_fails: "Quorums could run without the shadow lane and REX-07 would still read green." + - id: FALSIFY-RXL-002 + name: gx10_down_is_notrun_and_keeps_the_width + prediction: "a lane that did not run is NotRun, never a verdict; the width stays the counted lanes" + test_harness: "cargo test -p aprender-review-experiment --lib falsify_rxl_002" + expected_output: "exit 0" + if_fails: "A gx10 outage would enter the ledger as a verdict." + - id: FALSIFY-RXL-003 + name: a_counted_or_width_changing_shadow_is_a_violation + prediction: "counts=true, a missing counts flag, or a width other than the counted lanes fails coverage" + test_harness: "cargo test -p aprender-review-experiment --lib falsify_rxl_003" + expected_output: "exit 0" + if_fails: "The non-voting lane could start voting before promotion." + - id: FALSIFY-RXL-004 + name: a_receipt_without_a_weights_sha_does_not_hold + prediction: "a row with no apr tag or weights sha is an identity gap and coverage fails" + test_harness: "cargo test -p aprender-review-experiment --lib falsify_rxl_004" + expected_output: "exit 0" + if_fails: "Shadow rows could not be tied to the weights that produced them." + - id: FALSIFY-RXL-005 + name: a_free_text_reason_fails_to_parse + prediction: "an apr row outside the #436 wire form (free text, unknown key or value) is a violation" + test_harness: "cargo test -p aprender-review-experiment --lib falsify_rxl_005" + expected_output: "exit 0" + if_fails: "Free-text reasons would re-enter the ledger and split the spelling from receipt-lint." + - id: FALSIFY-RXL-006 + name: a_counted_lane_without_its_trace_does_not_hold + prediction: "a counted lane missing findings text, input_sha256 or output_sha256 is a trace gap; the row is kept and coverage fails" + test_harness: "cargo test -p aprender-review-experiment --lib falsify_rxl_006" + expected_output: "exit 0" + if_fails: "v2 would silently degrade to v1's count-only capture and the training triples would be lost again." + - id: FALSIFY-RXL-007 + name: an_outcome_join_appends_and_never_rewrites + prediction: "the latest join by `at` for (repo, head) wins; other heads are ignored; the row is byte-identical after" + test_harness: "cargo test -p aprender-review-experiment --lib falsify_rxl_007" + expected_output: "exit 0" + if_fails: "Outcome labels could overwrite history, and gold labels would lose their provenance." + - id: FALSIFY-RXL-008 + name: v1_rows_parse_and_migrate_without_invented_text + prediction: "both v1 shapes parse; a typed v1 row migrates with its count kept and findings None (a trace gap); a legacy shadow is refused; a v1 body under the v2 name, or an unknown schema, is an error" + test_harness: "cargo test -p aprender-review-experiment --lib falsify_rxl_008" + expected_output: "exit 0" + if_fails: "Old rows would stop reading, or migration would invent an empty findings list that trains as 'no issues'." diff --git a/contracts/review-replay-v1.yaml b/contracts/review-replay-v1.yaml new file mode 100644 index 0000000000..2fb9f7fd24 --- /dev/null +++ b/contracts/review-replay-v1.yaml @@ -0,0 +1,72 @@ +metadata: + version: 1.1.0 + kind: schema + description: | + PRM-S1 (PRM-001 v3 "PROMETHEUS" §2.4 review-replay-v1, §9 speed block; Refs #4354, #4447). + crates/aprender-review-experiment/src/replay.rs builds a frozen replay set of real quorum + diffs (INPUTS only, never a lane's output), defines the review-replay-receipt-v1 row, and + folds one run into the §9 speed block. `examples/replay.rs` is the runner, and it + drives `apr serve` and llama.cpp d1d3c3396 `llama-server` with the same GGUF on the same cell. + Strata by composed input tokens: 2k (0,2048], 8k (2048,8192], 16k (8192,16384], + 32k (16384,32768], 50 items each. One item per repo#PR and per diff sha. Sealed test items + (diff sha, hunk fingerprint, near-dup cluster) are excluded. Order within a stratum is + sha256(seed ‖ diff_sha), so the set is frozen by (version, seed, candidates). A short stratum is an + error and is never padded. Every ratio is apr / llama_cpp: TTFT targets ≤ 2.0 by E2, and prefill and + decode target ≥ 1.0 by E6. The queue budget is the p95 of the slowest counted voter, taken from + captured quorum latency. A summary refuses an inconsistent run rather than print a number. + references: + - "docs/specifications/review-experiment-protocol.md" + +equations: + stratum: + formula: "stratum(t) = min{ s ∈ (2k, 8k, 16k, 32k) : 0 < t ≤ upper(s) }, undefined for t = 0 or t > 32768" + ratio: + formula: "ratio(m) = median_apr(m) / median_llama_cpp(m); null if any row of either engine has m null" + queue_budget: + formula: "budget = max over counted voter lanes L of p95(latency_L); within_budget ⇔ p95(apr wall) ≤ budget" + verdict_identity: + formula: "identity = |{ i : verdict_tag(i) = verdict_prev_tag(i) }| / n, only on the same set_sha and the same items" + +falsification_tests: + - id: FALSIFY-REPLAY-001 + name: sealed_items_never_enter_the_set + prediction: "a sealed diff, or the same diff re-posted under another repo#PR, is dropped, and its stratum reports short rather than silently shrinking" + test_harness: "cargo test -p aprender-review-experiment --lib falsify_replay_001" + expected_output: "exit 0" + if_fails: "Sealed test diffs would be replayed and cached by a benchmark run. That is the leak §2.4 forbids." + - id: FALSIFY-REPLAY-002 + name: short_strata_are_refused_not_padded + prediction: "a stratum with fewer than per_stratum eligible items is an error that names it; an out-of-range (>32768) item is never moved into a stratum" + test_harness: "cargo test -p aprender-review-experiment --lib falsify_replay_002" + expected_output: "exit 0" + if_fails: "A 32k stratum filled from 16k inputs would report long-context prefill that was never measured." + - id: FALSIFY-REPLAY-003 + name: one_item_per_pr_and_order_free + prediction: "the set has one item per repo#PR and per diff sha, is identical under reversed candidate order, and changes with the seed" + test_harness: "cargo test -p aprender-review-experiment --lib falsify_replay_003" + expected_output: "exit 0" + if_fails: "One large PR could dominate a stratum, or re-collecting candidates could silently change the frozen set." + - id: FALSIFY-REPLAY-004 + name: ratios_are_apr_over_llama_and_budget_is_slowest_voter + prediction: "TTFT, per-stratum prefill and decode ratios equal apr/llama_cpp medians; the queue budget is the largest voter-lane p95; a single null server timing nulls the figure" + test_harness: "cargo test -p aprender-review-experiment --lib falsify_replay_004" + expected_output: "exit 0" + if_fails: "An inverted ratio would print a scoreboard win for a loss, and a subset median would hide the rows the server could not time." + - id: FALSIFY-REPLAY-005 + name: inconsistent_runs_never_print_a_number + prediction: "mixed set_sha or version, a GGUF, cell or apr_tag mismatch, unequal engine coverage, a duplicate row, any not-run row, or a missing voter latency is refused" + test_harness: "cargo test -p aprender-review-experiment --lib falsify_replay_005" + expected_output: "exit 0" + if_fails: "apr on one GGUF against llama.cpp on another, or apr with its timeouts dropped, would be reported as parity." + - id: FALSIFY-REPLAY-006 + name: verdict_identity_is_same_set_item_by_item + prediction: "identity vs the previous tag is the per-item equal-verdict fraction on the same set_sha and items; another set or missing items refuse" + test_harness: "cargo test -p aprender-review-experiment --lib falsify_replay_006" + expected_output: "exit 0" + if_fails: "A tag could change review verdicts while the speed block reported only faster wall-clock." + - id: FALSIFY-REPLAY-007 + name: v2_pins_the_prompt_both_engines_prefill + prediction: "a review-replay-v2 request sends chat_template_kwargs.enable_thinking=false and a v1 request is unchanged; a v2 summary refuses any row without a prompt-ids sha and any item whose apr and llama_cpp prompt-ids shas differ; the ids sha is order-sensitive" + test_harness: "cargo test -p aprender-review-experiment --lib falsify_replay_007" + expected_output: "exit 0" + if_fails: "Set b8a58011 (v1) compared llama.cpp decoding Qwen3.5 in thinking mode against apr answering directly: two different prompts, reported as one speed ratio." diff --git a/contracts/rex-cell-admission-v1.yaml b/contracts/rex-cell-admission-v1.yaml index a6d1ce2aca..c33dd9d6c1 100644 --- a/contracts/rex-cell-admission-v1.yaml +++ b/contracts/rex-cell-admission-v1.yaml @@ -1,5 +1,5 @@ metadata: - version: 1.0.0 + version: 1.1.0 kind: schema description: | PROMETHEUS (PRM-001, was REX-001) REX-04 per-cell admission, spec §2.1/§7. One JSONL @@ -21,6 +21,8 @@ equations: formula: "∀ c ∈ CELLS: |rows(c)| = 1 ∧ state(c) ∈ {Admitted, Refused, NotRun}" admitted: formula: "Admitted(c) ⇒ cosine(c) ≥ threshold(c) ∧ receipt_sha256(c) is 64-hex ∧ threshold_basis(c) ≠ ∅" + derived: + formula: "parity_from_receipt(R, x) = Ok(p) ⇒ n(R) ≥ x.min_positions ∧ p.cosine = min_i cos_i(R) ≥ x.threshold ∧ p.receipt_sha256 = sha256(R) ∧ sha(R).binary = x.apr_sha256 (when R carries one) ∧ sha(R).weights = x.weights_sha256 ∧ oracle(R) ∈ {llama.cpp@d1d3c3396, apr-parity-gpu-cpu}" s7: formula: "S-7 ⇔ |{c : Admitted(c)}| = 0" @@ -43,3 +45,21 @@ falsification_tests: test_harness: "cargo test -p aprender-review-experiment --lib falsify_rca_003" expected_output: "exit 0" if_fails: "The experiment would proceed to REX-05 with no admissible cell for (A)." + - id: FALSIFY-RCA-004 + name: admitted_is_derived_from_the_receipt + prediction: "`rex admit --parity-receipt R` fills Admitted.parity from R: the minimum cosine over R's positions (never its summary `cosine`), sha256 of R's bytes, R's oracle, and the DECLARED threshold and basis (never R's own); exactly min_positions and exactly the threshold admit" + test_harness: "cargo test -p aprender-review-experiment --lib falsify_rca_004" + expected_output: "exit 0" + if_fails: "An Admitted row would carry a hand-typed cosine or the receipt's self-chosen threshold instead of the measured worst position." + - id: FALSIFY-RCA-005 + name: a_short_or_failing_receipt_does_not_admit + prediction: "fewer than min_positions (64, evidence/parity/thresholds.yaml, I8/CF-4 #1864) positions, exit ≠ 0, a non-passing verdict, failed > 0, any position below threshold, a missing or non-numeric cosine (JSON carries no NaN), an undeclared oracle, an empty basis or a non-finite threshold is refused with the reason named" + test_harness: "cargo test -p aprender-review-experiment --lib falsify_rca_005" + expected_output: "exit 0" + if_fails: "A 7-position smoke check would admit a cell to the hardware ruling that parity evidence requires 64 positions for." + - id: FALSIFY-RCA-006 + name: the_receipt_must_be_the_rows_binary_and_weights + prediction: "a passing receipt whose binary_sha256 or weights sha256 differs from the row's apr_sha256 / weights_sha256 (or is absent) is refused" + test_harness: "cargo test -p aprender-review-experiment --lib falsify_rca_006" + expected_output: "exit 0" + if_fails: "Parity measured on another build or another model would admit this cell." diff --git a/contracts/rex-pilot-projection-v1.yaml b/contracts/rex-pilot-projection-v1.yaml new file mode 100644 index 0000000000..53677d5f51 --- /dev/null +++ b/contracts/rex-pilot-projection-v1.yaml @@ -0,0 +1,44 @@ +metadata: + version: 1.0.0 + kind: schema + description: | + PROMETHEUS (PRM-001, was REX-001) REX-05 pilot projection, spec §2.2/§7. `rex score + --pilot --split dev` (crates/aprender-review-experiment: pilot.rs) projects the + test-split Wilson 95 % half-width on recall at the pilot's point estimate (or at the + §2.2 design p = 0.5 when the pilot has no defect item in its denominator) and applies + the sample-size rule: above 0.12 the corpus grows and the hypotheses do not change. + An uncomputable width (no test defect items) also grows the corpus. The runtime + projection counts the 10 % determinism re-run and is None, never zero, when nothing + warm ran. The spec's "about 0.117" at 70 defects is the Wald width; the frozen + stats.rs Wilson width the rule reads is about 0.114. + references: + - "docs/specifications/review-experiment-protocol.md" + - "https://github.com/paiml/aprender/issues/4360" + +equations: + half_width: + formula: "h = wilson_half_width(round(p · n_def), n_def), p = recall_pilot if defined else 0.5" + rule: + formula: "grow_corpus ⇔ h undefined ∨ h > 0.12" + runtime: + formula: "requests = n_test + ⌈n_test / 10⌉; runtime_s = requests · mean(warm_ms) / 1000, undefined if no warm sample" + +falsification_tests: + - id: FALSIFY-RXP-001 + name: a_wide_projection_grows_the_corpus + prediction: "a projected half-width above 0.12, or one that cannot be computed, sets grow_corpus" + test_harness: "cargo test -p aprender-review-experiment --lib falsify_rxp_001" + expected_output: "exit 0" + if_fails: "The full run would start on a corpus too small to resolve the §2.2 interval." + - id: FALSIFY-RXP-002 + name: the_projection_uses_the_pilot_estimate + prediction: "the rule reads the pilot's recall, not a fixed p; 9/10 at n=50 keeps the corpus and 5/10 grows it" + test_harness: "cargo test -p aprender-review-experiment --lib falsify_rxp_002" + expected_output: "exit 0" + if_fails: "The sample-size decision would ignore the pilot it exists to read." + - id: FALSIFY-RXP-003 + name: runtime_counts_the_rerun_and_nothing_warm_is_unknown + prediction: "145 test items project 160 requests; no warm sample or a non-finite one projects None" + test_harness: "cargo test -p aprender-review-experiment --lib falsify_rxp_003" + expected_output: "exit 0" + if_fails: "The runtime budget would under-count the re-run or report zero seconds for a cell that never ran." diff --git a/contracts/rex-prereg-v1.yaml b/contracts/rex-prereg-v1.yaml index 830a420991..401e5400f4 100644 --- a/contracts/rex-prereg-v1.yaml +++ b/contracts/rex-prereg-v1.yaml @@ -1,30 +1,37 @@ metadata: - version: 1.0.0 + version: 3.0.0 kind: schema description: | - REX-001 pre-registration lock (rule R-1). Spec §2–§5, the analysis code + PRM-001 pre-registration lock (rule R-1). Spec §2–§5, the analysis code (crates/aprender-review-experiment/src/stats.rs), the analysis plan and prompt v1 - are frozen by sha256 in docs/audits/rex-001/prereg.lock. The prereg sha is carried - by every review-experiment receipt; a tree that no longer matches the lock is a - new spec version and its data is exploratory. + are frozen by sha256 in a lock file. The prereg sha is carried by every + review-experiment receipt; a tree that no longer matches the lock is a new spec + version and its data is exploratory. The contract id keeps its v1 name. + - rex-prereg-v1 (REX-00, 5f0ae10ac): docs/audits/rex-001/prereg.lock over + review-experiment-protocol.md. SUPERSEDED, never deleted or edited (operator + 2026-09-25); its spec §2–§5 and plan stay byte-frozen. + - rex-prereg-v2 (PRM-00b): docs/audits/prm-001/prereg.lock over PRM-001 v3 + (docs/specifications/PRM-001-prometheus.md) and analysis plan v2. v1 had 0 data rows. + The 17:09Z v2 draft lock (1f4acb3d) over the v2 ruling was a stray draft and is void. references: + - "docs/specifications/PRM-001-prometheus.md" + - "docs/audits/prm-001/analysis-plan.md" - "docs/specifications/review-experiment-protocol.md" - - "docs/audits/rex-001/analysis-plan.md" - "https://github.com/paiml/aprender/issues/4355" equations: prereg_sha: - formula: "sha256('rex-prereg-v1\\n' || 'spec_s2_s5 ' || H(spec[§2..§6)) || '\\nstats_rs ' || H(stats.rs) || '\\nanalysis_plan ' || H(plan) || '\\nprompt_v1 ' || H(prompt) || '\\n')" + formula: "sha256('rex-prereg-v2\\n' || 'spec_s2_s5 ' || H(spec[§2..§6)) || '\\nstats_rs ' || H(stats.rs) || '\\nanalysis_plan ' || H(plan) || '\\nprompt_v1 ' || H(prompt) || '\\n')" locked: formula: "prereg.lock[k] == H(component_k) for every k, and prereg.lock[prereg_sha] == prereg_sha" falsification_tests: - id: FALSIFY-REX-PREREG-001 name: lock_matches_tree - prediction: "the committed lock matches every frozen component" + prediction: "the committed v2 lock matches every frozen component" test_harness: "cargo test -p aprender-review-experiment --lib falsify_rex_prereg_001" expected_output: "exit 0" - if_fails: "A frozen component changed after REX-00. That is a new spec version (R-1): label prior data exploratory and cut v2; never edit the lock to match." + if_fails: "A frozen component changed after PRM-00b. That is a new spec version (R-1): label prior data exploratory and cut v3; never edit the lock to match." - id: FALSIFY-REX-PREREG-002 name: planted_section3_edit_is_caught prediction: "an edit to §3 (hypotheses) changes the prereg sha" @@ -33,7 +40,13 @@ falsification_tests: if_fails: "The §2–§5 span no longer covers §3; the lock is vacuous." - id: FALSIFY-REX-PREREG-003 name: out_of_scope_edit_is_not_a_version - prediction: "an edit to the status line (outside §2–§5) leaves the prereg sha unchanged" + prediction: "an edit to the header (outside §2–§5) leaves the prereg sha unchanged" test_harness: "cargo test -p aprender-review-experiment --lib falsify_rex_prereg_003" expected_output: "exit 0" - if_fails: "The span leaks past §6; routine status edits would invalidate data." + if_fails: "The span leaks past §6; routine header edits would invalidate data." + - id: FALSIFY-REX-PREREG-004 + name: v1_lock_is_frozen_and_verifies + prediction: "the v1 lock still carries prereg_sha ef51087d… and its spec §2–§5, plan and prompt digests match the in-tree v1 files" + test_harness: "cargo test -p aprender-review-experiment --lib falsify_rex_prereg_004" + expected_output: "exit 0" + if_fails: "A superseded lock or the v1 bytes it froze were edited (operator: never deleted or edited)." diff --git a/contracts/rex-promotion-ladder-v1.yaml b/contracts/rex-promotion-ladder-v1.yaml new file mode 100644 index 0000000000..76c05f0b48 --- /dev/null +++ b/contracts/rex-promotion-ladder-v1.yaml @@ -0,0 +1,48 @@ +metadata: + version: 2.0.0 + kind: schema + description: | + SUPERSEDED by rex-promotion-ladder-v2 (PRM-09, the tie-breaker rung); kept as the v2-spec record. + PROMETHEUS (PRM-001, was REX-001) REX-09 promotion ladder, spec §3 H4/H5, §4.5, R-11. + `rex ladder REPORT_JSON` (crates/aprender-review-experiment: ladder.rs) reads only a + `rex-001-report-v2` report (spec v2) whose prereg_sha is the lock and that is not exploratory, + and returns the 4B lane's rung: shadow by default; tripwire when H4 holds (explicit + `holds` verdict and a finite Holm-adjusted one-sided p ≤ 0.05; the CI is report-only); vote when H4 + and H5 both hold. H5 without H4 stays shadow. The §4.5 hardware ruling mode caps the + rung, and a report without one is shadow. The paiml-implement quorum gate and arbiter + `decide` consume `.mode`; they never re-derive it. + references: + - "docs/specifications/review-experiment-protocol.md" + - "https://github.com/paiml/aprender/issues/4364" + +equations: + ladder: + formula: "mode = min(ruling.mode, H4 ∧ H5 ? vote : H4 ? tripwire : shadow)" + holds: + formula: "holds(H) ⇔ verdict(H) = holds ∧ p_holm(H) finite ∧ 0 ≤ p_holm(H) ≤ 0.05" + +falsification_tests: + - id: FALSIFY-RXG-001 + name: a_failing_h4_keeps_the_lane_in_shadow + prediction: "the planted report with H4 failing, a holds verdict with Holm p > 0.05, NaN, negative or missing, or H4 absent all stay shadow" + test_harness: "cargo test -p aprender-review-experiment --lib falsify_rxg_001" + expected_output: "exit 0" + if_fails: "The 4B lane would enter tripwire without evidence its precision matches the voters (R-11)." + - id: FALSIFY-RXG-002 + name: h4_alone_is_tripwire_never_vote + prediction: "H4 with H5 failing is tripwire; a shadow hardware ruling or no ruling caps both gates passing at shadow" + test_harness: "cargo test -p aprender-review-experiment --lib falsify_rxg_002" + expected_output: "exit 0" + if_fails: "The lane could cast a deciding vote without the recall gate or lengthen the quorum past the queue budget." + - id: FALSIFY-RXG-003 + name: only_a_locked_confirmatory_report_is_read + prediction: "a v1 or foreign schema, a stale prereg sha, an exploratory report or unparsable JSON stays shadow" + test_harness: "cargo test -p aprender-review-experiment --lib falsify_rxg_003" + expected_output: "exit 0" + if_fails: "A promotion could be read from a report outside the pre-registration (R-1)." + - id: FALSIFY-RXG-004 + name: the_ci_is_report_only + prediction: "a CI crossing 0 beside Holm p ≤ 0.05 is vote; a clean CI beside Holm p > 0.05 is shadow" + test_harness: "cargo test -p aprender-review-experiment --lib falsify_rxg_004" + expected_output: "exit 0" + if_fails: "Two H4 decision rules would coexist and could disagree (PRM-001 v2 ruling item 3c)." diff --git a/contracts/rex-promotion-ladder-v2.yaml b/contracts/rex-promotion-ladder-v2.yaml new file mode 100644 index 0000000000..341a4a2ca6 --- /dev/null +++ b/contracts/rex-promotion-ladder-v2.yaml @@ -0,0 +1,84 @@ +metadata: + version: 1.1.0 + kind: schema + description: | + PRM-09 (PRM-001 v3 "PROMETHEUS" §5.4, §7, §9): the promotion ladder with the tie-breaker rung. + Supersedes rex-promotion-ladder-v1. `rex ladder REPORT_JSON [--current RUNG]` + (crates/aprender-review-experiment: ladder.rs, examples/rex.rs) reads only a `prm-001-report-v1` + report (the §9 template: spec PRM-001, version 3) whose prereg_sha is the lock and that is not + exploratory. Anything else, including the older rex-001-report-v2, is read as no evidence (shadow). + Rungs, lowest first: shadow < tripwire < tie-breaker < vote. + tripwire ⇔ H4 ∧ speed.within_budget (replay p95 ≤ the queue budget). + tie-breaker ⇔ tripwire ∧ H5 ∧ H7 ∧ H1 ∧ H2 ∧ lane.availability_7d ≥ 0.95. It decides only a 1–1 split. + vote ⇔ tie-breaker ∧ promotion.tie_breaks_decided ≥ 20 ∧ + promotion.tie_break_outcome_agreement ≥ promotion.voter_outcome_agreement. + H4 and H5 hold on a `holds` verdict with a finite Holm-adjusted p in [0, 0.05]; the CI is report-only. + H1 and H2 (count tests) and H7 (δ-free, outside the Holm family) hold on a `holds` verdict. + hardware_ruling.rung_eligible caps the rung; a report without a ruling is shadow. + A primary cell that is lambda, or is not named, caps the lane at tripwire (R-9, STOP S-4). + The ladder is sequential: from the current rung it moves at most one rung up, and a demotion + drops exactly one rung. The S-4 cap is the only exception and applies at once. + The paiml-implement quorum gate and the arbiter read `.mode`; they never re-derive it. + references: + - "docs/specifications/PRM-001-prometheus.md" + - "contracts/rex-promotion-ladder-v1.yaml" + +equations: + evidence: + formula: "e = H4 ∧ fast ? (H5 ∧ H7 ∧ H1 ∧ H2 ∧ avail ≥ 0.95 ? (n_tb ≥ 20 ∧ agree_tb ≥ agree_voters ? vote : tie-breaker) : tripwire) : shadow" + cap: + formula: "c = min(e, ruling.rung_eligible, primary is lambda or absent ? tripwire : vote); no ruling ⇒ c = shadow" + step: + formula: "mode = primary is lambda or absent ∧ current > tripwire ? tripwire : clamp(c, current − 1, current + 1)" + holds: + formula: "holds(H4|H5) ⇔ verdict = holds ∧ p_holm finite ∧ 0 ≤ p_holm ≤ 0.05; holds(H1|H2|H7) ⇔ verdict = holds" + +falsification_tests: + - id: FALSIFY-RXG-001 + name: a_failing_h4_keeps_the_lane_in_shadow + prediction: "the planted report with H4 failing, a holds verdict with Holm p > 0.05, NaN, negative or missing, or H4 absent all stay shadow" + test_harness: "cargo test -p aprender-review-experiment --lib falsify_rxg_001" + expected_output: "exit 0" + if_fails: "The lane would enter tripwire without evidence its precision matches the voters." + - id: FALSIFY-RXG-002 + name: h4_alone_is_tripwire_never_vote + prediction: "the planted H4-only report is tripwire; H5 failing names H5 and stays tripwire; within_budget false or absent is shadow; rung_eligible caps; a ruling without rung_eligible, or no ruling, is shadow" + test_harness: "cargo test -p aprender-review-experiment --lib falsify_rxg_002" + expected_output: "exit 0" + if_fails: "A lane over the queue budget, or with only H4, would cast deciding votes." + - id: FALSIFY-RXG-003 + name: only_a_locked_confirmatory_report_is_read + prediction: "a rex-001-report-v2 report, a version-2 spec, a stale prereg_sha, an exploratory report and non-JSON are all shadow" + test_harness: "cargo test -p aprender-review-experiment --lib falsify_rxg_003" + expected_output: "exit 0" + if_fails: "An unlocked or exploratory analysis could promote the lane." + - id: FALSIFY-RXG-004 + name: the_ci_is_report_only + prediction: "a CI crossing 0 with Holm p ≤ 0.05 still holds; a clean CI with Holm p > 0.05 does not" + test_harness: "cargo test -p aprender-review-experiment --lib falsify_rxg_004" + expected_output: "exit 0" + if_fails: "The ladder would decide on a statistic the prereg made report-only." + - id: FALSIFY-RXG-005 + name: tie_breaker_needs_h4_h5_h7_h1_h2_and_availability + prediction: "the planted H4+H5+H7 report (H1, H2 holding, availability 0.97) is tie-breaker; without or failing any of H5, H7, H1, H2 it is tripwire and names it (H5 also needs Holm p ≤ 0.05); availability 0.95 is tie-breaker, 0.9499 or absent is tripwire" + test_harness: "cargo test -p aprender-review-experiment --lib falsify_rxg_005" + expected_output: "exit 0" + if_fails: "A lane that is unavailable or nondeterministic would break ties in the quorum." + - id: FALSIFY-RXG-006 + name: lambda_primary_is_refused_above_tripwire + prediction: "a report passing every gate with hardware_ruling.primary lambda-cuda is tripwire with an S-4 reason naming the cell as written (lambda-cuda, never a Rust Debug Some(..)); a ruling without a primary is also tripwire and is named (unnamed)" + test_harness: "cargo test -p aprender-review-experiment --lib falsify_rxg_006" + expected_output: "exit 0" + if_fails: "The operator's desktop GPU would sit on the merge path (R-9)." + - id: FALSIFY-RXG-007 + name: vote_needs_twenty_agreeing_tie_breaks + prediction: "19 decided tie-breaks, tie-break agreement below the voters', no promotion block, or no voter agreement all stay tie-breaker" + test_harness: "cargo test -p aprender-review-experiment --lib falsify_rxg_007" + expected_output: "exit 0" + if_fails: "The lane would vote on ties it never decided, or decided worse than the voters." + - id: FALSIFY-RXG-008 + name: never_skips_a_rung_and_demotes_one + prediction: "full evidence from shadow → tripwire, tripwire → tie-breaker, tie-breaker → vote; no evidence from vote → tie-breaker, tripwire → shadow; a tripwire-only report takes vote to tie-breaker; the lambda cap takes vote or tie-breaker to tripwire at once, never promotes a shadow lane past its evidence" + test_harness: "cargo test -p aprender-review-experiment --lib falsify_rxg_008" + expected_output: "exit 0" + if_fails: "One good report would jump the lane to vote, or one bad report would drop it to shadow." diff --git a/contracts/sparse-logits-v1.yaml b/contracts/sparse-logits-v1.yaml new file mode 100644 index 0000000000..c0855f8196 --- /dev/null +++ b/contracts/sparse-logits-v1.yaml @@ -0,0 +1,55 @@ +metadata: + version: 1.0.0 + kind: schema + description: | + PRA-001 T11 (spec §2.7): the `sparse-logits-v1` teacher-logit blob written beside apr serve + rows (crates/aprender-review-experiment: sparse_logits.rs). Per token it stores the sampled + id, the top-k ids (u32) with log-softmax at T = 1.0 before any sampling filter (f16), the + full-vocab logsumexp (f32), and the retained mass M = Σexp(logprobs) (f16) so that 1 − M is + the true tail mass a sparse forward-KL correction needs. Header (closed JSON): schema, mode + (topk | rs-sample), model_sha256, tokenizer_sha256, apr_tag, backend (served_by), + temperature_of_record, k, n_tokens. Body: one zstd frame of LE columns. Raw column size is + n·(10 + 6k) bytes = 130 B/token at k = 20. Frontier lanes carry no blob (logits_sha = null). + Encoder and decoder share one validator: a blob that exists is a blob that reads. + references: + - "https://github.com/paiml/aprender/issues/4026" + +equations: + logprob: + formula: "lp_i = z_i − logsumexp(z), z at T = 1.0, lp_i ≤ 0" + mass: + formula: "M = Σ_{i ∈ topk} exp(lp_i) ∈ [0, 1], |M − Σexp(f16(lp_i))| ≤ 2^-8; residual = 1 − M" + raw_bytes: + formula: "raw(n, k) = n · (4 + 6k + 4 + 2)" + +falsification_tests: + - id: FALSIFY-SPL-001 + name: round_trip_is_exact + prediction: "header and every column (incl. ids > 65535) decode bit-identical to what was encoded" + test_harness: "cargo test -p aprender-review-experiment --lib falsify_spl_001" + expected_output: "exit 0" + if_fails: "Stored teacher distributions would differ from the served ones; Qwen ids above u16 would alias." + - id: FALSIFY-SPL-002 + name: residual_mass_is_the_true_tail + prediction: "1 − M equals the full-softmax mass outside the top-k within 1e-3, and survives the round trip" + test_harness: "cargo test -p aprender-review-experiment --lib falsify_spl_002" + expected_output: "exit 0" + if_fails: "A top-k cache without its tail mass is a biased teacher estimate (§2.7)." + - id: FALSIFY-SPL-003 + name: inconsistent_mass_is_refused + prediction: "a mass disagreeing with its own logprobs is refused by encode AND by decode of a forged blob" + test_harness: "cargo test -p aprender-review-experiment --lib falsify_spl_003" + expected_output: "exit 0" + if_fails: "A producer defect (post-filter mass, wrong temperature) would enter the cache silently." + - id: FALSIFY-SPL-004 + name: decoder_is_strict + prediction: "an unknown header key, an n_tokens that mismatches the columns, any truncation, a trailing byte or a bad magic is an error" + test_harness: "cargo test -p aprender-review-experiment --lib falsify_spl_004" + expected_output: "exit 0" + if_fails: "A malformed blob would decode to a plausible but wrong distribution." + - id: FALSIFY-SPL-005 + name: capture_round_trips_through_the_cas + prediction: "an apr serve body captured by `capture` is stored at blobs/sha256/ab/cd/.zst, its frame declares the raw size, it decodes to the same tokens with the same residual mass, a second capture is idempotent, a forged stored blob is refused, and a refused body writes nothing" + test_harness: "cargo test -p aprender-review-experiment --lib falsify_spl_005" + expected_output: "exit 0" + if_fails: "A row's logits_sha would name a blob that is absent, unreadable, or not the distribution the serve returned." diff --git a/contracts/trace-admission-prov-v1.yaml b/contracts/trace-admission-prov-v1.yaml new file mode 100644 index 0000000000..1878367c87 --- /dev/null +++ b/contracts/trace-admission-prov-v1.yaml @@ -0,0 +1,48 @@ +metadata: + version: 1.0.0 + kind: schema + description: | + PRM-C13 G-PROV (PRM-001 v3 "PROMETHEUS"): no hosted-model output enters the training pool. + `pool::admit` (crates/aprender-review-experiment: pool.rs, reusing PRM-C12 terms.rs `tags`/`Provider`) + decides each candidate row's eligibility by provenance BEFORE the secret scan, so a hosted row + never reaches the quarantine, whose rows may be released after review. A row is eligible only if + it is local: provider `local` with a weights-sha model_id and a full terms tag (terms.rs), or it + carries a `label_source` in the gold set {outcome, hrq_ruling, sealed_corpus}. A row from a hosted + provider (anthropic, google) is Ineligible(Hosted) whatever else it claims; a row with no or a + malformed provenance tag is Ineligible(Untagged); any other label_source is Ineligible(NotGold). + `label_source` is a provisional gold marker until PRM-C10 fixes the label schema. + references: + - "docs/specifications/PRM-001-prometheus.md" + - "contracts/agent-trace-terms-v1.yaml" + +equations: + eligible: + formula: "eligible(r) ⇔ json(r) ∧ provider(r) ∉ {anthropic, google} ∧ (label_source(r) ∈ {outcome, hrq_ruling, sealed_corpus} ∨ (label_source(r) absent ∧ tags(r) = Ok ∧ provider(r) = local))" + order: + formula: "verdict(r) = Refused(sealed) if sealed hit; else Ineligible(p) if ¬eligible(r); else Quarantined if secret(r); else Admitted" + +falsification_tests: + - id: FALSIFY-TAP-001 + name: planted_hosted_row_is_ineligible + prediction: "anthropic and google rows planted in a pool beside a local row are Ineligible(Hosted); the rebuilt pool holds only the local row" + test_harness: "cargo test -p aprender-review-experiment --lib falsify_tap_001" + expected_output: "exit 0" + if_fails: "Hosted-model outputs would train the shadow lane, breaching the provider terms and making it a copy of the voters." + - id: FALSIFY-TAP-002 + name: hosted_row_is_ineligible_whatever_it_claims + prediction: "a hosted row claiming label_source outcome, or carrying a secret, is still Ineligible(Hosted) and never Quarantined" + test_harness: "cargo test -p aprender-review-experiment --lib falsify_tap_002" + expected_output: "exit 0" + if_fails: "A gold claim or the quarantine's later release would be a path for hosted output into the pool." + - id: FALSIFY-TAP-003 + name: untagged_row_is_ineligible + prediction: "a bare row, a misspelt provider (\"Anthropic\") and a non-JSON row are Ineligible(Untagged)" + test_harness: "cargo test -p aprender-review-experiment --lib falsify_tap_003" + expected_output: "exit 0" + if_fails: "A row with unknown provenance would be admitted by default; admission must fail closed." + - id: FALSIFY-TAP-004 + name: gold_labels_are_eligible_and_only_gold + prediction: "each gold label_source is admitted; quorum_majority is Ineligible(NotGold)" + test_harness: "cargo test -p aprender-review-experiment --lib falsify_tap_004" + expected_output: "exit 0" + if_fails: "Silver (voter-majority) labels would pass as gold and train the shadow lane on the voters' errors." diff --git a/contracts/trace-admission-secret-v1.yaml b/contracts/trace-admission-secret-v1.yaml new file mode 100644 index 0000000000..de3a2d9a22 --- /dev/null +++ b/contracts/trace-admission-secret-v1.yaml @@ -0,0 +1,87 @@ +metadata: + version: 1.0.0 + kind: schema + description: | + Trace admission (PRA-001 T6, gates G-SEC / G-SEC-FN; agent-trace ruling 2026-09-25 + item 6). A captured lane row enters the fine-tune/distill pool only when it is + secret-scan clean AND carries no sealed-test hash (review-corpus-contamination-v1). + The secret scan is the UNION of two in-process Rust scanners, no Go or Python: + (a) builtin detectors (AWS key id/secret, GitHub token, PEM private key, Anthropic, + Slack, JWT, high-entropy assignment), which honour no allowlist; (b) the gitleaks + v8.30.1 ruleset vendored verbatim as data (data/gitleaks/, MIT, sha256-pinned) and + interpreted in-process: keyword prefilter, regex, secret group, entropy floor, + per-rule and global allowlists. Without a file path the scan fails safe: a + path-scoped rule runs on all content, a path-conditioned (AND) allowlist allows + nothing, and the one path-only rule (pkcs12-file) is listed, not dropped. + Any hit QUARANTINES the row byte-for-byte; it is never redacted in place. A + sealed-test hit REFUSES the row and is checked first. An empty sealed index admits + nothing (Unchecked). Canaries are built at run time, so no key literal is in source. + Implementation: crates/aprender-review-experiment/src/{secret,pool}.rs. + Pending on T1/T3 (the trace index does not exist yet): the backfill run + (`make trace-secret-scan-backfill`, G11) and the index's secret_scan record + {scanner_versions, hits, status}; secret::SCANNER_VERSIONS is its version field. + references: + - "contracts/review-corpus-contamination-v1.yaml" + - "docs/specifications/review-experiment-protocol.md" + +equations: + admit: + formula: "pool(R) = { r in R : sealed != {} and leak(r) = false and builtin(r) ∪ gitleaks(r) = {} }" + recall: + formula: "for each canary c labelled (scanner s, rule k): k in scan(c), and scan_s(c) without k = {}" + +falsification_tests: + - id: FALSIFY-TAS-001 + name: every_canary_fires_its_labelled_rule + prediction: "all 20 canaries (8 builtin, 12 gitleaks: AWS, GitHub PAT, private key, JWT, high-entropy generic, …) fire their labelled rule, bare and inside a JSONL row: 100% recall" + test_harness: "cargo test -p aprender-review-experiment --lib falsify_tas_001" + expected_output: "exit 0" + if_fails: "A secret shape the gate claims to catch reaches the pool (G-SEC)." + - id: FALSIFY-TAS-002 + name: removing_a_rule_turns_its_canary_red + prediction: "with a canary's labelled rule removed, its scanner reports nothing on that canary" + test_harness: "cargo test -p aprender-review-experiment --lib falsify_tas_002" + expected_output: "exit 0" + if_fails: "Recall is carried by some other rule, so the per-rule recall claim is untested (G-SEC-FN)." + - id: FALSIFY-TAS-003 + name: aws_doc_key_needs_the_builtin_scanner + prediction: "the AWS documentation key, which gitleaks allowlists (.+EXAMPLE$), is hit by the builtin scanner alone; with that rule removed nothing catches it" + test_harness: "cargo test -p aprender-review-experiment --lib falsify_tas_003" + expected_output: "exit 0" + if_fails: "The union is not load-bearing, or an upstream allowlist is silently trusted." + - id: FALSIFY-TAS-004 + name: planted_aws_key_is_quarantined + prediction: "a lane output carrying the AWS key literal is Quarantined byte-identical and is not in the pool; a clean neighbour is admitted" + test_harness: "cargo test -p aprender-review-experiment --lib falsify_tas_004" + expected_output: "exit 0" + if_fails: "Secrets reach the training pool, or a quarantined row was rewritten in place." + - id: FALSIFY-TAS-005 + name: planted_test_split_diff_is_refused + prediction: "a row carrying a sealed test diff, or only its diff sha, is Refused and not in the pool" + test_harness: "cargo test -p aprender-review-experiment --lib falsify_tas_005" + expected_output: "exit 0" + if_fails: "Sealed test items leak into training data (S-3)." + - id: FALSIFY-TAS-006 + name: scanner_case_tables + prediction: "builtin must-match rows (incl. a JSON-escaped key) fire exactly their rule and must-not-match rows fire none; gitleaks drops a sub-entropy match and does not apply a path-conditioned allowlist; ordinary review rows fire neither scanner" + test_harness: "cargo test -p aprender-review-experiment --lib -- case_table ordinary_review_rows_are_clean" + expected_output: "exit 0" + if_fails: "A key shape is missed, or clean rows are quarantined." + - id: FALSIFY-TAS-007 + name: ruleset_pinned_and_complete + prediction: "the vendored toml matches its sha256; all 222 declared rules are either compiled or listed path-only; no regex fails to compile" + test_harness: "cargo test -p aprender-review-experiment --lib -- vendored_gitleaks_toml_is_sha_pinned every_gitleaks_rule_compiles" + expected_output: "exit 0" + if_fails: "The ruleset drifted from upstream, or a rule was silently dropped." + - id: FALSIFY-TAS-008 + name: empty_sealed_index_admits_nothing + prediction: "with no sealed items indexed, no row is admitted" + test_harness: "cargo test -p aprender-review-experiment --lib empty_sealed_index_refuses_everything" + expected_output: "exit 0" + if_fails: "A missing test manifest silently admits every row." + - id: FALSIFY-TAS-009 + name: backfill_quarantines_the_aws_key_and_twenty_canaries + prediction: "a backfill over the AWS key literal plus the 20 labelled canaries (one lane row each) and one clean row quarantines all 21 byte-identical, admits the clean row, and the receipt counts each canary's labelled rule; with no sealed index nothing is admitted" + test_harness: "cargo test -p aprender-review-experiment --lib falsify_tas_009" + expected_output: "exit 0" + if_fails: "The PRM-C6 backfill lets a planted secret into the pool, rewrites a quarantined row, or its receipt miscounts." diff --git a/contracts/trace-datacard-v1.yaml b/contracts/trace-datacard-v1.yaml new file mode 100644 index 0000000000..fa300c3d30 --- /dev/null +++ b/contracts/trace-datacard-v1.yaml @@ -0,0 +1,95 @@ +metadata: + version: 1.0.0 + kind: schema + description: | + PRA-001 T14 (spec §2.11): the dataset card for one snapshot of the agent-trace-v1 index. + aprender_review_experiment::datacard reads every index/agent-trace-v1/YYYY/MM/DD.jsonl under + the traces root, in path order. It binds them with a sha256sum manifest, and the snapshot id + is the first 16 hex of the manifest's sha256. For that snapshot it generates a Croissant 1.0 + + RAI 1.0 JSON-LD card and a Datasheet (motivation, composition, collection, preprocessing, + uses, distribution, maintenance) under datacard//, regenerated per snapshot and + never hand-edited. + The official validator (mlcroissant) is Python, which S-PY forbids, so datacard::validate is a + structural Croissant 1.0 + RAI checker in Rust. It checks the context, sc:Dataset, + conformsTo, the resources and their sha256, the record set, dataTypes, @id uniqueness, + reference resolution, jsonPath and the ten RAI fields. write_snapshot refuses to write a + card that fails it. + The card carries aggregate counts only. HF publication is gold AND provider local AND + secret-scan clean; provider anthropic and google rows are never public-eligible. + PRM-001 v2 PRM-C14 (was PRA T14) adds the weekly yield/bytes receipt, trace-weekly-receipt-v1 + (weekly.json beside the card). Per ISO 8601 week it records quorums/day, rows, index bytes and + the CAS blobs the rows address: stored bytes, plus raw bytes from the zstd frame header + (RFC 8878). It checks them against the G16 [O] band (50-200 quorums/day, <= 0.45 TB/yr raw) + and reads within | outside | unmeasured. A missing blob or an undeclared raw size is + unmeasured, never within. + references: + - "docs/specifications/research-design-pr-agent-qwen-3.5.md" + - "http://mlcommons.org/croissant/1.0" + - "http://mlcommons.org/croissant/RAI/1.0" + - "https://github.com/paiml/aprender/issues/4354" + +equations: + snapshot: + formula: "id = sha256(manifest)[0..16]; manifest = ⊕_{f ∈ sort(index files)} sha256(bytes(f)) ‖ ' ' ‖ path(f)" + public: + formula: "eligible(row) ⇔ label_tier = gold ∧ provider = local ∧ secret_scan.status = clean" + card: + formula: "write(card) ⇒ validate(card) = ∅; exists(dir/f) ∧ bytes(dir/f) ≠ regen(f) ⇒ refuse" + g16: + formula: "within(w) ⇔ missing(w) + undeclared(w) = 0 ∧ 50 ≤ min_d q(d) ∧ max_d q(d) ≤ 200 ∧ (index(w) + raw(w)) / days(w) · 365 ≤ 0.45 TB" + +falsification_tests: + - id: FALSIFY-TDC-001 + name: the_generated_card_passes_croissant_validation + prediction: "the generated card validates clean. Each planted defect (no conformsTo, empty name, wrong @type, FileObject with no sha256, source naming an unknown resource, duplicate @id, unknown dataType, missing rai:dataLimitations, @context missing rai) is refused with an error that names it" + test_harness: "cargo test -p aprender-review-experiment --lib falsify_tdc_001" + expected_output: "exit 0" + if_fails: "A card that is not Croissant 1.0 + RAI could be published, or the validator passes anything." + - id: FALSIFY-TDC-002 + name: a_snapshot_is_deterministic_and_bound_to_the_index_bytes + prediction: "the same index gives the same snapshot id, card bytes and datasheet; every manifest line's sha equals the file's sha256; the card's manifest sha equals sha256(manifest); one appended day gives a new snapshot id and sha" + test_harness: "cargo test -p aprender-review-experiment --lib falsify_tdc_002" + expected_output: "exit 0" + if_fails: "A card would not identify which index bytes it describes, or would not change when the index does." + - id: FALSIFY-TDC-003 + name: only_gold_local_rows_are_public_eligible + prediction: "gold anthropic, gold google, silver local and pending rows are not public-eligible, nor is a gold local row with a secret-scan hit; composition counts per provider, label_tier and split are exact" + test_harness: "cargo test -p aprender-review-experiment --lib falsify_tdc_003" + expected_output: "exit 0" + if_fails: "The card would count provider anthropic/google rows or unscanned rows as releasable (§2.11)." + - id: FALSIFY-TDC-004 + name: a_hand_edited_card_is_refused_and_regeneration_is_idempotent + prediction: "write_snapshot writes croissant.json, datasheet.md and manifest.sha256 under datacard//; a second run on the same index is a no-op; after a hand edit the next run errors with 'hand-edited' and leaves the edited bytes in place" + test_harness: "cargo test -p aprender-review-experiment --lib falsify_tdc_004" + expected_output: "exit 0" + if_fails: "A hand-edited card would be silently kept or silently clobbered." + - id: FALSIFY-TDC-005 + name: no_row_content_reaches_the_card + prediction: "a sentinel finding text, a diff sha and an output sha planted in the fixture rows appear in neither the card nor the datasheet" + test_harness: "cargo test -p aprender-review-experiment --lib falsify_tdc_005" + expected_output: "exit 0" + if_fails: "Trace content would leak into a card (PRIVATE, §6)." + - id: FALSIFY-TDC-006 + name: a_malformed_index_is_refused_not_skipped + prediction: "a bad JSON line errors with its file:line; a row whose schema is not agent-trace-v1 errors with its schema; an empty index errors 'no index'" + test_harness: "cargo test -p aprender-review-experiment --lib falsify_tdc_006" + expected_output: "exit 0" + if_fails: "A card would describe an index with rows silently dropped." + - id: FALSIFY-TDC-007 + name: the_weekly_receipt_measures_g16_or_says_unmeasured + prediction: "per ISO week the receipt counts rows, distinct quorums/day, index bytes and distinct blobs; a missing blob or an undeclared zstd size reads unmeasured; all blobs measured at 1-2 quorums/day reads outside; 50 quorums/day with tiny blobs reads within; no sha or row text appears in it" + test_harness: "cargo test -p aprender-review-experiment --lib falsify_tdc_007" + expected_output: "exit 0" + if_fails: "G16 could read verified from a week whose bytes were never measured." + - id: FALSIFY-TDC-008 + name: iso_weeks_and_zstd_sizes_match_their_standards + prediction: "iso_week agrees with ISO 8601 across year boundaries (2027-01-01 = 2026-W53, 2024-12-30 = 2025-W01, 2021-01-03 = 2020-W53); zstd_content_size reads 1/2/4-byte FCS (2-byte is +256), skips a dict id, and returns None for no FCS, a non-zstd magic or a truncated header" + test_harness: "cargo test -p aprender-review-experiment --lib falsify_tdc_008" + expected_output: "exit 0" + if_fails: "Rows would land in the wrong week, or raw bytes would be misread." + - id: FALSIFY-TDC-009 + name: g16_is_decided_only_when_every_week_is_measured + prediction: "g16_decided returns each week's within/outside only when every week is measured; one unmeasured week (named), no weeks, an unknown verdict or a missing weeks array is an error, so G16 stays [U]" + test_harness: "cargo test -p aprender-review-experiment --lib falsify_tdc_009" + expected_output: "exit 0" + if_fails: "G16 is reported [V] from a week whose blobs were never measured." diff --git a/contracts/trace-dedup-v1.yaml b/contracts/trace-dedup-v1.yaml new file mode 100644 index 0000000000..fcfe0e9ba6 --- /dev/null +++ b/contracts/trace-dedup-v1.yaml @@ -0,0 +1,57 @@ +metadata: + version: 1.0.0 + kind: schema + description: | + PRM-001 v3 PRM-C8 (was PRA T8), gate G-DUP. `dedup::clusters` + (crates/aprender-review-experiment/src/dedup.rs) groups trace diffs into near-duplicate + clusters so the split guard (PRM-C9, trace-split-guard-v1) can close each cluster to + one split. It reuses the G-CON fingerprint in cluster.rs (identifiers normalised to `$`, + whitespace dropped, 12-token shingles, bottom-64 sketch), so a re-indented, re-pathed or + renamed copy is the same diff here as it is to the contamination check. Candidates come + from an inverted index over sketch hashes (LSH); a candidate pair matches when either + sketch is at least 0.7 contained in the other's shingle set; union-find closes chains. + Diffs too short to sketch merge only on identical normalised tokens. A cluster's id is + its earliest member by (at, id), independent of input order. Recall bound, stated: two + items whose sketches share no hash are never compared. + references: + - "docs/specifications/research-design-pr-agent-qwen-3.5.md" + - "contracts/review-corpus-contamination-v1.yaml" + - "https://github.com/paiml/aprender/issues/4354" + +equations: + candidate: + formula: "cand(i, j) ⇔ sketch(i) ∩ sketch(j) ≠ ∅" + near_dup: + formula: "near(i, j) ⇔ cand(i, j) ∧ max(|sketch(i) ∩ S(j)| / |sketch(i)|, |sketch(j) ∩ S(i)| / |sketch(j)|) ≥ 0.7" + tiny: + formula: "|S(i)| < 32 ⇒ (i ~ j ⇔ tokens(i) = tokens(j))" + closure: + formula: "cluster = transitive closure of near ∪ tiny" + cluster_id: # cluster-id-guard allow (a formula name: the id is the earliest member, permutation-invariant, not a k-means label) + formula: "id(C) = argmin_{m ∈ C} (at(m), id(m)); invariant under permutation of the input" + +falsification_tests: + - id: FALSIFY-TDD-001 + name: a_perturbed_copy_joins_its_original_and_an_unrelated_diff_does_not + prediction: "a re-pathed, re-indented, renamed copy and a diff embedded in a larger one share the original's cluster, in either arrival order; an unrelated diff stays alone" + test_harness: "cargo test -p aprender-review-experiment --lib falsify_tdd_001" + expected_output: "exit 0" + if_fails: "A copy of a test item could sit in train under a different sha." + - id: FALSIFY-TDD-002 + name: the_cluster_id_is_the_earliest_member_whatever_the_order + prediction: "the id is the member with the least (at, id), for any input order" + test_harness: "cargo test -p aprender-review-experiment --lib falsify_tdd_002" + expected_output: "exit 0" + if_fails: "A cluster's split would depend on read order, not on time." + - id: FALSIFY-TDD-003 + name: near_dup_chains_close_transitively + prediction: "X~Y and Y~Z put X, Y, Z in one cluster although X and Z alone do not match" + test_harness: "cargo test -p aprender-review-experiment --lib falsify_tdd_003" + expected_output: "exit 0" + if_fails: "A chain of edits could straddle splits one link at a time." + - id: FALSIFY-TDD-004 + name: tiny_diffs_merge_only_when_identical + prediction: "diffs below 32 shingles merge only on identical normalised tokens" + test_harness: "cargo test -p aprender-review-experiment --lib falsify_tdd_004" + expected_output: "exit 0" + if_fails: "Every one-line diff would collapse into one cluster, or none would dedup." diff --git a/contracts/trace-outcome-join-v1.yaml b/contracts/trace-outcome-join-v1.yaml new file mode 100644 index 0000000000..0b9ccc0c05 --- /dev/null +++ b/contracts/trace-outcome-join-v1.yaml @@ -0,0 +1,62 @@ +metadata: + version: 1.0.0 + kind: schema + description: | + PRM-001 v3 PRM-C10 (was PRA T10), gate G-MAT: "gold only after the 14 d outcome window; + pending is never a negative; 0 immature gold". Done when: "3-day-old gold refused; weekly + matured count". `outcome_join::resolve` (crates/aprender-review-experiment/src/outcome_join.rs) + reads a PR's merge / close / revert / escape events into the v3 lifecycle `outcome` + {pending, merged, reverted_le14d, regression_escape, closed_unmerged}. A revert at most 14 UTC + dates after the merge is reverted_le14d; a later revert or any escape after the merge is + regression_escape. The outcome stays `pending`, with no `outcome_matured_at`, until 14 UTC dates + after the merge (or close). A revert or escape of a PR never merged, or an unreadable date, is + refused. `outcome_join::gold` emits `label_source: outcome` rows for matured outcomes only and + refuses a non-pending row with no maturity date; it emits `label_source: hrq_ruling` rows for + HRQ rulings as soon as they are made (latest per round wins; no ruler or no date is refused). + `outcome_join::matured_between` is the weekly `gold_matured_7d` count. + references: + - "docs/specifications/research-design-pr-agent-qwen-3.5.md" + - "contracts/trace-split-guard-v1.yaml" + - "https://github.com/paiml/aprender/issues/4354" + +equations: + outcome: + formula: "m = first merge; outcome = reverted_le14d if ∃ revert r ≥ m with r − m ≤ 14, regression_escape if ∃ revert r > m + 14 or escape ≥ m, merged otherwise; closed_unmerged if closed and never merged" + maturity: + formula: "matured ⇔ day(now) − day(m or close) ≥ 14; ¬matured ⇒ outcome = pending ∧ outcome_matured_at = None" + gold: + formula: "gold = { outcome rows : matured } ∪ { latest HRQ ruling per (repo#PR, head) }; pending ∉ gold" + weekly: + formula: "gold_matured_7d(from, to) = |{ r : from ≤ outcome_matured_at(r) < to }|" + +falsification_tests: + - id: FALSIFY-TOJ-001 + name: a_3_day_old_outcome_is_refused_as_gold + prediction: "a merge or revert 3 d ago reads pending and yields no gold; 13 UTC dates is still pending; 14 matures with outcome_matured_at = merge + 14" + test_harness: "cargo test -p aprender-review-experiment --lib falsify_toj_001" + expected_output: "exit 0" + if_fails: "An outcome could be trained on before a revert had time to happen." + - id: FALSIFY-TOJ-002 + name: the_outcome_follows_the_revert_window + prediction: "revert on day 14 = reverted_le14d, day 15 = regression_escape, escape = regression_escape, closed = closed_unmerged; the window runs from the merge" + test_harness: "cargo test -p aprender-review-experiment --lib falsify_toj_002" + expected_output: "exit 0" + if_fails: "A late regression could be labelled a clean merge, or a fast revert an escape." + - id: FALSIFY-TOJ-003 + name: pending_is_never_a_label_and_bad_joins_refuse + prediction: "a pending PR yields no gold; a revert or escape with no merge, an unreadable date, and a forged immature row are all errors" + test_harness: "cargo test -p aprender-review-experiment --lib falsify_toj_003" + expected_output: "exit 0" + if_fails: "A pending PR could be read as a negative label." + - id: FALSIFY-TOJ-004 + name: hrq_rulings_are_gold + prediction: "a ruling is gold at once, label_source hrq_ruling (a pool GOLD_SOURCES value); the latest per round wins; no ruler or no date is refused" + test_harness: "cargo test -p aprender-review-experiment --lib falsify_toj_004" + expected_output: "exit 0" + if_fails: "A superseded or anonymous ruling could become a training label." + - id: FALSIFY-TOJ-005 + name: the_weekly_matured_count + prediction: "matured_between counts outcomes whose window closed in [from, to) and no pending PR" + test_harness: "cargo test -p aprender-review-experiment --lib falsify_toj_005" + expected_output: "exit 0" + if_fails: "The weekly receipt could count immature or out-of-week gold." diff --git a/contracts/trace-split-guard-v1.yaml b/contracts/trace-split-guard-v1.yaml new file mode 100644 index 0000000000..8be449aab2 --- /dev/null +++ b/contracts/trace-split-guard-v1.yaml @@ -0,0 +1,55 @@ +metadata: + version: 1.0.0 + kind: schema + description: | + PRM-001 v3 PRM-C9 (was PRA T9), gate G-DUP: "split by repo#PR, time-ordered, >= 14 d + embargo, clusters closed to their earliest member; 0 cross-split clusters". + `split_guard::assign` (crates/aprender-review-experiment/src/split_guard.rs) writes the + `split_guard{group_key, split, dedup_cluster}` a trace row carries. Units sharing a PR or + a trace-dedup-v1 cluster form one component; the component's earliest first-seen day + picks its split against the plan's val_from / test_from; a component first seen within + 14 d before a boundary is embargoed (split None, in no pool); a component holding a + near-dup of a sealed item is `sealed` (in no pool). A plan whose val window is not longer + than the embargo, or an unreadable date, is refused. `split_guard::audit` checks ANY + assignment, not only its own, for PRs and clusters that straddle splits; G-DUP holds + when both lists are empty. Embargoed members are in no pool and never make a straddle. + references: + - "docs/specifications/research-design-pr-agent-qwen-3.5.md" + - "contracts/trace-dedup-v1.yaml" + - "https://github.com/paiml/aprender/issues/4354" + +equations: + component: + formula: "u ~ w ⇔ repo#PR(u) = repo#PR(w) ∨ cluster(u) = cluster(w); components = transitive closure of ~" + split: + formula: "d = min_{u ∈ C} day(at(u)); split(C) = sealed if C ∋ near-dup of a sealed item, else train if d < val−14, None if d < val, val if d < test−14, None if d < test, else test" + plan: + formula: "assign refused ⇔ test − val ≤ 14 ∨ some date unreadable" + audit: + formula: "straddle(K) = { k : |{ split(u) : key(u) = k, split(u) ≠ None }| > 1 }; G-DUP ⇔ straddle(repo#PR) = ∅ ∧ straddle(cluster) = ∅" + +falsification_tests: + - id: FALSIFY-TSG-001 + name: a_pr_never_straddles_splits + prediction: "a PR's later round keeps its first round's split; a straddling assignment is named by audit" + test_harness: "cargo test -p aprender-review-experiment --lib falsify_tsg_001" + expected_output: "exit 0" + if_fails: "Two rounds of one PR could sit in train and test." + - id: FALSIFY-TSG-002 + name: a_near_dup_cluster_closes_to_its_earliest_members_split + prediction: "a PR/cluster chain takes its earliest member's split; a per-row split of the same units shows a cross-split cluster" + test_harness: "cargo test -p aprender-review-experiment --lib falsify_tsg_002" + expected_output: "exit 0" + if_fails: "A re-indented copy of a train diff could be scored as test." + - id: FALSIFY-TSG-003 + name: the_embargo_is_at_least_14_days + prediction: "first seen 14 d or less before a boundary is embargoed; 15 d is not; a short val window and a bad date are refused" + test_harness: "cargo test -p aprender-review-experiment --lib falsify_tsg_003" + expected_output: "exit 0" + if_fails: "Work from the same fortnight could span train and test." + - id: FALSIFY-TSG-004 + name: a_perturbed_sealed_item_is_refused_with_its_whole_pr + prediction: "a renamed, re-indented sealed diff puts its whole component in `sealed`, named by the sealed id" + test_harness: "cargo test -p aprender-review-experiment --lib falsify_tsg_004" + expected_output: "exit 0" + if_fails: "A sealed test item could reach a training pool under a new sha." diff --git a/contracts/witness/f4a1686f98d541afc73cc61d63d1a9726ef28aaee70bad35dff472cef083bf67.json b/contracts/witness/f4a1686f98d541afc73cc61d63d1a9726ef28aaee70bad35dff472cef083bf67.json index 7f0aa6a9fe..ed0d51a484 100644 --- a/contracts/witness/f4a1686f98d541afc73cc61d63d1a9726ef28aaee70bad35dff472cef083bf67.json +++ b/contracts/witness/f4a1686f98d541afc73cc61d63d1a9726ef28aaee70bad35dff472cef083bf67.json @@ -1,7 +1,7 @@ { - "census_id_set_sha256": "ced96e20e1de8700f4de7592803f602f279ccecc086cbb4fbd86b349b9028779", + "census_id_set_sha256": "f06f8234b25ef4b1ae1a4aaf43f12f97ffebba06e2d65e9dbf848df9aeeb241c", "relations_sha256": "f4a1686f98d541afc73cc61d63d1a9726ef28aaee70bad35dff472cef083bf67", - "reasoner_git_sha": "28006ac347c6c8ff337874e39711ea67dbfe8bcf", + "reasoner_git_sha": "b5ad4f9204ea328b43e2864e490d574bd6593187", "checkable_n": 84, "result": { "kind": "model", @@ -10,5 +10,5 @@ } }, "pc_reasoner": "fired", - "cpu_ms": 1 + "cpu_ms": 2 } diff --git a/contracts/workload-profile-v1.yaml b/contracts/workload-profile-v1.yaml new file mode 100644 index 0000000000..2cb912daac --- /dev/null +++ b/contracts/workload-profile-v1.yaml @@ -0,0 +1,58 @@ +metadata: + version: 1.0.0 + kind: schema + description: | + PRM-S2 (PRM-001 v3 "PROMETHEUS" §0.2, §2.4): the review workload profile. It reads + `agent-trace-v1` rows (`timing.tokens{input,output}`) and reports the measured distribution of + input and output tokens per lane. `rex workload ROWS --min-n N` (crates/aprender-review-experiment: + workload.rs, examples/rex.rs) computes it. Rows with `trace_status = capture_failed`, rows that + do not parse, and rows without a positive `tokens.input` or with no `tokens.output` are excluded. + They are counted as excluded and never read as zero tokens. Quantiles are nearest-rank + (stats::percentile, the same rule as every other rex percentile). + §0.2 says a diff of 2k–32k tokens goes in and a verdict plus at most 5 bullets comes out, so review + is prefill-heavy. It is decided per lane, with both bounds taken from §0.2's own text: + in_band ⇔ input p50 ≥ 2000 ∧ input p95 ≤ 32000; prefill_heavy ⇔ output p95 < input p50. + Only lanes with at least min_n rows decide. The claim is confirmed when every deciding lane holds + both, falsified when a deciding lane fails either (the lane is named), and undecided when no lane + reaches min_n. An undecided claim is refused and never read as confirmed. + Strata weights (§2.4: 2k / 8k / 16k / 32k) are taken over distinct work units (`diff_sha256`). + A unit's size is the largest `tokens.input` any lane measured for it, and its stratum is the + smallest bound ≥ that size. A unit over 32k is counted as `over_32k` and left out of the + weights, which sum to 1 over the in-range units. + references: + - "docs/specifications/PRM-001-prometheus.md" + - "contracts/trace-datacard-v1.yaml" + +equations: + quantile: + formula: "p_q(lane, k) = nearest_rank_q({tokens.k(r) : r ∈ rows(lane), measured(r)}), k ∈ {input, output}, q ∈ {0.5, 0.95}" + claim: + formula: "D = {lane : n(lane) ≥ min_n}; claim = None if D = ∅ else Some(∀ lane ∈ D: p50_in ≥ 2000 ∧ p95_in ≤ 32000 ∧ p95_out < p50_in)" + strata: + formula: "size(u) = max_{r ∈ u} tokens.input(r); stratum(u) = min{b ∈ {2000, 8000, 16000, 32000} : size(u) ≤ b}; w_b = |{u : stratum(u) = b}| / |{u : size(u) ≤ 32000}|" + +falsification_tests: + - id: FALSIFY-WLP-001 + name: quantiles_are_nearest_rank_on_measured_rows_only + prediction: "capture_failed, unparseable, input-0 and output-missing rows are excluded and counted, and they do not move any lane's p50/p95; p50/p95 equal nearest rank on the planted sample" + test_harness: "cargo test -p aprender-review-experiment --lib falsify_wlp_001" + expected_output: "exit 0" + if_fails: "A failed capture would read as a 0-token review and drag the profile toward decode-bound." + - id: FALSIFY-WLP-002 + name: claim_is_decided_per_lane_and_refused_when_thin + prediction: "in-band prefill-heavy lanes confirm; a lane whose output p95 ≥ input p50, or whose input p50 < 2000 or p95 > 32000, falsifies and is named; lanes below min_n do not decide; with no lane at min_n the claim is undecided and refused" + test_harness: "cargo test -p aprender-review-experiment --lib falsify_wlp_002" + expected_output: "exit 0" + if_fails: "§0.2 would stay an assertion: speed work would chase prefill on a workload nobody measured." + - id: FALSIFY-WLP-003 + name: strata_weights_are_over_distinct_diffs + prediction: "one diff reviewed by 3 lanes counts once, at its largest lane measurement; bounds are inclusive (2000 → 2k, 2001 → 8k); a unit over 32k is over_32k and unweighted; weights sum to 1" + test_harness: "cargo test -p aprender-review-experiment --lib falsify_wlp_003" + expected_output: "exit 0" + if_fails: "Replay strata would be weighted by lane count rather than by the diffs the lane actually sees." + - id: FALSIFY-WLP-004 + name: receipt_block_matches_the_v3_template + prediction: "the rendered block is `workload: {input_tokens_p50_p95: {lane: [p50, p95]}, output_tokens_p50_p95: {…}}` plus claim, strata and excluded; an undecided claim renders null, not true" + test_harness: "cargo test -p aprender-review-experiment --lib falsify_wlp_004" + expected_output: "exit 0" + if_fails: "The §7 receipt reader would miss the workload block or read an undecided claim as confirmed." diff --git a/crates/apr-cli/src/commands/chat_generate_safetensors.rs b/crates/apr-cli/src/commands/chat_generate_safetensors.rs index 78d20d11b1..b327de1d4e 100644 --- a/crates/apr-cli/src/commands/chat_generate_safetensors.rs +++ b/crates/apr-cli/src/commands/chat_generate_safetensors.rs @@ -72,6 +72,7 @@ impl ChatSession { seed: realizar::apr_transformer::DEFAULT_SEED, repeat_penalty: 1.0, repeat_last_n: 0, + logprobs_top_k: 0, // apr_transformer::generation::is_eos_token (GH-330) treated // token 0 as EOS unconditionally; Session has no such builtin, // so it is carried here as an explicit stop token to keep this diff --git a/crates/apr-cli/src/commands/inference_output.rs b/crates/apr-cli/src/commands/inference_output.rs index bec1955d6d..95155e2661 100644 --- a/crates/apr-cli/src/commands/inference_output.rs +++ b/crates/apr-cli/src/commands/inference_output.rs @@ -188,6 +188,8 @@ struct InferenceOutput { token_texts: Option>, /// Prompt and completion counts plus the finish reason (#3718). usage: RunUsage, + /// #4026: see `RunResult::logprobs`. + logprobs: Option, } /// Execute inference on model @@ -222,6 +224,7 @@ fn execute_inference( input_path.map_or_else(|| "stdin".to_string(), |p| p.display().to_string()); Ok(InferenceOutput { + logprobs: None, text: format!( "[Inference requires --features inference]\nModel: {}\nInput: {}\nFormat: {}\nGPU: {}", model_path.display(), @@ -335,7 +338,8 @@ pub(crate) fn realizar_config( .with_repeat_penalty(options.repeat_penalty) .with_repeat_last_n(options.repeat_last_n) .with_force_chat_template(options.chat_template) - .with_thinking(options.thinking); + .with_thinking(options.thinking) + .with_logprobs_top_k(options.logprobs_top_k); if options.no_gpu { config = config.without_gpu(); @@ -377,7 +381,19 @@ fn execute_with_realizar( // #3718: the report carries what only the decode path knows (finish reason, // context window); the prompt count is `input_token_count`, taken after the // chat template, so it is the number the model actually read. + let _ = realizar::session::take_last_turn_steps(); // #4026: never an earlier run's steps let (result, report) = run_inference_report(&config).map_err(inference_error)?; + let (prompt_ids, generated_ids) = result + .tokens + .split_at(result.input_token_count.min(result.tokens.len())); + let logprobs = run_logprobs( + options.logprobs_top_k, + &result.format, + prompt_ids, + generated_ids, + report.finish_reason == Some(realizar::infer::run_report::FinishReason::Stop), + realizar::session::take_last_turn_steps(), + )?; // Report performance if benchmarking if options.benchmark { @@ -427,9 +443,63 @@ fn execute_with_realizar( .generation_ms .map(|g| (result.inference_ms - g).max(0.0).round() as u64), }, + logprobs, }) } +/// #4026: the `logprobs` object of `apr run --json`, or a refusal by name when +/// `--logprobs K` was asked and the path that served the run recorded nothing. +/// An empty list is never reported for a path that could not see the logits, and +/// steps whose chosen tokens are not this run's generated tokens (another turn's, +/// such as a warmup or a retry on the same thread) are refused, never reported. +/// +/// One exception: a turn that ended on a stop token (`stopped`) may record one +/// step more than it reports, because the dense GGUF route drops the stop token +/// from the reply but the session recorded the step that chose it. That trailing +/// step is dropped, so the steps always cover the reported tokens and nothing +/// else; the hybrid route keeps the stop token in the reply and matches 1:1. +#[cfg(feature = "inference")] +fn run_logprobs( + top_k: usize, + format: &str, + prompt_token_ids: &[u32], + generated_token_ids: &[u32], + stopped: bool, + steps: Option>, +) -> Result> { + if top_k == 0 { + return Ok(None); + } + let steps = steps.ok_or_else(|| { + CliError::InvalidInput(format!( + "--logprobs {top_k}: the {format} path that served this run does not record \ + per-step logprobs (only the GGUF session engine does, on CPU and CUDA) — \ + refused rather than reported empty (#4026)" + )) + })?; + let mut steps = steps; + if stopped && steps.len() == generated_token_ids.len() + 1 { + steps.pop(); + } + if !steps + .iter() + .map(|s| s.chosen) + .eq(generated_token_ids.iter().copied()) + { + return Err(CliError::InvalidInput(format!( + "--logprobs {top_k}: the {} recorded steps do not match the {} tokens this run \ + generated — refused rather than reported against the wrong tokens (#4026)", + steps.len(), + generated_token_ids.len() + ))); + } + Ok(Some(serde_json::json!({ + "top_k": top_k, + "prompt_token_ids": prompt_token_ids, + "steps": steps, + }))) +} + #[cfg(all(test, feature = "inference"))] mod tests_2403 { use super::inference_error; @@ -456,3 +526,107 @@ mod tests_2403 { ); } } + +#[cfg(all(test, feature = "inference"))] +mod tests_4026 { + use super::run_logprobs; + use realizar::gguf::{StepLogprobs, TopLogprob}; + + #[test] + fn logprobs_off_reports_nothing_even_without_steps() { + assert_eq!( + run_logprobs(0, "GGUF", &[1], &[2], false, None).expect("ok"), + None + ); + } + + #[test] + fn logprobs_on_a_path_that_records_nothing_is_refused_by_name() { + let err = run_logprobs(3, "SafeTensors", &[1], &[2], false, None).expect_err("refused"); + let msg = err.to_string(); + assert!(msg.contains("SafeTensors"), "{msg}"); + assert!(msg.contains("--logprobs 3"), "{msg}"); + } + + fn step(step: usize, chosen: u32) -> StepLogprobs { + StepLogprobs { + step, + chosen, + top: vec![TopLogprob { + token_id: chosen, + logit: 4.0, + logprob: -0.25, + }], + chosen_logprob: -0.25, + logsumexp_full: 4.25, + } + } + + #[test] + fn logprobs_report_the_prompt_ids_and_every_step() { + let v = run_logprobs(1, "GGUF", &[5, 6], &[17], false, Some(vec![step(0, 17)])) + .expect("ok") + .expect("some"); + assert_eq!(v["top_k"], 1); + assert_eq!(v["prompt_token_ids"], serde_json::json!([5, 6])); + assert_eq!(v["steps"][0]["chosen"], 17); + assert_eq!(v["steps"][0]["top"][0]["token_id"], 17); + assert_eq!(v["steps"][0]["top"][0]["logprob"], -0.25); + } + + /// A step list from another turn on this thread (one step short, or one too + /// many) is refused, never reported against this run's tokens. + #[test] + fn logprobs_from_a_turn_of_another_length_are_refused() { + for steps in [ + vec![step(0, 17)], + vec![step(0, 17), step(1, 18), step(2, 19)], + ] { + let msg = run_logprobs(2, "GGUF", &[5], &[17, 18], false, Some(steps)) + .expect_err("refused") + .to_string(); + assert!(msg.contains("--logprobs 2"), "{msg}"); + assert!(msg.contains("do not match the 2 tokens"), "{msg}"); + } + } + + /// Same length, a different chosen token: still another turn's steps. + #[test] + fn logprobs_whose_chosen_tokens_differ_are_refused() { + let steps = vec![step(0, 17), step(1, 99)]; + let err = + run_logprobs(2, "GGUF", &[5], &[17, 18], false, Some(steps)).expect_err("refused"); + assert!(err.to_string().contains("2 recorded steps"), "{err}"); + } + + /// B1 (review 2 at 206172a585): a dense GGUF turn that ends on EOS drops the + /// stop token from its reply but recorded the step that chose it (2 = EOS + /// here). That one trailing step is dropped and the rest are reported. + #[test] + fn logprobs_of_a_turn_that_ended_on_a_dropped_stop_token_are_reported() { + let steps = vec![step(0, 17), step(1, 18), step(2, 2)]; + let v = run_logprobs(2, "GGUF", &[5], &[17, 18], true, Some(steps)) + .expect("ok") + .expect("some"); + assert_eq!(v["steps"].as_array().map(Vec::len), Some(2)); + assert_eq!(v["steps"][1]["chosen"], 18); + } + + /// The stop exception covers exactly one trailing step: two too many, one + /// short, a wrong chosen token before the stop, or one too many on a turn + /// that did not stop (budget) are all still refused. + #[test] + fn the_stop_exception_still_refuses_a_real_mismatch() { + for (stopped, steps) in [ + (true, vec![step(0, 17), step(1, 18), step(2, 2), step(3, 2)]), + (true, vec![step(0, 17)]), + (true, vec![step(0, 17), step(1, 99), step(2, 2)]), + (false, vec![step(0, 17), step(1, 18), step(2, 2)]), + ] { + let msg = run_logprobs(2, "GGUF", &[5], &[17, 18], stopped, Some(steps)) + .expect_err("refused") + .to_string(); + assert!(msg.contains("do not match the 2 tokens"), "{msg}"); + } + } +} diff --git a/crates/apr-cli/src/commands/run.rs b/crates/apr-cli/src/commands/run.rs index da2899b05f..8edb089364 100644 --- a/crates/apr-cli/src/commands/run.rs +++ b/crates/apr-cli/src/commands/run.rs @@ -175,6 +175,8 @@ pub(crate) struct RunOptions { /// #3723: `--thinking on|off`. `None` renders the production default; realizar applies it /// to the rendered prompt and refuses `on` for a template with no thinking mode. pub thinking: Option, + /// #4026: `--logprobs K`. 0 records nothing. + pub logprobs_top_k: usize, /// `--stream`: emit one NDJSON event per generated token. /// /// Known here (not only at the print site) because streaming is the one @@ -211,6 +213,7 @@ impl Default for RunOptions { split_prompt: false, chat_template: false, thinking: None, + logprobs_top_k: 0, stream: false, } } @@ -273,6 +276,8 @@ pub(crate) struct RunResult { pub token_texts: Option>, /// Prompt and completion counts plus the finish reason (#3718). pub usage: RunUsage, + /// #4026: `{top_k, prompt_token_ids, steps}` when `--logprobs K` > 0, else `None`. + pub logprobs: Option, } /// Resolve a user-supplied model argument into a [`ModelSource`]. @@ -361,6 +366,7 @@ pub(crate) fn run_model(source: &str, options: &RunOptions) -> Result generated_tokens: output.generated_tokens, token_texts: output.token_texts, usage: output.usage, + logprobs: output.logprobs, }) } @@ -381,62 +387,83 @@ pub(crate) fn resolve_model(source: &ModelSource, force: bool, offline: bool) -> match source { ModelSource::Local(path) => Ok(path.clone()), ModelSource::HuggingFace { org, repo, file } => { - // Check multiple cache locations for the model - // GH-528: Skip cache when --force is set to re-download - if !force { - if let Some(path) = find_cached_model(org, repo, file.as_deref()) { - return Ok(path); - } - } - - if offline { - // OFFLINE MODE: Reject any network access attempt. - // - // CRUX-A-20: a BARE `hf://org/repo` gets a different message, - // because "not cached" would be a claim we cannot support. The - // caller reached here having asked the Hub API which file the - // repo means (`run_model` → `resolve_hf_model`) and been - // refused, so `file` is None and the pacha cache — keyed on the - // full `hf://org/repo/` — cannot be probed at all. The - // file may well be cached under a name we cannot name. - if file.is_none() { - return Err(CliError::ValidationFailed(format!( - "OFFLINE MODE: cannot resolve hf://{org}/{repo} to a file. \ - Which file a bare repo means is only knowable from the \ - HuggingFace API, and network access is disabled. Name the \ - file (e.g. hf://{org}/{repo}/model.safetensors), pass a \ - local path, or cache it first with: apr import hf://{org}/{repo}" - ))); - } - return Err(CliError::ValidationFailed(format!( - "OFFLINE MODE: Model hf://{org}/{repo} not cached. \ - Network access is disabled. Cache the model first with: \ - apr import hf://{org}/{repo}" - ))); - } + resolve_hf_source(org, repo, file.as_deref(), force, offline) + } + ModelSource::Url(url) => resolve_url_source(source, url, force, offline), + } +} - // Auto-download like ollama - eprintln!("{}", format!("Downloading hf://{org}/{repo}...").yellow()); - download_hf_model(org, repo, file.as_deref()) +/// The `hf://` arm of [`resolve_model`], split out unchanged: inline, the two +/// remote arms held its cognitive complexity at 29 against the pre-commit +/// ceiling of 25. +fn resolve_hf_source( + org: &str, + repo: &str, + file: Option<&str>, + force: bool, + offline: bool, +) -> Result { + // Check multiple cache locations for the model + // GH-528: Skip cache when --force is set to re-download + if !force { + if let Some(path) = find_cached_model(org, repo, file) { + return Ok(path); } - ModelSource::Url(url) => { - let cache_path = source.cache_path(); - // GH-528: Skip cache when --force is set - if !force && cache_path.exists() { - // Cached URLs are allowed even in offline mode - find_model_in_dir(&cache_path) - } else if offline { - // OFFLINE MODE: Reject any network access attempt - Err(CliError::ValidationFailed(format!( - "OFFLINE MODE: URL {url} not cached. \ - Network access is disabled. Download and cache the model first." - ))) - } else { - // Auto-download from URL - eprintln!("{}", format!("Downloading {url}...").yellow()); - download_url_model(url) - } + } + + if offline { + // OFFLINE MODE: Reject any network access attempt. + // + // CRUX-A-20: a BARE `hf://org/repo` gets a different message, + // because "not cached" would be a claim we cannot support. The + // caller reached here having asked the Hub API which file the + // repo means (`run_model` → `resolve_hf_model`) and been + // refused, so `file` is None and the pacha cache — keyed on the + // full `hf://org/repo/` — cannot be probed at all. The + // file may well be cached under a name we cannot name. + if file.is_none() { + return Err(CliError::ValidationFailed(format!( + "OFFLINE MODE: cannot resolve hf://{org}/{repo} to a file. \ + Which file a bare repo means is only knowable from the \ + HuggingFace API, and network access is disabled. Name the \ + file (e.g. hf://{org}/{repo}/model.safetensors), pass a \ + local path, or cache it first with: apr import hf://{org}/{repo}" + ))); } + return Err(CliError::ValidationFailed(format!( + "OFFLINE MODE: Model hf://{org}/{repo} not cached. \ + Network access is disabled. Cache the model first with: \ + apr import hf://{org}/{repo}" + ))); + } + + // Auto-download like ollama + eprintln!("{}", format!("Downloading hf://{org}/{repo}...").yellow()); + download_hf_model(org, repo, file) +} + +/// The URL arm of [`resolve_model`], split out unchanged (see [`resolve_hf_source`]). +fn resolve_url_source( + source: &ModelSource, + url: &str, + force: bool, + offline: bool, +) -> Result { + let cache_path = source.cache_path(); + // GH-528: Skip cache when --force is set + if !force && cache_path.exists() { + // Cached URLs are allowed even in offline mode + find_model_in_dir(&cache_path) + } else if offline { + // OFFLINE MODE: Reject any network access attempt + Err(CliError::ValidationFailed(format!( + "OFFLINE MODE: URL {url} not cached. \ + Network access is disabled. Download and cache the model first." + ))) + } else { + // Auto-download from URL + eprintln!("{}", format!("Downloading {url}...").yellow()); + download_url_model(url) } } diff --git a/crates/apr-cli/src/commands/run_07.rs b/crates/apr-cli/src/commands/run_07.rs index 658d6c501f..1b458ef41a 100644 --- a/crates/apr-cli/src/commands/run_07.rs +++ b/crates/apr-cli/src/commands/run_07.rs @@ -11,6 +11,7 @@ include!("run_tests_chrome_trace.rs"); include!("run_tests_layer_trace.rs"); include!("run_tests_stream_output.rs"); include!("run_tests_accel_reconcile.rs"); +include!("run_tests_logprobs_surface_4026.rs"); include!("run_tests_usage_3718.rs"); include!("run_tests_top_k_default_3754.rs"); } diff --git a/crates/apr-cli/src/commands/run_entry.rs b/crates/apr-cli/src/commands/run_entry.rs index 9b5ca2cb41..efab44e127 100644 --- a/crates/apr-cli/src/commands/run_entry.rs +++ b/crates/apr-cli/src/commands/run_entry.rs @@ -46,6 +46,8 @@ pub(crate) fn run( chat_template: bool, // #3723: `--thinking on|off` (None: the production default), applied in realizar. thinking: Option, + // #4026: `--logprobs K`: the K most likely tokens at every generated step. + logprobs_top_k: usize, ) -> Result<()> { // GH-516: Warn on --language/--task since whisper integration is not yet wired up if language.is_some() { @@ -60,6 +62,7 @@ pub(crate) fn run( // no banner, no `Source:` line and no generation — the caller sees a named // refusal and nothing that looks like a result. refuse_forced_accelerator_without_forward(source, accel_forced)?; + refuse_logprobs_without_machine_output(logprobs_top_k, stream, output_format, benchmark)?; // GH-240: Suppress header/source in JSON mode for clean machine-parseable output if output_format != "json" { @@ -118,6 +121,7 @@ pub(crate) fn run( split_prompt, chat_template, thinking, + logprobs_top_k, stream, }; @@ -302,6 +306,29 @@ pub(crate) fn emits_machine_output(stream: bool, output_format: &str, benchmark: !benchmark && (stream || output_format == "json") } +/// #4026: `--logprobs K` is reported only in the `--json` and `--stream` documents +/// ([`emits_machine_output`]). On the text and `--benchmark` surfaces the run would record +/// them, print none and exit 0, so K > 0 is refused by name there, before the load. +/// +/// # Errors +/// [`CliError::InvalidInput`] when `top_k > 0` and the flags select a human surface. +pub(crate) fn refuse_logprobs_without_machine_output( + top_k: usize, + stream: bool, + output_format: &str, + benchmark: bool, +) -> Result<()> { + if top_k == 0 || emits_machine_output(stream, output_format, benchmark) { + return Ok(()); + } + let surface = if benchmark { "--benchmark" } else { "text" }; + Err(CliError::InvalidInput(format!( + "--logprobs {top_k}: the {surface} output does not report logprobs; only the \ + --json and --stream documents do (#4026). Add --json (without --benchmark), \ + or drop --logprobs." + ))) +} + /// Compare the accelerator the user ASKED for against the one that RAN. /// /// Delegates the decision to [`crate::registry::after_generation`], which is @@ -594,6 +621,32 @@ fn print_run_output( Ok(()) } +/// `tok_per_sec` when the inference engine reported one (GH-250); otherwise +/// derived from `tokens_generated / duration_secs`, or `0.0` on a zero +/// duration (nothing decoded, so there is no rate to report). +fn effective_tok_per_sec(result: &RunResult, tokens_generated: usize) -> f64 { + result.tok_per_sec.unwrap_or_else(|| { + if result.duration_secs > 0.0 { + tokens_generated as f64 / result.duration_secs + } else { + 0.0 + } + }) +} + +/// #3602/#3826/#3606: the `backend` object distinguishing a deliberate CPU +/// run from a rejected GPU run. See the long comment on its call site in +/// [`build_final_json`] for why each field exists and why `accel_forced` +/// alone cannot be dropped in favor of `gpu_attempted`. +fn backend_json(result: &RunResult, accel_forced: bool) -> serde_json::Value { + serde_json::json!({ + "requested": if accel_forced { "gpu" } else { "default" }, + "ran": if result.used_gpu == Some(true) { "gpu" } else { "cpu" }, + "fell_back": result.used_gpu == Some(false) + && (accel_forced || result.gpu_attempted == Some(true)), + }) +} + /// Build the terminal JSON blob shared by `--json` and `--stream` final events. fn build_final_json( result: &RunResult, @@ -602,13 +655,7 @@ fn build_final_json( accel_forced: bool, ) -> serde_json::Value { let tokens_generated = result.tokens_generated.unwrap_or(0); - let tok_per_sec = result.tok_per_sec.unwrap_or_else(|| { - if result.duration_secs > 0.0 { - tokens_generated as f64 / result.duration_secs - } else { - 0.0 - } - }); + let tok_per_sec = effective_tok_per_sec(result, tokens_generated); // GH-250: Include generated token IDs for parity checking let tokens_json = result.generated_tokens.as_deref().unwrap_or(&[]); serde_json::json!({ @@ -634,6 +681,11 @@ fn build_final_json( // is the whole window (load, upload, F2, generation), kept for compatibility. "generation_ms": result.usage.generation_ms, "setup_ms": result.usage.setup_ms, + // #4026: always present, and `null` unless `--logprobs K` asked for it; then + // the prompt ids the model read and, per generated step, the K most likely + // tokens before any penalty or sampling. A path that cannot record them + // refused the run. + "logprobs": result.logprobs, // #3602: `used_gpu: false` alone collapses two different outcomes — "no // accelerator was asked for" and "one was asked for, attempted, and // REFUSED at runtime". A consumer cannot tell a CPU run from a rejected @@ -673,12 +725,7 @@ fn build_final_json( // never Fail, so a backend that did not report has not reported a // fallback. That is `reconcile_accelerator`'s own rule, and the JSON // must not contradict the check that runs beside it. - "backend": { - "requested": if accel_forced { "gpu" } else { "default" }, - "ran": if result.used_gpu == Some(true) { "gpu" } else { "cpu" }, - "fell_back": result.used_gpu == Some(false) - && (accel_forced || result.gpu_attempted == Some(true)), - }, + "backend": backend_json(result, accel_forced), }) } diff --git a/crates/apr-cli/src/commands/run_tests_accel_reconcile.rs b/crates/apr-cli/src/commands/run_tests_accel_reconcile.rs index b496ab747a..dd99b3f85f 100644 --- a/crates/apr-cli/src/commands/run_tests_accel_reconcile.rs +++ b/crates/apr-cli/src/commands/run_tests_accel_reconcile.rs @@ -54,6 +54,7 @@ #[test] fn a_forced_accelerator_that_ran_on_cpu_is_refused() { let result = RunResult { + logprobs: None, text: "Hello".to_string(), duration_secs: 33.646, cached: true, @@ -142,6 +143,7 @@ fn a_successful_gpu_run_is_not_labelled_a_fallback() { fn gpu_result(used_gpu: Option) -> RunResult { RunResult { + logprobs: None, text: "Hello".to_string(), duration_secs: 1.0, cached: true, diff --git a/crates/apr-cli/src/commands/run_tests_chrome_trace.rs b/crates/apr-cli/src/commands/run_tests_chrome_trace.rs index 1c03f69512..d6cd2d92de 100644 --- a/crates/apr-cli/src/commands/run_tests_chrome_trace.rs +++ b/crates/apr-cli/src/commands/run_tests_chrome_trace.rs @@ -16,6 +16,7 @@ #[test] fn test_chrome_trace_event_categories() { let result = RunResult { + logprobs: None, text: "test".to_string(), duration_secs: 2.0, cached: false, @@ -48,6 +49,7 @@ #[test] fn test_chrome_trace_zero_tokens() { let result = RunResult { + logprobs: None, text: String::new(), duration_secs: 0.5, cached: true, @@ -69,6 +71,7 @@ #[test] fn test_chrome_trace_no_tokens_generated_field() { let result = RunResult { + logprobs: None, text: "output".to_string(), duration_secs: 1.0, cached: false, @@ -90,6 +93,7 @@ #[test] fn test_chrome_trace_metadata_source() { let result = RunResult { + logprobs: None, text: "hi".to_string(), duration_secs: 0.1, cached: false, @@ -111,6 +115,7 @@ #[test] fn test_chrome_trace_event_format() { let result = RunResult { + logprobs: None, text: "test".to_string(), duration_secs: 1.0, cached: false, @@ -142,6 +147,7 @@ #[test] fn test_chrome_trace_token_count() { let result = RunResult { + logprobs: None, text: "test".to_string(), duration_secs: 5.0, cached: false, @@ -163,6 +169,7 @@ #[test] fn test_chrome_trace_display_time_unit() { let result = RunResult { + logprobs: None, text: "t".to_string(), duration_secs: 1.0, cached: false, @@ -184,6 +191,7 @@ // This test writes to CWD. Use a unique tempdir and set_current_dir. // May conflict with parallel tests, but the function itself should not panic. let result = RunResult { + logprobs: None, text: "Hello world".to_string(), duration_secs: 1.0, cached: false, @@ -218,6 +226,7 @@ let target = dir.join("requested.json"); let result = RunResult { + logprobs: None, text: "Hello world".to_string(), duration_secs: 1.0, cached: false, @@ -249,6 +258,7 @@ #[test] fn test_print_benchmark_results_text() { let result = RunResult { + logprobs: None, text: "test output".to_string(), duration_secs: 2.0, cached: false, @@ -266,6 +276,7 @@ #[test] fn test_print_benchmark_results_json() { let result = RunResult { + logprobs: None, text: "test".to_string(), duration_secs: 1.0, cached: false, @@ -283,6 +294,7 @@ #[test] fn test_print_benchmark_zero_duration() { let result = RunResult { + logprobs: None, text: "".to_string(), duration_secs: 0.0, cached: false, diff --git a/crates/apr-cli/src/commands/run_tests_inference_output.rs b/crates/apr-cli/src/commands/run_tests_inference_output.rs index 824885b810..0f2f8ade08 100644 --- a/crates/apr-cli/src/commands/run_tests_inference_output.rs +++ b/crates/apr-cli/src/commands/run_tests_inference_output.rs @@ -8,6 +8,7 @@ #[test] fn inference_output_fields() { let output = InferenceOutput { + logprobs: None, text: "hello".to_string(), tokens_generated: Some(5), inference_ms: Some(10.0), @@ -27,6 +28,7 @@ #[test] fn inference_output_no_metrics() { let output = InferenceOutput { + logprobs: None, text: "result".to_string(), tokens_generated: None, inference_ms: None, diff --git a/crates/apr-cli/src/commands/run_tests_layer_trace.rs b/crates/apr-cli/src/commands/run_tests_layer_trace.rs index adc85c3ee9..48391aaee4 100644 --- a/crates/apr-cli/src/commands/run_tests_layer_trace.rs +++ b/crates/apr-cli/src/commands/run_tests_layer_trace.rs @@ -12,6 +12,7 @@ fn layer_trace_result(duration_secs: f64, tokens: usize) -> RunResult { RunResult { + logprobs: None, text: "hi".to_string(), duration_secs, cached: true, diff --git a/crates/apr-cli/src/commands/run_tests_logprobs_surface_4026.rs b/crates/apr-cli/src/commands/run_tests_logprobs_surface_4026.rs new file mode 100644 index 0000000000..b3cc5e79af --- /dev/null +++ b/crates/apr-cli/src/commands/run_tests_logprobs_surface_4026.rs @@ -0,0 +1,66 @@ +// ============================================================================= +// #4026: `--logprobs K` is refused by name on every surface that would drop it +// ============================================================================= + +/// Over every flag combination, K > 0 passes exactly where the machine document +/// is printed, and K = 0 passes everywhere. Pinned to `emits_machine_output`, so +/// the refusal and the arms of `print_run_output` cannot drift apart. +#[test] +fn logprobs_refusal_matches_the_machine_output_predicate() { + for &stream in &[false, true] { + for &benchmark in &[false, true] { + for fmt in ["json", "text", "table"] { + assert!( + refuse_logprobs_without_machine_output(0, stream, fmt, benchmark).is_ok(), + "K=0 refused at stream={stream} fmt={fmt} benchmark={benchmark}" + ); + assert_eq!( + refuse_logprobs_without_machine_output(5, stream, fmt, benchmark).is_ok(), + emits_machine_output(stream, fmt, benchmark), + "K=5 at stream={stream} fmt={fmt} benchmark={benchmark}" + ); + } + } + } +} + +/// The text surface: the run used to record the logprobs, print none and exit 0. +#[test] +fn logprobs_on_the_text_surface_is_refused_by_name() { + let msg = refuse_logprobs_without_machine_output(3, false, "text", false) + .expect_err("refused") + .to_string(); + assert!(msg.contains("--logprobs 3"), "{msg}"); + assert!(msg.contains("text output"), "{msg}"); +} + +/// `--json --benchmark` prints the human benchmark blob, not the JSON document, +/// so it is a `--benchmark` refusal even with `--json`. +#[test] +fn logprobs_with_json_benchmark_is_refused_by_name() { + let msg = refuse_logprobs_without_machine_output(2, false, "json", true) + .expect_err("refused") + .to_string(); + assert!(msg.contains("--logprobs 2"), "{msg}"); + assert!(msg.contains("--benchmark output"), "{msg}"); +} + +/// `--logprobs` is capped at 20 at the flag, as `top_logprobs` is on HTTP: 20 +/// parses, 21 does not. Parsed on a 16 MB stack, as in the #3754 tests. +#[test] +fn logprobs_flag_is_capped_at_twenty() { + let parses = |k: &'static str| { + std::thread::Builder::new() + .stack_size(16 * 1024 * 1024) + .spawn(move || { + use clap::Parser; + crate::Cli::try_parse_from(["apr", "run", "m.gguf", "--json", "--logprobs", k]) + .is_ok() + }) + .expect("spawn parse thread") + .join() + .expect("join parse thread") + }; + assert!(parses("20"), "--logprobs 20 must parse"); + assert!(!parses("21"), "--logprobs 21 must be refused at the flag"); +} diff --git a/crates/apr-cli/src/commands/run_tests_model_source.rs b/crates/apr-cli/src/commands/run_tests_model_source.rs index 3e5bd41760..d35e3a5230 100644 --- a/crates/apr-cli/src/commands/run_tests_model_source.rs +++ b/crates/apr-cli/src/commands/run_tests_model_source.rs @@ -31,6 +31,7 @@ #[test] fn test_run_result_debug() { let result = RunResult { + logprobs: None, text: "Hello".to_string(), duration_secs: 0.1, cached: true, @@ -141,6 +142,7 @@ #[test] fn test_run_result_clone() { let result = RunResult { + logprobs: None, text: "Test".to_string(), duration_secs: 1.0, cached: false, diff --git a/crates/apr-cli/src/commands/run_tests_parse_token_ids.rs b/crates/apr-cli/src/commands/run_tests_parse_token_ids.rs index 778dd02cbb..6210a31830 100644 --- a/crates/apr-cli/src/commands/run_tests_parse_token_ids.rs +++ b/crates/apr-cli/src/commands/run_tests_parse_token_ids.rs @@ -60,6 +60,7 @@ #[test] fn run_result_tokens_generated_none_vs_zero() { let result_none = RunResult { + logprobs: None, text: String::new(), duration_secs: 0.0, cached: false, @@ -72,6 +73,7 @@ usage: Default::default(), }; let result_zero = RunResult { + logprobs: None, text: String::new(), duration_secs: 0.0, cached: false, @@ -94,6 +96,7 @@ #[test] fn run_result_field_independence() { let result = RunResult { + logprobs: None, text: "output".to_string(), duration_secs: 1.234, cached: true, diff --git a/crates/apr-cli/src/commands/run_tests_stream_output.rs b/crates/apr-cli/src/commands/run_tests_stream_output.rs index 0f61f43a66..f53320b7ba 100644 --- a/crates/apr-cli/src/commands/run_tests_stream_output.rs +++ b/crates/apr-cli/src/commands/run_tests_stream_output.rs @@ -4,6 +4,7 @@ #[test] fn stream_output_emits_n_plus_one_json_lines() { let result = RunResult { + logprobs: None, text: "Hello world".to_string(), duration_secs: 0.25, cached: false, @@ -74,6 +75,7 @@ fn stream_output_emits_n_plus_one_json_lines() { #[test] fn stream_token_events_carry_their_own_decoded_text() { let result = RunResult { + logprobs: None, text: "I'm here to help".to_string(), duration_secs: 1.0, cached: true, @@ -121,6 +123,7 @@ fn stream_token_events_carry_their_own_decoded_text() { #[test] fn stream_token_events_degrade_to_empty_text_without_a_tokenizer() { let result = RunResult { + logprobs: None, text: "abc".to_string(), duration_secs: 1.0, cached: true, @@ -148,6 +151,7 @@ fn stream_token_events_degrade_to_empty_text_without_a_tokenizer() { #[test] fn stream_output_no_tokens_emits_only_final() { let result = RunResult { + logprobs: None, text: String::new(), duration_secs: 0.0, cached: true, @@ -175,6 +179,7 @@ fn stream_output_no_tokens_emits_only_final() { #[test] fn stream_output_none_tokens_emits_only_final() { let result = RunResult { + logprobs: None, text: String::new(), duration_secs: 0.0, cached: false, @@ -201,6 +206,7 @@ fn stream_output_none_tokens_emits_only_final() { #[test] fn build_final_json_matches_legacy_json_shape() { let result = RunResult { + logprobs: None, text: "abc".to_string(), duration_secs: 1.0, cached: true, diff --git a/crates/apr-cli/src/commands/run_tests_usage_3718.rs b/crates/apr-cli/src/commands/run_tests_usage_3718.rs index 7a18aa1366..cf374137f9 100644 --- a/crates/apr-cli/src/commands/run_tests_usage_3718.rs +++ b/crates/apr-cli/src/commands/run_tests_usage_3718.rs @@ -2,6 +2,7 @@ fn result_with(usage: RunUsage) -> RunResult { RunResult { + logprobs: None, text: "4".to_string(), duration_secs: 1.0, cached: true, diff --git a/crates/apr-cli/src/commands_enum.rs b/crates/apr-cli/src/commands_enum.rs index aa3f305237..15b571ab56 100644 --- a/crates/apr-cli/src/commands_enum.rs +++ b/crates/apr-cli/src/commands_enum.rs @@ -250,6 +250,21 @@ pub enum Commands { /// F-CLIPARITY-01 / PMAT-384 / paiml/aprender#571 #[arg(long, default_value = "64", group = batuta_common::cli_roles::SamplingKind::RepeatLastN.id())] repeat_last_n: usize, + /// Record the K most likely next tokens (token id, logit, logprob) at every + /// generated step, plus the prompt ids actually encoded, in `--json` output + /// (#4026). 0 (the default) records nothing; at most 20, as on the HTTP + /// `top_logprobs`. Refused by name on a path that cannot record them + /// (`--trace`, APR/SafeTensors, wgpu, qwen3_moe) and on an output that does + /// not print them (text, `--benchmark`, `--batch-jsonl`). With K > 0 a greedy + /// step picks its token on the host from the recorded logits, not by the + /// device argmax, so an exact tie may break differently than without it. + #[arg( + long, + value_name = "K", + default_value_t = 0, + value_parser = clap::builder::RangedU64ValueParser::::new().range(0..=20) + )] + logprobs: usize, /// Process prompt tokens one-by-one instead of batched prefill. /// Useful for debugging prefill correctness (comparing per-token attention). /// F-CLIPARITY-01 / PMAT-385 / paiml/aprender#572 diff --git a/crates/apr-cli/src/dispatch.rs b/crates/apr-cli/src/dispatch.rs index 2487a5bad9..bf71981ba0 100644 --- a/crates/apr-cli/src/dispatch.rs +++ b/crates/apr-cli/src/dispatch.rs @@ -214,13 +214,15 @@ or drop `--backend`." /// Batch JSONL mode: load the model once and process every prompt. #3723: the batch path /// renders its own prompts, so `--thinking`, which it cannot honour, is refused by name -/// rather than ignored. Extracted from `dispatch_runtime_commands` unchanged. +/// rather than ignored. Extracted from `dispatch_runtime_commands` unchanged. #4026: the +/// batch path records no per-step logprobs, so `--logprobs K` > 0 is refused the same way. #[cfg(feature = "inference")] #[allow(clippy::too_many_arguments)] fn run_batch_jsonl( source: &str, batch_file: &std::path::Path, thinking_requested: bool, + logprobs_top_k: usize, max_tokens: usize, temperature: f32, top_k: usize, @@ -234,6 +236,13 @@ renders its own prompts. Run the prompts through `apr run --thinking` instead." .to_string(), )); } + if logprobs_top_k > 0 { + return Err(CliError::ValidationFailed(format!( + "--logprobs {logprobs_top_k} is not supported with --batch-jsonl (#4026): the \ +batch path records no per-step logprobs. Run each prompt through \ +`apr run --json --logprobs {logprobs_top_k}` instead." + ))); + } run::run_batch( source, batch_file, @@ -251,123 +260,7 @@ fn dispatch_runtime_commands(cli: &Cli) -> Option> { // GH-685: forward cli.verbose to check Commands::Check { file, no_gpu, json } => crate::error::resolve_model_path(file) .and_then(|r| commands::check::run(&r, *no_gpu, *json || cli.json, cli.verbose)), - Commands::Run { - source, - positional_prompt, - input, - prompt, - max_tokens, - stream, - language, - task, - format, - no_gpu, - gpu, - revalidate, - offline, - benchmark, - trace, - trace_steps, - trace_verbose, - trace_output, - trace_level, - trace_payload, - profile, - temperature, - top_k, - top_p, - seed, - repeat_penalty, - repeat_last_n, - chat, - split_prompt, - batch_jsonl, - verbose, - backend: BackendArg { backend }, - thinking, - } => { - request_f2_revalidate(*revalidate); - // GH-614: --backend cpu forces CPU-only inference - let backend_forces_cpu = backend.as_deref() == Some("cpu"); - if let Err(e) = check_run_backend(backend.as_deref()) { - return Some(Err(e)); - } - // PERF-021: `apr run` is the surface #2696 was MEASURED through — - // 15.7 tok/s decode, 0.099x llama.cpp — and it was the surface with - // no guard. The jidoka refusal landed only on `apr serve`, one - // command over from where the defect was recorded. - // - // Placed ABOVE `effective_no_gpu` and above the `batch_jsonl` early - // return below: that return bypasses `dispatch_run` entirely, so a - // check any lower is skipped by `apr run --gpu --batch-jsonl f.jsonl`. - if let Err(e) = crate::accel::ensure_available( - *gpu && !*no_gpu, - &crate::accel::asked_flag(*gpu, backend.as_deref()), - ) { - return Some(Err(e)); - } - - let accel_forced = run_accelerator_forced(*gpu, *no_gpu, backend.as_deref()); - - // GH-326: --gpu overrides --no-gpu when both specified - let effective_no_gpu = if *gpu { - false - } else { - *no_gpu || backend_forces_cpu - }; - - // Batch JSONL mode: load model once, process all prompts - #[cfg(feature = "inference")] - if let Some(ref batch_file) = batch_jsonl { - return Some(run_batch_jsonl( - source, - batch_file, - thinking.mode().is_some(), - *max_tokens, - *temperature, - *top_k, - effective_no_gpu, - *verbose || cli.verbose, - )); - } - - // GH-240: merge global --json flag into output format - let effective_format = if cli.json { "json" } else { format.as_str() }; - let trace_steps = trace_steps.as_deref().map(batuta_common::cli_roles::strings); - dispatch_run( - source, - positional_prompt.as_ref().map(PromptText::as_string), - input.as_deref(), - prompt.as_ref().map(PromptText::as_string), - *max_tokens, - *stream, - language.as_deref(), - task.as_deref(), - effective_format, - effective_no_gpu, - accel_forced, - *offline, - *benchmark, - *verbose || cli.verbose, - *trace, - *trace_payload, - trace_steps.as_deref(), - *trace_verbose, - trace_output.as_deref().map(Path::to_path_buf), - trace_level.as_str(), - *profile, - *chat, - // PMAT-496: Sampling parameters — no longer silently dropped - *temperature, - *top_k, - *top_p, - *seed, - *repeat_penalty, - *repeat_last_n, - *split_prompt, - thinking.mode(), - ) - } + Commands::Run { .. } => return dispatch_run_command(cli), Commands::Serve { command } => dispatch_serve_command(command, cli), @@ -407,6 +300,135 @@ fn dispatch_runtime_commands(cli: &Cli) -> Option> { }) } +/// `apr run`, split out of [`dispatch_runtime_commands`] unchanged except that the batch +/// path now gets `--logprobs` to refuse (#4026): inline, this arm held that function's +/// cognitive complexity at 34 against the pre-commit ceiling of 25. `None` for any other +/// command. +fn dispatch_run_command(cli: &Cli) -> Option> { + let Commands::Run { + source, + positional_prompt, + input, + prompt, + max_tokens, + stream, + language, + task, + format, + no_gpu, + gpu, + revalidate, + offline, + benchmark, + trace, + trace_steps, + trace_verbose, + trace_output, + trace_level, + trace_payload, + profile, + temperature, + top_k, + top_p, + seed, + repeat_penalty, + repeat_last_n, + logprobs, + chat, + split_prompt, + batch_jsonl, + verbose, + backend: BackendArg { backend }, + thinking, + } = cli.command.as_ref() + else { + return None; + }; + request_f2_revalidate(*revalidate); + // GH-614: --backend cpu forces CPU-only inference + let backend_forces_cpu = backend.as_deref() == Some("cpu"); + if let Err(e) = check_run_backend(backend.as_deref()) { + return Some(Err(e)); + } + // PERF-021: `apr run` is the surface #2696 was MEASURED through — + // 15.7 tok/s decode, 0.099x llama.cpp — and it was the surface with + // no guard. The jidoka refusal landed only on `apr serve`, one + // command over from where the defect was recorded. + // + // Placed ABOVE `effective_no_gpu` and above the `batch_jsonl` early + // return below: that return bypasses `dispatch_run` entirely, so a + // check any lower is skipped by `apr run --gpu --batch-jsonl f.jsonl`. + if let Err(e) = crate::accel::ensure_available( + *gpu && !*no_gpu, + &crate::accel::asked_flag(*gpu, backend.as_deref()), + ) { + return Some(Err(e)); + } + + let accel_forced = run_accelerator_forced(*gpu, *no_gpu, backend.as_deref()); + + // GH-326: --gpu overrides --no-gpu when both specified + let effective_no_gpu = if *gpu { + false + } else { + *no_gpu || backend_forces_cpu + }; + + // Batch JSONL mode: load model once, process all prompts + #[cfg(feature = "inference")] + if let Some(ref batch_file) = batch_jsonl { + return Some(run_batch_jsonl( + source, + batch_file, + thinking.mode().is_some(), + *logprobs, + *max_tokens, + *temperature, + *top_k, + effective_no_gpu, + *verbose || cli.verbose, + )); + } + + // GH-240: merge global --json flag into output format + let effective_format = if cli.json { "json" } else { format.as_str() }; + let trace_steps = trace_steps.as_deref().map(batuta_common::cli_roles::strings); + Some(dispatch_run( + source, + positional_prompt.as_ref().map(PromptText::as_string), + input.as_deref(), + prompt.as_ref().map(PromptText::as_string), + *max_tokens, + *stream, + language.as_deref(), + task.as_deref(), + effective_format, + effective_no_gpu, + accel_forced, + *offline, + *benchmark, + *verbose || cli.verbose, + *trace, + *trace_payload, + trace_steps.as_deref(), + *trace_verbose, + trace_output.as_deref().map(Path::to_path_buf), + trace_level.as_str(), + *profile, + *chat, + // PMAT-496: Sampling parameters — no longer silently dropped + *temperature, + *top_k, + *top_p, + *seed, + *repeat_penalty, + *repeat_last_n, + *split_prompt, + thinking.mode(), + *logprobs, + )) +} + /// Borrowed view of the parsed `apr code` flags, so [`dispatch_code_command`] /// takes one argument instead of ten. struct CodeArgs<'a> { @@ -671,46 +693,16 @@ fn dispatch_diagnostic_commands(cli: &Cli) -> Option> { save_tensor_dir, save_tensor_layers, } => crate::error::resolve_model_path(file).and_then(|r| { - // SHIP-007 layer-0 stage diff: when --save-tensor is set on a - // .apr file, dispatch to the end-to-end save-tensor wrapper - // (PR-A clap → PR-B plan → PR-C-real step1+2 wrapper). For - // .gguf/.safetensors and the common no-flag case, fall through - // to the existing trace path. #[cfg(feature = "inference")] - if let Some(stages) = save_tensor.as_deref() { - let ext_lower = r - .extension() - .and_then(|e| e.to_str()) - .map(str::to_ascii_lowercase); - match ext_lower.as_deref() { - Some("apr") => { - return crate::commands::trace_save_tensor::run_save_tensor_apr( - &r, - stages, - save_tensor_dir.as_deref(), - save_tensor_layers, - ); - } - Some("gguf") => { - // M-MOE-SUB-2 step (a) CLI completion: GGUF dispatches - // to the MoE-traced wireup if the arch is qwen3_moe; - // dense-GGUF will be wired in SHIP-007 PR-E. - return crate::commands::trace_save_tensor::run_save_tensor_gguf_moe( - &r, - stages, - save_tensor_dir.as_deref(), - save_tensor_layers, - ); - } - _ => { - eprintln!( - "apr trace --save-tensor: only .apr and .gguf (qwen3_moe arch) \ - supported today; .safetensors will be wired in SHIP-007 PR-E \ - (got {})", - r.display() - ); - } - } + if let Some(done) = save_tensor.as_deref().and_then(|stages| { + dispatch_trace_save_tensor( + &r, + stages, + save_tensor_dir.as_deref(), + save_tensor_layers, + ) + }) { + return done; } trace::run( &r, @@ -752,42 +744,66 @@ fn dispatch_diagnostic_commands(cli: &Cli) -> Option> { quant_roundtrip, threshold, no_threshold, - } => { + } => crate::error::resolve_model_path(file1).and_then(|r1| { + let r2 = crate::error::resolve_model_path(file2)?; if *quant_roundtrip { // CRUX-B-20: per-tensor quant roundtrip error report. - crate::error::resolve_model_path(file1).and_then(|r1| { - crate::error::resolve_model_path(file2).and_then(|r2| { - dispatch_quant_roundtrip( - &r1, - &r2, - *threshold, - *no_threshold, - *json || cli.json, - ) - }) - }) + dispatch_quant_roundtrip(&r1, &r2, *threshold, *no_threshold, *json || cli.json) } else { - crate::error::resolve_model_path(file1).and_then(|r1| { - crate::error::resolve_model_path(file2).and_then(|r2| { - diff::run( - &r1, - &r2, - *weights, - *values, - filter.as_deref(), - *limit, - *transpose_aware, - *json || cli.json, - ) - }) - }) + diff::run( + &r1, + &r2, + *weights, + *values, + filter.as_deref(), + *limit, + *transpose_aware, + *json || cli.json, + ) } - } + }), _ => return None, }) } +/// SHIP-007 layer-0 stage diff: `apr trace --save-tensor` on a `.apr` file runs the +/// end-to-end save-tensor wrapper (PR-A clap → PR-B plan → PR-C-real step1+2 wrapper). +/// `None` (after naming the extension) falls through to the existing trace path. +/// Split out of [`dispatch_diagnostic_commands`] unchanged: inline, it held that +/// function's cognitive complexity at 30 against the pre-commit ceiling of 25. +#[cfg(feature = "inference")] +fn dispatch_trace_save_tensor( + r: &Path, + stages: &str, + dir: Option<&Path>, + layers: &str, +) -> Option> { + let ext_lower = r + .extension() + .and_then(|e| e.to_str()) + .map(str::to_ascii_lowercase); + match ext_lower.as_deref() { + Some("apr") => Some(crate::commands::trace_save_tensor::run_save_tensor_apr( + r, stages, dir, layers, + )), + // M-MOE-SUB-2 step (a) CLI completion: GGUF dispatches to the MoE-traced + // wireup if the arch is qwen3_moe; dense-GGUF will be wired in SHIP-007 PR-E. + Some("gguf") => Some( + crate::commands::trace_save_tensor::run_save_tensor_gguf_moe(r, stages, dir, layers), + ), + _ => { + eprintln!( + "apr trace --save-tensor: only .apr and .gguf (qwen3_moe arch) \ + supported today; .safetensors will be wired in SHIP-007 PR-E \ + (got {})", + r.display() + ); + None + } + } +} + /// CRUX-B-20 — render an `apr diff --quant-roundtrip` report. fn dispatch_quant_roundtrip( reference: &std::path::Path, @@ -1218,3 +1234,49 @@ fn dispatch_model_commands(cli: &Cli) -> Option> { _ => return None, }) } + +#[cfg(all(test, feature = "inference"))] +mod batch_jsonl_logprobs_4026 { + use super::run_batch_jsonl; + use std::path::Path; + + /// #4026: the batch path records no per-step logprobs, so `--logprobs K` > 0 is + /// refused by name before anything is read. Neither path needs to exist. + #[test] + fn logprobs_under_batch_jsonl_is_refused_by_name() { + let err = run_batch_jsonl( + "/nonexistent-4026/model.gguf", + Path::new("/nonexistent-4026/prompts.jsonl"), + false, + 4, + 8, + 0.0, + 1, + true, + false, + ) + .expect_err("refused"); + let msg = err.to_string(); + assert!(msg.contains("--logprobs 4"), "{msg}"); + assert!(msg.contains("--batch-jsonl"), "{msg}"); + } + + /// K = 0 is not refused: the call reaches the batch path and fails there, on the + /// missing files, with an error that does not name `--logprobs`. + #[test] + fn logprobs_off_is_not_refused_under_batch_jsonl() { + let err = run_batch_jsonl( + "/nonexistent-4026/model.gguf", + Path::new("/nonexistent-4026/prompts.jsonl"), + false, + 0, + 8, + 0.0, + 1, + true, + false, + ) + .expect_err("missing files"); + assert!(!err.to_string().contains("--logprobs"), "{err}"); + } +} diff --git a/crates/apr-cli/src/dispatch_run.rs b/crates/apr-cli/src/dispatch_run.rs index dfa2ccec3a..e7b0734531 100644 --- a/crates/apr-cli/src/dispatch_run.rs +++ b/crates/apr-cli/src/dispatch_run.rs @@ -36,6 +36,8 @@ fn dispatch_run( split_prompt: bool, // #3723: `--thinking on|off`, None when absent. thinking: Option, + // #4026: `--logprobs K`, 0 when absent. + logprobs_top_k: usize, ) -> Result<(), CliError> { let effective_trace = trace || trace_payload; let effective_trace_level = if trace_payload { @@ -74,6 +76,7 @@ fn dispatch_run( split_prompt, chat_template, thinking, + logprobs_top_k, ) } diff --git a/crates/apr-cli/src/lib_parse_rosetta.rs b/crates/apr-cli/src/lib_parse_rosetta.rs index 66d5336f35..990df064f0 100644 --- a/crates/apr-cli/src/lib_parse_rosetta.rs +++ b/crates/apr-cli/src/lib_parse_rosetta.rs @@ -320,6 +320,7 @@ seed: 299_792_458, repeat_penalty: 1.0, repeat_last_n: 64, + logprobs: 0, split_prompt: false, batch_jsonl: None, verbose: false, diff --git a/crates/aprender-orchestrate/src/agent/driver/realizar.rs b/crates/aprender-orchestrate/src/agent/driver/realizar.rs index 0400bc8045..03622c537e 100644 --- a/crates/aprender-orchestrate/src/agent/driver/realizar.rs +++ b/crates/aprender-orchestrate/src/agent/driver/realizar.rs @@ -97,6 +97,7 @@ impl LlmDriver for RealizarDriver { use_mock_backend: false, force_chat_template: false, thinking: None, + logprobs_top_k: 0, stop_tokens: vec![], }; @@ -181,54 +182,20 @@ fn parse_tool_calls_envelope(text: &str) -> (String, Vec) { let mut cursor = text; loop { - // Find next tool call start — try first, then ```json - let xml_pos = cursor.find(""); - let md_pos = cursor.find("```json"); - - let (start, tag_len, is_markdown) = match (xml_pos, md_pos) { - (Some(x), Some(m)) if x <= m => (x, "".len(), false), - (Some(x), None) => (x, "".len(), false), - (_, Some(m)) => (m, "```json".len(), true), - (None, None) => { - remaining.push_str(cursor); - break; - } + let Some((start, tag_len, is_markdown)) = next_envelope_start(cursor) else { + remaining.push_str(cursor); + break; }; remaining.push_str(&cursor[..start]); - let after_tag = &cursor[start + tag_len..]; - - // Find closing tag and extract JSON - let (json_str, advance_past) = if is_markdown { - // Markdown: ```json\n...\n``` - if let Some(end) = after_tag.find("```") { - (&after_tag[..end], &after_tag[end + "```".len()..]) - } else { - (after_tag, "") - } - } else if let Some(end) = after_tag.find("") { - (&after_tag[..end], &after_tag[end + "".len()..]) - } else { - // PMAT-158: No closing tag — try parsing to end-of-string - (after_tag, "") - }; - let json_str = json_str.trim(); - - if let Ok(parsed) = serde_json::from_str::(json_str) { - // Must have "name" field to be a tool call (not just any JSON) - if let Some(name) = parsed.get("name").and_then(|n| n.as_str()) { - let name = name.to_string(); - let input = parsed.get("input").cloned().unwrap_or(serde_json::json!({})); - call_counter += 1; - tool_calls.push(ToolCall { id: format!("local-{call_counter}"), name, input }); - } else { - remaining.push_str(&cursor[start..]); - break; - } - } else { + let (json_str, advance_past) = split_envelope_body(&cursor[start + tag_len..], is_markdown); + + let Some((name, input)) = envelope_tool_call(json_str) else { remaining.push_str(&cursor[start..]); break; - } + }; + call_counter += 1; + tool_calls.push(ToolCall { id: format!("local-{call_counter}"), name, input }); cursor = advance_past; if cursor.is_empty() { @@ -239,6 +206,41 @@ fn parse_tool_calls_envelope(text: &str) -> (String, Vec) { (remaining.trim().to_string(), tool_calls) } +/// Find the next tool call start: `` when it comes first, else ```json. +/// Returns its byte offset, the opening tag's length, and whether it is the +/// markdown fence; `None` when neither occurs. Split out of +/// [`parse_tool_calls_envelope`], whose cognitive complexity was 32 against the +/// pre-commit ceiling of 25. +fn next_envelope_start(cursor: &str) -> Option<(usize, usize, bool)> { + match (cursor.find(""), cursor.find("```json")) { + (Some(x), Some(m)) if x <= m => Some((x, "".len(), false)), + (Some(x), None) => Some((x, "".len(), false)), + (_, Some(m)) => Some((m, "```json".len(), true)), + (None, None) => None, + } +} + +/// Split the text after an opening tag into the JSON body and the text past the +/// closing tag (```` ``` ```` for the markdown fence, `` otherwise). +/// PMAT-158: no closing tag — the body runs to end-of-string. +fn split_envelope_body(after_tag: &str, is_markdown: bool) -> (&str, &str) { + let close = if is_markdown { "```" } else { "" }; + match after_tag.find(close) { + Some(end) => (&after_tag[..end], &after_tag[end + close.len()..]), + None => (after_tag, ""), + } +} + +/// The tool call in an envelope body, or `None` when the body is not JSON or has +/// no string `"name"` field (a tool call, not just any JSON). `"input"` defaults +/// to `{}`. +fn envelope_tool_call(json_str: &str) -> Option<(String, serde_json::Value)> { + let parsed = serde_json::from_str::(json_str.trim()).ok()?; + let name = parsed.get("name").and_then(|n| n.as_str())?.to_string(); + let input = parsed.get("input").cloned().unwrap_or(serde_json::json!({})); + Some((name, input)) +} + /// CCPA-m296 salvage parser: recover a tool call the model emitted OUTSIDE the /// exact `` / ```json envelope, but in an unambiguous, recoverable /// shape. Two recoverable shapes are accepted, in priority order: diff --git a/crates/aprender-review-experiment/Cargo.toml b/crates/aprender-review-experiment/Cargo.toml index 6895c15ca9..be2ca3645a 100644 --- a/crates/aprender-review-experiment/Cargo.toml +++ b/crates/aprender-review-experiment/Cargo.toml @@ -12,10 +12,14 @@ repository.workspace = true publish = false [dependencies] +half = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } +regex = { workspace = true } +toml = { workspace = true } sha2 = { workspace = true } ureq = { workspace = true } +zstd = { workspace = true } [lints] workspace = true diff --git a/crates/aprender-review-experiment/data/gitleaks/LICENSE b/crates/aprender-review-experiment/data/gitleaks/LICENSE new file mode 100644 index 0000000000..3c270b36c5 --- /dev/null +++ b/crates/aprender-review-experiment/data/gitleaks/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2019 Zachary Rice + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/crates/aprender-review-experiment/data/gitleaks/gitleaks-v8.30.1.toml b/crates/aprender-review-experiment/data/gitleaks/gitleaks-v8.30.1.toml new file mode 100644 index 0000000000..256f64790e --- /dev/null +++ b/crates/aprender-review-experiment/data/gitleaks/gitleaks-v8.30.1.toml @@ -0,0 +1,3209 @@ +# This file has been auto-generated. Do not edit manually. +# If you would like to contribute new rules, please use +# cmd/generate/config/main.go and follow the contributing guidelines +# at https://github.com/gitleaks/gitleaks/blob/master/CONTRIBUTING.md +# +# How the hell does secret scanning work? Read this: +# https://lookingatcomputer.substack.com/p/regex-is-almost-all-you-need +# +# This is the default gitleaks configuration file. +# Rules and allowlists are defined within this file. +# Rules instruct gitleaks on what should be considered a secret. +# Allowlists instruct gitleaks on what is allowed, i.e. not a secret. + +title = "gitleaks config" + +# minVersion indicates the minimum Gitleaks version required to use this config. +# If the running version is older, a warning will be logged and not all +# config-enabled features are guaranteed to work. +minVersion = "v8.25.0" + +# TODO: change to [[allowlists]] +[allowlist] +description = "global allow lists" +paths = [ + '''gitleaks\.toml''', + '''(?i)\.(?:bmp|gif|jpe?g|png|svg|tiff?)$''', + '''(?i)\.(?:eot|[ot]tf|woff2?)$''', + '''(?i)\.(?:docx?|xlsx?|pdf|bin|socket|vsidx|v2|suo|wsuo|.dll|pdb|exe|gltf)$''', + '''go\.(?:mod|sum|work(?:\.sum)?)$''', + '''(?:^|/)vendor/modules\.txt$''', + '''(?:^|/)vendor/(?:github\.com|golang\.org/x|google\.golang\.org|gopkg\.in|istio\.io|k8s\.io|sigs\.k8s\.io)(?:/.*)?$''', + '''(?:^|/)gradlew(?:\.bat)?$''', + '''(?:^|/)gradle\.lockfile$''', + '''(?:^|/)mvnw(?:\.cmd)?$''', + '''(?:^|/)\.mvn/wrapper/MavenWrapperDownloader\.java$''', + '''(?:^|/)node_modules(?:/.*)?$''', + '''(?:^|/)(?:deno\.lock|npm-shrinkwrap\.json|package-lock\.json|pnpm-lock\.yaml|yarn\.lock)$''', + '''(?:^|/)bower_components(?:/.*)?$''', + '''(?:^|/)(?:angular|bootstrap|jquery(?:-?ui)?|plotly|swagger-?ui)[a-zA-Z0-9.-]*(?:\.min)?\.js(?:\.map)?$''', + '''(?:^|/)javascript\.json$''', + '''(?:^|/)(?:Pipfile|poetry)\.lock$''', + '''(?i)(?:^|/)(?:v?env|virtualenv)/lib(?:64)?(?:/.*)?$''', + '''(?i)(?:^|/)(?:lib(?:64)?/python[23](?:\.\d{1,2})+|python/[23](?:\.\d{1,2})+/lib(?:64)?)(?:/.*)?$''', + '''(?i)(?:^|/)[a-z0-9_.]+-[0-9.]+\.dist-info(?:/.+)?$''', + '''(?:^|/)vendor/(?:bundle|ruby)(?:/.*?)?$''', + '''\.gem$''', + '''verification-metadata\.xml''', + '''Database.refactorlog''', + '''(?:^|/)\.git$''', +] +regexes = [ + '''(?i)^true|false|null$''', + '''^(?i:a+|b+|c+|d+|e+|f+|g+|h+|i+|j+|k+|l+|m+|n+|o+|p+|q+|r+|s+|t+|u+|v+|w+|x+|y+|z+|\*+|\.+)$''', + '''^\$(?:\d+|{\d+})$''', + '''^\$(?:[A-Z_]+|[a-z_]+)$''', + '''^\${(?:[A-Z_]+|[a-z_]+)}$''', + '''^\{\{[ \t]*[\w ().|]+[ \t]*}}$''', + '''^\$\{\{[ \t]*(?:(?:env|github|secrets|vars)(?:\.[A-Za-z]\w+)+[\w "'&./=|]*)[ \t]*}}$''', + '''^%(?:[A-Z_]+|[a-z_]+)%$''', + '''^%[+\-# 0]?[bcdeEfFgGoOpqstTUvxX]$''', + '''^\{\d{0,2}}$''', + '''^@(?:[A-Z_]+|[a-z_]+)@$''', + '''^/Users/(?i)[a-z0-9]+/[\w .-/]+$''', + '''^/(?:bin|etc|home|opt|tmp|usr|var)/[\w ./-]+$''', +] +stopwords = [ + "014df517-39d1-4453-b7b3-9930c563627c", + "abcdefghijklmnopqrstuvwxyz", +] + +[[rules]] +id = "1password-secret-key" +description = "Uncovered a possible 1Password secret key, potentially compromising access to secrets in vaults." +regex = '''\bA3-[A-Z0-9]{6}-(?:(?:[A-Z0-9]{11})|(?:[A-Z0-9]{6}-[A-Z0-9]{5}))-[A-Z0-9]{5}-[A-Z0-9]{5}-[A-Z0-9]{5}\b''' +entropy = 3.8 +keywords = ["a3-"] + +[[rules]] +id = "1password-service-account-token" +description = "Uncovered a possible 1Password service account token, potentially compromising access to secrets in vaults." +regex = '''ops_eyJ[a-zA-Z0-9+/]{250,}={0,3}''' +entropy = 4 +keywords = ["ops_"] + +[[rules]] +id = "adafruit-api-key" +description = "Identified a potential Adafruit API Key, which could lead to unauthorized access to Adafruit services and sensitive data exposure." +regex = '''(?i)[\w.-]{0,50}?(?:adafruit)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9_-]{32})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["adafruit"] + +[[rules]] +id = "adobe-client-id" +description = "Detected a pattern that resembles an Adobe OAuth Web Client ID, posing a risk of compromised Adobe integrations and data breaches." +regex = '''(?i)[\w.-]{0,50}?(?:adobe)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-f0-9]{32})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 2 +keywords = ["adobe"] + +[[rules]] +id = "adobe-client-secret" +description = "Discovered a potential Adobe Client Secret, which, if exposed, could allow unauthorized Adobe service access and data manipulation." +regex = '''\b(p8e-(?i)[a-z0-9]{32})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 2 +keywords = ["p8e-"] + +[[rules]] +id = "age-secret-key" +description = "Discovered a potential Age encryption tool secret key, risking data decryption and unauthorized access to sensitive information." +regex = '''AGE-SECRET-KEY-1[QPZRY9X8GF2TVDW0S3JN54KHCE6MUA7L]{58}''' +keywords = ["age-secret-key-1"] + +[[rules]] +id = "airtable-api-key" +description = "Uncovered a possible Airtable API Key, potentially compromising database access and leading to data leakage or alteration." +regex = '''(?i)[\w.-]{0,50}?(?:airtable)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9]{17})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["airtable"] + +[[rules]] +id = "airtable-personnal-access-token" +description = "Uncovered a possible Airtable Personal AccessToken, potentially compromising database access and leading to data leakage or alteration." +regex = '''\b(pat[[:alnum:]]{14}\.[a-f0-9]{64})\b''' +keywords = ["airtable"] + +[[rules]] +id = "algolia-api-key" +description = "Identified an Algolia API Key, which could result in unauthorized search operations and data exposure on Algolia-managed platforms." +regex = '''(?i)[\w.-]{0,50}?(?:algolia)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9]{32})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["algolia"] + +[[rules]] +id = "alibaba-access-key-id" +description = "Detected an Alibaba Cloud AccessKey ID, posing a risk of unauthorized cloud resource access and potential data compromise." +regex = '''\b(LTAI(?i)[a-z0-9]{20})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 2 +keywords = ["ltai"] + +[[rules]] +id = "alibaba-secret-key" +description = "Discovered a potential Alibaba Cloud Secret Key, potentially allowing unauthorized operations and data access within Alibaba Cloud." +regex = '''(?i)[\w.-]{0,50}?(?:alibaba)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9]{30})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 2 +keywords = ["alibaba"] + +[[rules]] +id = "anthropic-admin-api-key" +description = "Detected an Anthropic Admin API Key, risking unauthorized access to administrative functions and sensitive AI model configurations." +regex = '''\b(sk-ant-admin01-[a-zA-Z0-9_\-]{93}AA)(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["sk-ant-admin01"] + +[[rules]] +id = "anthropic-api-key" +description = "Identified an Anthropic API Key, which may compromise AI assistant integrations and expose sensitive data to unauthorized access." +regex = '''\b(sk-ant-api03-[a-zA-Z0-9_\-]{93}AA)(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["sk-ant-api03"] + +[[rules]] +id = "artifactory-api-key" +description = "Detected an Artifactory api key, posing a risk unauthorized access to the central repository." +regex = '''\bAKCp[A-Za-z0-9]{69}\b''' +entropy = 4.5 +keywords = ["akcp"] + +[[rules]] +id = "artifactory-reference-token" +description = "Detected an Artifactory reference token, posing a risk of impersonation and unauthorized access to the central repository." +regex = '''\bcmVmd[A-Za-z0-9]{59}\b''' +entropy = 4.5 +keywords = ["cmvmd"] + +[[rules]] +id = "asana-client-id" +description = "Discovered a potential Asana Client ID, risking unauthorized access to Asana projects and sensitive task information." +regex = '''(?i)[\w.-]{0,50}?(?:asana)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([0-9]{16})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["asana"] + +[[rules]] +id = "asana-client-secret" +description = "Identified an Asana Client Secret, which could lead to compromised project management integrity and unauthorized access." +regex = '''(?i)[\w.-]{0,50}?(?:asana)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9]{32})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["asana"] + +[[rules]] +id = "atlassian-api-token" +description = "Detected an Atlassian API token, posing a threat to project management and collaboration tool security and data confidentiality." +regex = '''(?i)[\w.-]{0,50}?(?:(?-i:ATLASSIAN|[Aa]tlassian)|(?-i:CONFLUENCE|[Cc]onfluence)|(?-i:JIRA|[Jj]ira))(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9]{20}[a-f0-9]{4})(?:[\x60'"\s;]|\\[nr]|$)|\b(ATATT3[A-Za-z0-9_\-=]{186})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 3.5 +keywords = [ + "atlassian", + "confluence", + "jira", + "atatt3", +] + +[[rules]] +id = "authress-service-client-access-key" +description = "Uncovered a possible Authress Service Client Access Key, which may compromise access control services and sensitive data." +regex = '''\b((?:sc|ext|scauth|authress)_(?i)[a-z0-9]{5,30}\.[a-z0-9]{4,6}\.(?-i:acc)[_-][a-z0-9-]{10,32}\.[a-z0-9+/_=-]{30,120})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 2 +keywords = [ + "sc_", + "ext_", + "scauth_", + "authress_", +] + +[[rules]] +id = "aws-access-token" +description = "Identified a pattern that may indicate AWS credentials, risking unauthorized cloud resource access and data breaches on AWS platforms." +regex = '''\b((?:A3T[A-Z0-9]|AKIA|ASIA|ABIA|ACCA)[A-Z2-7]{16})\b''' +entropy = 3 +keywords = [ + "a3t", + "akia", + "asia", + "abia", + "acca", +] +[[rules.allowlists]] +regexes = [ + '''.+EXAMPLE$''', +] + +[[rules]] +id = "aws-amazon-bedrock-api-key-long-lived" +description = "Identified a pattern that may indicate long-lived Amazon Bedrock API keys, risking unauthorized Amazon Bedrock usage" +regex = '''\b(ABSK[A-Za-z0-9+/]{109,269}={0,2})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 3 +keywords = ["absk"] + +[[rules]] +id = "aws-amazon-bedrock-api-key-short-lived" +description = "Identified a pattern that may indicate short-lived Amazon Bedrock API keys, risking unauthorized Amazon Bedrock usage" +regex = '''bedrock-api-key-YmVkcm9jay5hbWF6b25hd3MuY29t''' +entropy = 3 +keywords = ["bedrock-api-key-"] + +[[rules]] +id = "azure-ad-client-secret" +description = "Azure AD Client Secret" +regex = '''(?:^|[\\'"\x60\s>=:(,)])([a-zA-Z0-9_~.]{3}\dQ~[a-zA-Z0-9_~.-]{31,34})(?:$|[\\'"\x60\s<),])''' +entropy = 3 +keywords = ["q~"] + +[[rules]] +id = "beamer-api-token" +description = "Detected a Beamer API token, potentially compromising content management and exposing sensitive notifications and updates." +regex = '''(?i)[\w.-]{0,50}?(?:beamer)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}(b_[a-z0-9=_\-]{44})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["beamer"] + +[[rules]] +id = "bitbucket-client-id" +description = "Discovered a potential Bitbucket Client ID, risking unauthorized repository access and potential codebase exposure." +regex = '''(?i)[\w.-]{0,50}?(?:bitbucket)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9]{32})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["bitbucket"] + +[[rules]] +id = "bitbucket-client-secret" +description = "Discovered a potential Bitbucket Client Secret, posing a risk of compromised code repositories and unauthorized access." +regex = '''(?i)[\w.-]{0,50}?(?:bitbucket)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9=_\-]{64})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["bitbucket"] + +[[rules]] +id = "bittrex-access-key" +description = "Identified a Bittrex Access Key, which could lead to unauthorized access to cryptocurrency trading accounts and financial loss." +regex = '''(?i)[\w.-]{0,50}?(?:bittrex)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9]{32})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["bittrex"] + +[[rules]] +id = "bittrex-secret-key" +description = "Detected a Bittrex Secret Key, potentially compromising cryptocurrency transactions and financial security." +regex = '''(?i)[\w.-]{0,50}?(?:bittrex)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9]{32})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["bittrex"] + +[[rules]] +id = "cisco-meraki-api-key" +description = "Cisco Meraki is a cloud-managed IT solution that provides networking, security, and device management through an easy-to-use interface." +regex = '''[\w.-]{0,50}?(?i:[\w.-]{0,50}?(?:(?-i:[Mm]eraki|MERAKI))(?:[ \t\w.-]{0,20})[\s'"]{0,3})(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([0-9a-f]{40})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 3 +keywords = ["meraki"] + +[[rules]] +id = "clickhouse-cloud-api-secret-key" +description = "Identified a pattern that may indicate clickhouse cloud API secret key, risking unauthorized clickhouse cloud api access and data breaches on ClickHouse Cloud platforms." +regex = '''\b(4b1d[A-Za-z0-9]{38})\b''' +entropy = 3 +keywords = ["4b1d"] + +[[rules]] +id = "clojars-api-token" +description = "Uncovered a possible Clojars API token, risking unauthorized access to Clojure libraries and potential code manipulation." +regex = '''(?i)CLOJARS_[a-z0-9]{60}''' +entropy = 2 +keywords = ["clojars_"] + +[[rules]] +id = "cloudflare-api-key" +description = "Detected a Cloudflare API Key, potentially compromising cloud application deployments and operational security." +regex = '''(?i)[\w.-]{0,50}?(?:cloudflare)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9_-]{40})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 2 +keywords = ["cloudflare"] + +[[rules]] +id = "cloudflare-global-api-key" +description = "Detected a Cloudflare Global API Key, potentially compromising cloud application deployments and operational security." +regex = '''(?i)[\w.-]{0,50}?(?:cloudflare)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-f0-9]{37})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 2 +keywords = ["cloudflare"] + +[[rules]] +id = "cloudflare-origin-ca-key" +description = "Detected a Cloudflare Origin CA Key, potentially compromising cloud application deployments and operational security." +regex = '''\b(v1\.0-[a-f0-9]{24}-[a-f0-9]{146})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 2 +keywords = [ + "cloudflare", + "v1.0-", +] + +[[rules]] +id = "codecov-access-token" +description = "Found a pattern resembling a Codecov Access Token, posing a risk of unauthorized access to code coverage reports and sensitive data." +regex = '''(?i)[\w.-]{0,50}?(?:codecov)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9]{32})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["codecov"] + +[[rules]] +id = "cohere-api-token" +description = "Identified a Cohere Token, posing a risk of unauthorized access to AI services and data manipulation." +regex = '''[\w.-]{0,50}?(?i:[\w.-]{0,50}?(?:cohere|CO_API_KEY)(?:[ \t\w.-]{0,20})[\s'"]{0,3})(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-zA-Z0-9]{40})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 4 +keywords = [ + "cohere", + "co_api_key", +] + +[[rules]] +id = "coinbase-access-token" +description = "Detected a Coinbase Access Token, posing a risk of unauthorized access to cryptocurrency accounts and financial transactions." +regex = '''(?i)[\w.-]{0,50}?(?:coinbase)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9_-]{64})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["coinbase"] + +[[rules]] +id = "confluent-access-token" +description = "Identified a Confluent Access Token, which could compromise access to streaming data platforms and sensitive data flow." +regex = '''(?i)[\w.-]{0,50}?(?:confluent)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9]{16})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["confluent"] + +[[rules]] +id = "confluent-secret-key" +description = "Found a Confluent Secret Key, potentially risking unauthorized operations and data access within Confluent services." +regex = '''(?i)[\w.-]{0,50}?(?:confluent)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9]{64})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["confluent"] + +[[rules]] +id = "contentful-delivery-api-token" +description = "Discovered a Contentful delivery API token, posing a risk to content management systems and data integrity." +regex = '''(?i)[\w.-]{0,50}?(?:contentful)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9=_\-]{43})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["contentful"] + +[[rules]] +id = "curl-auth-header" +description = "Discovered a potential authorization token provided in a curl command header, which could compromise the curl accessed resource." +regex = '''\bcurl\b(?:.*?|.*?(?:[\r\n]{1,2}.*?){1,5})[ \t\n\r](?:-H|--header)(?:=|[ \t]{0,5})(?:"(?i)(?:Authorization:[ \t]{0,5}(?:Basic[ \t]([a-z0-9+/]{8,}={0,3})|(?:Bearer|(?:Api-)?Token)[ \t]([\w=~@.+/-]{8,})|([\w=~@.+/-]{8,}))|(?:(?:X-(?:[a-z]+-)?)?(?:Api-?)?(?:Key|Token)):[ \t]{0,5}([\w=~@.+/-]{8,}))"|'(?i)(?:Authorization:[ \t]{0,5}(?:Basic[ \t]([a-z0-9+/]{8,}={0,3})|(?:Bearer|(?:Api-)?Token)[ \t]([\w=~@.+/-]{8,})|([\w=~@.+/-]{8,}))|(?:(?:X-(?:[a-z]+-)?)?(?:Api-?)?(?:Key|Token)):[ \t]{0,5}([\w=~@.+/-]{8,}))')(?:\B|\s|\z)''' +entropy = 2.75 +keywords = ["curl"] + +[[rules]] +id = "curl-auth-user" +description = "Discovered a potential basic authorization token provided in a curl command, which could compromise the curl accessed resource." +regex = '''\bcurl\b(?:.*|.*(?:[\r\n]{1,2}.*){1,5})[ \t\n\r](?:-u|--user)(?:=|[ \t]{0,5})("(:[^"]{3,}|[^:"]{3,}:|[^:"]{3,}:[^"]{3,})"|'([^:']{3,}:[^']{3,})'|((?:"[^"]{3,}"|'[^']{3,}'|[\w$@.-]+):(?:"[^"]{3,}"|'[^']{3,}'|[\w${}@.-]+)))(?:\s|\z)''' +entropy = 2 +keywords = ["curl"] +[[rules.allowlists]] +regexes = [ + '''[^:]+:(?:change(?:it|me)|pass(?:word)?|pwd|test|token|\*+|x+)''', + '''['"]?<[^>]+>['"]?:['"]?<[^>]+>|<[^:]+:[^>]+>['"]?''', + '''[^:]+:\[[^]]+]''', + '''['"]?[^:]+['"]?:['"]?\$(?:\d|\w+|\{(?:\d|\w+)})['"]?''', + '''\$\([^)]+\):\$\([^)]+\)''', + '''['"]?\$?{{[^}]+}}['"]?:['"]?\$?{{[^}]+}}['"]?''', +] + +[[rules]] +id = "databricks-api-token" +description = "Uncovered a Databricks API token, which may compromise big data analytics platforms and sensitive data processing." +regex = '''\b(dapi[a-f0-9]{32}(?:-\d)?)(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 3 +keywords = ["dapi"] + +[[rules]] +id = "datadog-access-token" +description = "Detected a Datadog Access Token, potentially risking monitoring and analytics data exposure and manipulation." +regex = '''(?i)[\w.-]{0,50}?(?:datadog)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9]{40})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["datadog"] + +[[rules]] +id = "defined-networking-api-token" +description = "Identified a Defined Networking API token, which could lead to unauthorized network operations and data breaches." +regex = '''(?i)[\w.-]{0,50}?(?:dnkey)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}(dnkey-[a-z0-9=_\-]{26}-[a-z0-9=_\-]{52})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["dnkey"] + +[[rules]] +id = "digitalocean-access-token" +description = "Found a DigitalOcean OAuth Access Token, risking unauthorized cloud resource access and data compromise." +regex = '''\b(doo_v1_[a-f0-9]{64})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 3 +keywords = ["doo_v1_"] + +[[rules]] +id = "digitalocean-pat" +description = "Discovered a DigitalOcean Personal Access Token, posing a threat to cloud infrastructure security and data privacy." +regex = '''\b(dop_v1_[a-f0-9]{64})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 3 +keywords = ["dop_v1_"] + +[[rules]] +id = "digitalocean-refresh-token" +description = "Uncovered a DigitalOcean OAuth Refresh Token, which could allow prolonged unauthorized access and resource manipulation." +regex = '''(?i)\b(dor_v1_[a-f0-9]{64})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["dor_v1_"] + +[[rules]] +id = "discord-api-token" +description = "Detected a Discord API key, potentially compromising communication channels and user data privacy on Discord." +regex = '''(?i)[\w.-]{0,50}?(?:discord)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-f0-9]{64})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["discord"] + +[[rules]] +id = "discord-client-id" +description = "Identified a Discord client ID, which may lead to unauthorized integrations and data exposure in Discord applications." +regex = '''(?i)[\w.-]{0,50}?(?:discord)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([0-9]{18})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 2 +keywords = ["discord"] + +[[rules]] +id = "discord-client-secret" +description = "Discovered a potential Discord client secret, risking compromised Discord bot integrations and data leaks." +regex = '''(?i)[\w.-]{0,50}?(?:discord)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9=_\-]{32})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 2 +keywords = ["discord"] + +[[rules]] +id = "doppler-api-token" +description = "Discovered a Doppler API token, posing a risk to environment and secrets management security." +regex = '''dp\.pt\.(?i)[a-z0-9]{43}''' +entropy = 2 +keywords = ["dp.pt."] + +[[rules]] +id = "droneci-access-token" +description = "Detected a Droneci Access Token, potentially compromising continuous integration and deployment workflows." +regex = '''(?i)[\w.-]{0,50}?(?:droneci)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9]{32})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["droneci"] + +[[rules]] +id = "dropbox-api-token" +description = "Identified a Dropbox API secret, which could lead to unauthorized file access and data breaches in Dropbox storage." +regex = '''(?i)[\w.-]{0,50}?(?:dropbox)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9]{15})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["dropbox"] + +[[rules]] +id = "dropbox-long-lived-api-token" +description = "Found a Dropbox long-lived API token, risking prolonged unauthorized access to cloud storage and sensitive data." +regex = '''(?i)[\w.-]{0,50}?(?:dropbox)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9]{11}(AAAAAAAAAA)[a-z0-9\-_=]{43})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["dropbox"] + +[[rules]] +id = "dropbox-short-lived-api-token" +description = "Discovered a Dropbox short-lived API token, posing a risk of temporary but potentially harmful data access and manipulation." +regex = '''(?i)[\w.-]{0,50}?(?:dropbox)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}(sl\.[a-z0-9\-=_]{135})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["dropbox"] + +[[rules]] +id = "duffel-api-token" +description = "Uncovered a Duffel API token, which may compromise travel platform integrations and sensitive customer data." +regex = '''duffel_(?:test|live)_(?i)[a-z0-9_\-=]{43}''' +entropy = 2 +keywords = ["duffel_"] + +[[rules]] +id = "dynatrace-api-token" +description = "Detected a Dynatrace API token, potentially risking application performance monitoring and data exposure." +regex = '''dt0c01\.(?i)[a-z0-9]{24}\.[a-z0-9]{64}''' +entropy = 4 +keywords = ["dt0c01."] + +[[rules]] +id = "easypost-api-token" +description = "Identified an EasyPost API token, which could lead to unauthorized postal and shipment service access and data exposure." +regex = '''\bEZAK(?i)[a-z0-9]{54}\b''' +entropy = 2 +keywords = ["ezak"] + +[[rules]] +id = "easypost-test-api-token" +description = "Detected an EasyPost test API token, risking exposure of test environments and potentially sensitive shipment data." +regex = '''\bEZTK(?i)[a-z0-9]{54}\b''' +entropy = 2 +keywords = ["eztk"] + +[[rules]] +id = "etsy-access-token" +description = "Found an Etsy Access Token, potentially compromising Etsy shop management and customer data." +regex = '''(?i)[\w.-]{0,50}?(?:(?-i:ETSY|[Ee]tsy))(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9]{24})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 3 +keywords = ["etsy"] + +[[rules]] +id = "facebook-access-token" +description = "Discovered a Facebook Access Token, posing a risk of unauthorized access to Facebook accounts and personal data exposure." +regex = '''(?i)\b(\d{15,16}(\||%)[0-9a-z\-_]{27,40})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 3 +keywords = ["facebook"] + +[[rules]] +id = "facebook-page-access-token" +description = "Discovered a Facebook Page Access Token, posing a risk of unauthorized access to Facebook accounts and personal data exposure." +regex = '''\b(EAA[MC](?i)[a-z0-9]{100,})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 4 +keywords = [ + "eaam", + "eaac", +] + +[[rules]] +id = "facebook-secret" +description = "Discovered a Facebook Application secret, posing a risk of unauthorized access to Facebook accounts and personal data exposure." +regex = '''(?i)[\w.-]{0,50}?(?:facebook)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-f0-9]{32})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 3 +keywords = ["facebook"] + +[[rules]] +id = "fastly-api-token" +description = "Uncovered a Fastly API key, which may compromise CDN and edge cloud services, leading to content delivery and security issues." +regex = '''(?i)[\w.-]{0,50}?(?:fastly)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9=_\-]{32})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["fastly"] + +[[rules]] +id = "finicity-api-token" +description = "Detected a Finicity API token, potentially risking financial data access and unauthorized financial operations." +regex = '''(?i)[\w.-]{0,50}?(?:finicity)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-f0-9]{32})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["finicity"] + +[[rules]] +id = "finicity-client-secret" +description = "Identified a Finicity Client Secret, which could lead to compromised financial service integrations and data breaches." +regex = '''(?i)[\w.-]{0,50}?(?:finicity)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9]{20})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["finicity"] + +[[rules]] +id = "finnhub-access-token" +description = "Found a Finnhub Access Token, risking unauthorized access to financial market data and analytics." +regex = '''(?i)[\w.-]{0,50}?(?:finnhub)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9]{20})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["finnhub"] + +[[rules]] +id = "flickr-access-token" +description = "Discovered a Flickr Access Token, posing a risk of unauthorized photo management and potential data leakage." +regex = '''(?i)[\w.-]{0,50}?(?:flickr)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9]{32})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["flickr"] + +[[rules]] +id = "flutterwave-encryption-key" +description = "Uncovered a Flutterwave Encryption Key, which may compromise payment processing and sensitive financial information." +regex = '''FLWSECK_TEST-(?i)[a-h0-9]{12}''' +entropy = 2 +keywords = ["flwseck_test"] + +[[rules]] +id = "flutterwave-public-key" +description = "Detected a Finicity Public Key, potentially exposing public cryptographic operations and integrations." +regex = '''FLWPUBK_TEST-(?i)[a-h0-9]{32}-X''' +entropy = 2 +keywords = ["flwpubk_test"] + +[[rules]] +id = "flutterwave-secret-key" +description = "Identified a Flutterwave Secret Key, risking unauthorized financial transactions and data breaches." +regex = '''FLWSECK_TEST-(?i)[a-h0-9]{32}-X''' +entropy = 2 +keywords = ["flwseck_test"] + +[[rules]] +id = "flyio-access-token" +description = "Uncovered a Fly.io API key" +regex = '''\b((?:fo1_[\w-]{43}|fm1[ar]_[a-zA-Z0-9+\/]{100,}={0,3}|fm2_[a-zA-Z0-9+\/]{100,}={0,3}))(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 4 +keywords = [ + "fo1_", + "fm1", + "fm2_", +] + +[[rules]] +id = "frameio-api-token" +description = "Found a Frame.io API token, potentially compromising video collaboration and project management." +regex = '''fio-u-(?i)[a-z0-9\-_=]{64}''' +keywords = ["fio-u-"] + +[[rules]] +id = "freemius-secret-key" +description = "Detected a Freemius secret key, potentially exposing sensitive information." +regex = '''(?i)["']secret_key["']\s*=>\s*["'](sk_[\S]{29})["']''' +path = '''(?i)\.php$''' +keywords = ["secret_key"] + +[[rules]] +id = "freshbooks-access-token" +description = "Discovered a Freshbooks Access Token, posing a risk to accounting software access and sensitive financial data exposure." +regex = '''(?i)[\w.-]{0,50}?(?:freshbooks)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9]{64})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["freshbooks"] + +[[rules]] +id = "gcp-api-key" +description = "Uncovered a GCP API key, which could lead to unauthorized access to Google Cloud services and data breaches." +regex = '''\b(AIza[\w-]{35})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 4 +keywords = ["aiza"] +[[rules.allowlists]] +regexes = [ + '''AIzaSyabcdefghijklmnopqrstuvwxyz1234567''', + '''AIzaSyAnLA7NfeLquW1tJFpx_eQCxoX-oo6YyIs''', + '''AIzaSyCkEhVjf3pduRDt6d1yKOMitrUEke8agEM''', + '''AIzaSyDMAScliyLx7F0NPDEJi1QmyCgHIAODrlU''', + '''AIzaSyD3asb-2pEZVqMkmL6M9N6nHZRR_znhrh0''', + '''AIzayDNSXIbFmlXbIE6mCzDLQAqITYefhixbX4A''', + '''AIzaSyAdOS2zB6NCsk1pCdZ4-P6GBdi_UUPwX7c''', + '''AIzaSyASWm6HmTMdYWpgMnjRBjxcQ9CKctWmLd4''', + '''AIzaSyANUvH9H9BsUccjsu2pCmEkOPjjaXeDQgY''', + '''AIzaSyA5_iVawFQ8ABuTZNUdcwERLJv_a_p4wtM''', + '''AIzaSyA4UrcGxgwQFTfaI3no3t7Lt1sjmdnP5sQ''', + '''AIzaSyDSb51JiIcB6OJpwwMicseKRhhrOq1cS7g''', + '''AIzaSyBF2RrAIm4a0mO64EShQfqfd2AFnzAvvuU''', + '''AIzaSyBcE-OOIbhjyR83gm4r2MFCu4MJmprNXsw''', + '''AIzaSyB8qGxt4ec15vitgn44duC5ucxaOi4FmqE''', + '''AIzaSyA8vmApnrHNFE0bApF4hoZ11srVL_n0nvY''', +] + +[[rules]] +id = "generic-api-key" +description = "Detected a Generic API Key, potentially exposing access to various services and sensitive operations." +regex = '''(?i)[\w.-]{0,50}?(?:access|auth|(?-i:[Aa]pi|API)|credential|creds|key|passw(?:or)?d|secret|token)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([\w.=-]{10,150}|[a-z0-9][a-z0-9+/]{11,}={0,3})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 3.5 +keywords = [ + "access", + "api", + "auth", + "key", + "credential", + "creds", + "passwd", + "password", + "secret", + "token", +] +[[rules.allowlists]] +regexes = [ + '''^[a-zA-Z_.-]+$''', +] +[[rules.allowlists]] +description = "Allowlist for Generic API Keys" +regexTarget = "match" +regexes = [ + '''(?i)(?:access(?:ibility|or)|access[_.-]?id|random[_.-]?access|api[_.-]?(?:id|name|version)|rapid|capital|[a-z0-9-]*?api[a-z0-9-]*?:jar:|author|X-MS-Exchange-Organization-Auth|Authentication-Results|(?:credentials?[_.-]?id|withCredentials)|(?:bucket|foreign|hot|idx|natural|primary|pub(?:lic)?|schema|sequence)[_.-]?key|(?:turkey)|key[_.-]?(?:alias|board|code|frame|id|length|mesh|name|pair|press(?:ed)?|ring|selector|signature|size|stone|storetype|word|up|down|left|right)|key[_.-]?vault[_.-]?(?:id|name)|keyVaultToStoreSecrets|key(?:store|tab)[_.-]?(?:file|path)|issuerkeyhash|(?-i:[DdMm]onkey|[DM]ONKEY)|keying|(?:secret)[_.-]?(?:length|name|size)|UserSecretsId|(?:csrf)[_.-]?token|(?:io\.jsonwebtoken[ \t]?:[ \t]?[\w-]+)|(?:api|credentials|token)[_.-]?(?:endpoint|ur[il])|public[_.-]?token|(?:key|token)[_.-]?file|(?-i:(?:[A-Z_]+=\n[A-Z_]+=|[a-z_]+=\n[a-z_]+=)(?:\n|\z))|(?-i:(?:[A-Z.]+=\n[A-Z.]+=|[a-z.]+=\n[a-z.]+=)(?:\n|\z)))''', +] +stopwords = [ + "000000", + "6fe4476ee5a1832882e326b506d14126", + "_ec2_", + "aaaaaa", + "about", + "abstract", + "academy", + "acces", + "account", + "act-", + "act.", + "act_", + "action", + "active", + "actively", + "activity", + "adapter", + "add-", + "add-on", + "add.", + "add_", + "addon", + "addres", + "admin", + "adobe", + "advanced", + "adventure", + "agent", + "agile", + "air-", + "air.", + "air_", + "ajax", + "akka", + "alert", + "alfred", + "algorithm", + "all-", + "all.", + "all_", + "alloy", + "alpha", + "amazon", + "amqp", + "analysi", + "analytic", + "analyzer", + "android", + "angular", + "angularj", + "animate", + "animation", + "another", + "ansible", + "answer", + "ant-", + "ant.", + "ant_", + "any-", + "any.", + "any_", + "apache", + "app-", + "app.", + "app_", + "apple", + "arch", + "archive", + "archived", + "arduino", + "array", + "art-", + "art.", + "art_", + "article", + "asp-", + "asp.", + "asp_", + "asset", + "async", + "atom", + "attention", + "audio", + "audit", + "aura", + "auth", + "author", + "authorize", + "auto", + "automated", + "automatic", + "awesome", + "aws_", + "azure", + "back", + "backbone", + "backend", + "backup", + "bar-", + "bar.", + "bar_", + "base", + "based", + "bash", + "basic", + "batch", + "been", + "beer", + "behavior", + "being", + "benchmark", + "best", + "beta", + "better", + "big-", + "big.", + "big_", + "binary", + "binding", + "bit-", + "bit.", + "bit_", + "bitcoin", + "block", + "blog", + "board", + "book", + "bookmark", + "boost", + "boot", + "bootstrap", + "bosh", + "bot-", + "bot.", + "bot_", + "bower", + "box-", + "box.", + "box_", + "boxen", + "bracket", + "branch", + "bridge", + "browser", + "brunch", + "buffer", + "bug-", + "bug.", + "bug_", + "build", + "builder", + "building", + "buildout", + "buildpack", + "built", + "bundle", + "busines", + "but-", + "but.", + "but_", + "button", + "cache", + "caching", + "cakephp", + "calendar", + "call", + "camera", + "campfire", + "can-", + "can.", + "can_", + "canva", + "captcha", + "capture", + "card", + "carousel", + "case", + "cassandra", + "cat-", + "cat.", + "cat_", + "category", + "center", + "cento", + "challenge", + "change", + "changelog", + "channel", + "chart", + "chat", + "cheat", + "check", + "checker", + "chef", + "ches", + "chinese", + "chosen", + "chrome", + "ckeditor", + "clas", + "classe", + "classic", + "clean", + "cli-", + "cli.", + "cli_", + "client", + "clojure", + "clone", + "closure", + "cloud", + "club", + "cluster", + "cms-", + "cms_", + "coco", + "code", + "coding", + "coffee", + "color", + "combination", + "combo", + "command", + "commander", + "comment", + "commit", + "common", + "community", + "compas", + "compiler", + "complete", + "component", + "composer", + "computer", + "computing", + "con-", + "con.", + "con_", + "concept", + "conf", + "config", + "connect", + "connector", + "console", + "contact", + "container", + "contao", + "content", + "contest", + "context", + "control", + "convert", + "converter", + "conway'", + "cookbook", + "cookie", + "cool", + "copy", + "cordova", + "core", + "couchbase", + "couchdb", + "countdown", + "counter", + "course", + "craft", + "crawler", + "create", + "creating", + "creator", + "credential", + "crm-", + "crm.", + "crm_", + "cros", + "crud", + "csv-", + "csv.", + "csv_", + "cube", + "cucumber", + "cuda", + "current", + "currently", + "custom", + "daemon", + "dark", + "dart", + "dash", + "dashboard", + "data", + "database", + "date", + "day-", + "day.", + "day_", + "dead", + "debian", + "debug", + "debugger", + "deck", + "define", + "del-", + "del.", + "del_", + "delete", + "demo", + "deploy", + "design", + "designer", + "desktop", + "detection", + "detector", + "dev-", + "dev.", + "dev_", + "develop", + "developer", + "device", + "devise", + "diff", + "digital", + "directive", + "directory", + "discovery", + "display", + "django", + "dns-", + "dns_", + "doc-", + "doc.", + "doc_", + "docker", + "docpad", + "doctrine", + "document", + "doe-", + "doe.", + "doe_", + "dojo", + "dom-", + "dom.", + "dom_", + "domain", + "don't", + "done", + "dot-", + "dot.", + "dot_", + "dotfile", + "download", + "draft", + "drag", + "drill", + "drive", + "driven", + "driver", + "drop", + "dropbox", + "drupal", + "dsl-", + "dsl.", + "dsl_", + "dynamic", + "easy", + "ecdsa", + "eclipse", + "edit", + "editing", + "edition", + "editor", + "element", + "emac", + "email", + "embed", + "embedded", + "ember", + "emitter", + "emulator", + "encoding", + "endpoint", + "engine", + "english", + "enhanced", + "entity", + "entry", + "env_", + "episode", + "erlang", + "error", + "espresso", + "event", + "evented", + "example", + "exchange", + "exercise", + "experiment", + "expire", + "exploit", + "explorer", + "export", + "exporter", + "expres", + "ext-", + "ext.", + "ext_", + "extended", + "extension", + "external", + "extra", + "extractor", + "fabric", + "facebook", + "factory", + "fake", + "fast", + "feature", + "feed", + "fewfwef", + "ffmpeg", + "field", + "file", + "filter", + "find", + "finder", + "firefox", + "firmware", + "first", + "fish", + "fix-", + "fix_", + "flash", + "flask", + "flat", + "flex", + "flexible", + "flickr", + "flow", + "fluent", + "fluentd", + "fluid", + "folder", + "font", + "force", + "foreman", + "fork", + "form", + "format", + "formatter", + "forum", + "foundry", + "framework", + "free", + "friend", + "friendly", + "front-end", + "frontend", + "ftp-", + "ftp.", + "ftp_", + "fuel", + "full", + "fun-", + "fun.", + "fun_", + "func", + "future", + "gaia", + "gallery", + "game", + "gateway", + "gem-", + "gem.", + "gem_", + "gen-", + "gen.", + "gen_", + "general", + "generator", + "generic", + "genetic", + "get-", + "get.", + "get_", + "getenv", + "getting", + "ghost", + "gist", + "git-", + "git.", + "git_", + "github", + "gitignore", + "gitlab", + "glas", + "gmail", + "gnome", + "gnu-", + "gnu.", + "gnu_", + "goal", + "golang", + "gollum", + "good", + "google", + "gpu-", + "gpu.", + "gpu_", + "gradle", + "grail", + "graph", + "graphic", + "great", + "grid", + "groovy", + "group", + "grunt", + "guard", + "gui-", + "gui.", + "gui_", + "guide", + "guideline", + "gulp", + "gwt-", + "gwt.", + "gwt_", + "hack", + "hackathon", + "hacker", + "hacking", + "hadoop", + "haml", + "handler", + "hardware", + "has-", + "has_", + "hash", + "haskell", + "have", + "haxe", + "hello", + "help", + "helper", + "here", + "hero", + "heroku", + "high", + "hipchat", + "history", + "home", + "homebrew", + "homepage", + "hook", + "host", + "hosting", + "hot-", + "hot.", + "hot_", + "house", + "how-", + "how.", + "how_", + "html", + "http", + "hub-", + "hub.", + "hub_", + "hubot", + "human", + "icon", + "ide-", + "ide.", + "ide_", + "idea", + "identity", + "idiomatic", + "image", + "impact", + "import", + "important", + "importer", + "impres", + "index", + "infinite", + "info", + "injection", + "inline", + "input", + "inside", + "inspector", + "instagram", + "install", + "installer", + "instant", + "intellij", + "interface", + "internet", + "interview", + "into", + "intro", + "ionic", + "iphone", + "ipython", + "irc-", + "irc_", + "iso-", + "iso.", + "iso_", + "issue", + "jade", + "jasmine", + "java", + "jbos", + "jekyll", + "jenkin", + "jetbrains", + "job-", + "job.", + "job_", + "joomla", + "jpa-", + "jpa.", + "jpa_", + "jquery", + "json", + "just", + "kafka", + "karma", + "kata", + "kernel", + "keyboard", + "kindle", + "kit-", + "kit.", + "kit_", + "kitchen", + "knife", + "koan", + "kohana", + "lab-", + "lab.", + "lab_", + "lambda", + "lamp", + "language", + "laravel", + "last", + "latest", + "latex", + "launcher", + "layer", + "layout", + "lazy", + "ldap", + "leaflet", + "league", + "learn", + "learning", + "led-", + "led.", + "led_", + "leetcode", + "les-", + "les.", + "les_", + "level", + "leveldb", + "lib-", + "lib.", + "lib_", + "librarie", + "library", + "license", + "life", + "liferay", + "light", + "lightbox", + "like", + "line", + "link", + "linked", + "linkedin", + "linux", + "lisp", + "list", + "lite", + "little", + "load", + "loader", + "local", + "location", + "lock", + "log-", + "log.", + "log_", + "logger", + "logging", + "logic", + "login", + "logstash", + "longer", + "look", + "love", + "lua-", + "lua.", + "lua_", + "mac-", + "mac.", + "mac_", + "machine", + "made", + "magento", + "magic", + "mail", + "make", + "maker", + "making", + "man-", + "man.", + "man_", + "manage", + "manager", + "manifest", + "manual", + "map-", + "map.", + "map_", + "mapper", + "mapping", + "markdown", + "markup", + "master", + "math", + "matrix", + "maven", + "md5", + "mean", + "media", + "mediawiki", + "meetup", + "memcached", + "memory", + "menu", + "merchant", + "message", + "messaging", + "meta", + "metadata", + "meteor", + "method", + "metric", + "micro", + "middleman", + "migration", + "minecraft", + "miner", + "mini", + "minimal", + "mirror", + "mit-", + "mit.", + "mit_", + "mobile", + "mocha", + "mock", + "mod-", + "mod.", + "mod_", + "mode", + "model", + "modern", + "modular", + "module", + "modx", + "money", + "mongo", + "mongodb", + "mongoid", + "mongoose", + "monitor", + "monkey", + "more", + "motion", + "moved", + "movie", + "mozilla", + "mqtt", + "mule", + "multi", + "multiple", + "music", + "mustache", + "mvc-", + "mvc.", + "mvc_", + "mysql", + "nagio", + "name", + "native", + "need", + "neo-", + "neo.", + "neo_", + "nest", + "nested", + "net-", + "net.", + "net_", + "nette", + "network", + "new-", + "new.", + "new_", + "next", + "nginx", + "ninja", + "nlp-", + "nlp.", + "nlp_", + "node", + "nodej", + "nosql", + "not-", + "not.", + "not_", + "note", + "notebook", + "notepad", + "notice", + "notifier", + "now-", + "now.", + "now_", + "number", + "oauth", + "object", + "objective", + "obsolete", + "ocaml", + "octopres", + "official", + "old-", + "old.", + "old_", + "onboard", + "online", + "only", + "open", + "opencv", + "opengl", + "openshift", + "openwrt", + "option", + "oracle", + "org-", + "org.", + "org_", + "origin", + "original", + "orm-", + "orm.", + "orm_", + "osx-", + "osx_", + "our-", + "our.", + "our_", + "out-", + "out.", + "out_", + "output", + "over", + "overview", + "own-", + "own.", + "own_", + "pack", + "package", + "packet", + "page", + "panel", + "paper", + "paperclip", + "para", + "parallax", + "parallel", + "parse", + "parser", + "parsing", + "particle", + "party", + "password", + "patch", + "path", + "pattern", + "payment", + "paypal", + "pdf-", + "pdf.", + "pdf_", + "pebble", + "people", + "perl", + "personal", + "phalcon", + "phoenix", + "phone", + "phonegap", + "photo", + "php-", + "php.", + "php_", + "physic", + "picker", + "pipeline", + "platform", + "play", + "player", + "please", + "plu-", + "plu.", + "plu_", + "plug-in", + "plugin", + "plupload", + "png-", + "png.", + "png_", + "poker", + "polyfill", + "polymer", + "pool", + "pop-", + "pop.", + "pop_", + "popcorn", + "popup", + "port", + "portable", + "portal", + "portfolio", + "post", + "power", + "powered", + "powerful", + "prelude", + "pretty", + "preview", + "principle", + "print", + "pro-", + "pro.", + "pro_", + "problem", + "proc", + "product", + "profile", + "profiler", + "program", + "progres", + "project", + "protocol", + "prototype", + "provider", + "proxy", + "public", + "pull", + "puppet", + "pure", + "purpose", + "push", + "pusher", + "pyramid", + "python", + "quality", + "query", + "queue", + "quick", + "rabbitmq", + "rack", + "radio", + "rail", + "railscast", + "random", + "range", + "raspberry", + "rdf-", + "rdf.", + "rdf_", + "react", + "reactive", + "read", + "reader", + "readme", + "ready", + "real", + "real-time", + "reality", + "realtime", + "recipe", + "recorder", + "red-", + "red.", + "red_", + "reddit", + "redi", + "redmine", + "reference", + "refinery", + "refresh", + "registry", + "related", + "release", + "remote", + "rendering", + "repo", + "report", + "request", + "require", + "required", + "requirej", + "research", + "resource", + "response", + "resque", + "rest", + "restful", + "resume", + "reveal", + "reverse", + "review", + "riak", + "rich", + "right", + "ring", + "robot", + "role", + "room", + "router", + "routing", + "rpc-", + "rpc.", + "rpc_", + "rpg-", + "rpg.", + "rpg_", + "rspec", + "ruby-", + "ruby.", + "ruby_", + "rule", + "run-", + "run.", + "run_", + "runner", + "running", + "runtime", + "rust", + "rvm-", + "rvm.", + "rvm_", + "salt", + "sample", + "sandbox", + "sas-", + "sas.", + "sas_", + "sbt-", + "sbt.", + "sbt_", + "scala", + "scalable", + "scanner", + "schema", + "scheme", + "school", + "science", + "scraper", + "scratch", + "screen", + "script", + "scroll", + "scs-", + "scs.", + "scs_", + "sdk-", + "sdk.", + "sdk_", + "sdl-", + "sdl.", + "sdl_", + "search", + "secure", + "security", + "see-", + "see.", + "see_", + "seed", + "select", + "selector", + "selenium", + "semantic", + "sencha", + "send", + "sentiment", + "serie", + "server", + "service", + "session", + "set-", + "set.", + "set_", + "setting", + "setup", + "sha1", + "sha2", + "sha256", + "share", + "shared", + "sharing", + "sheet", + "shell", + "shield", + "shipping", + "shop", + "shopify", + "shortener", + "should", + "show", + "showcase", + "side", + "silex", + "simple", + "simulator", + "single", + "site", + "skeleton", + "sketch", + "skin", + "slack", + "slide", + "slider", + "slim", + "small", + "smart", + "smtp", + "snake", + "snapshot", + "snippet", + "soap", + "social", + "socket", + "software", + "solarized", + "solr", + "solution", + "solver", + "some", + "soon", + "source", + "space", + "spark", + "spatial", + "spec", + "sphinx", + "spine", + "spotify", + "spree", + "spring", + "sprite", + "sql-", + "sql.", + "sql_", + "sqlite", + "ssh-", + "ssh.", + "ssh_", + "stack", + "staging", + "standard", + "stanford", + "start", + "started", + "starter", + "startup", + "stat", + "statamic", + "state", + "static", + "statistic", + "statsd", + "statu", + "steam", + "step", + "still", + "stm-", + "stm.", + "stm_", + "storage", + "store", + "storm", + "story", + "strategy", + "stream", + "streaming", + "string", + "stripe", + "structure", + "studio", + "study", + "stuff", + "style", + "sublime", + "sugar", + "suite", + "summary", + "super", + "support", + "supported", + "svg-", + "svg.", + "svg_", + "svn-", + "svn.", + "svn_", + "swagger", + "swift", + "switch", + "switcher", + "symfony", + "symphony", + "sync", + "synopsi", + "syntax", + "system", + "tab-", + "tab.", + "tab_", + "table", + "tag-", + "tag.", + "tag_", + "talk", + "target", + "task", + "tcp-", + "tcp.", + "tcp_", + "tdd-", + "tdd.", + "tdd_", + "team", + "tech", + "template", + "term", + "terminal", + "testing", + "tetri", + "text", + "textmate", + "theme", + "theory", + "three", + "thrift", + "time", + "timeline", + "timer", + "tiny", + "tinymce", + "tip-", + "tip.", + "tip_", + "title", + "todo", + "todomvc", + "token", + "tool", + "toolbox", + "toolkit", + "top-", + "top.", + "top_", + "tornado", + "touch", + "tower", + "tracker", + "tracking", + "traffic", + "training", + "transfer", + "translate", + "transport", + "tree", + "trello", + "try-", + "try.", + "try_", + "tumblr", + "tut-", + "tut.", + "tut_", + "tutorial", + "tweet", + "twig", + "twitter", + "type", + "typo", + "ubuntu", + "uiview", + "ultimate", + "under", + "unit", + "unity", + "universal", + "unix", + "update", + "updated", + "upgrade", + "upload", + "uploader", + "uri-", + "uri.", + "uri_", + "url-", + "url.", + "url_", + "usage", + "usb-", + "usb.", + "usb_", + "use-", + "use.", + "use_", + "used", + "useful", + "user", + "using", + "util", + "utilitie", + "utility", + "vagrant", + "validator", + "value", + "variou", + "varnish", + "version", + "via-", + "via.", + "via_", + "video", + "view", + "viewer", + "vim-", + "vim.", + "vim_", + "vimrc", + "virtual", + "vision", + "visual", + "vpn", + "want", + "warning", + "watch", + "watcher", + "wave", + "way-", + "way.", + "way_", + "weather", + "web-", + "web_", + "webapp", + "webgl", + "webhook", + "webkit", + "webrtc", + "website", + "websocket", + "welcome", + "what", + "what'", + "when", + "where", + "which", + "why-", + "why.", + "why_", + "widget", + "wifi", + "wiki", + "win-", + "win.", + "win_", + "window", + "wip-", + "wip.", + "wip_", + "within", + "without", + "wizard", + "word", + "wordpres", + "work", + "worker", + "workflow", + "working", + "workshop", + "world", + "wrapper", + "write", + "writer", + "writing", + "written", + "www-", + "www.", + "www_", + "xamarin", + "xcode", + "xml-", + "xml.", + "xml_", + "xmpp", + "xxxxxx", + "yahoo", + "yaml", + "yandex", + "yeoman", + "yet-", + "yet.", + "yet_", + "yii-", + "yii.", + "yii_", + "youtube", + "yui-", + "yui.", + "yui_", + "zend", + "zero", + "zip-", + "zip.", + "zip_", + "zsh-", + "zsh.", + "zsh_", +] +[[rules.allowlists]] +regexTarget = "line" +regexes = [ + '''--mount=type=secret,''', + '''import[ \t]+{[ \t\w,]+}[ \t]+from[ \t]+['"][^'"]+['"]''', +] +[[rules.allowlists]] +condition = "AND" +paths = [ + '''\.bb$''','''\.bbappend$''','''\.bbclass$''','''\.inc$''', +] +regexTarget = "line" +regexes = [ + '''LICENSE[^=]*=\s*"[^"]+''', + '''LIC_FILES_CHKSUM[^=]*=\s*"[^"]+''', + '''SRC[^=]*=\s*"[a-zA-Z0-9]+''', +] + +[[rules]] +id = "github-app-token" +description = "Identified a GitHub App Token, which may compromise GitHub application integrations and source code security." +regex = '''(?:ghu|ghs)_[0-9a-zA-Z]{36}''' +entropy = 3 +keywords = [ + "ghu_", + "ghs_", +] +[[rules.allowlists]] +paths = [ + '''(?:^|/)@octokit/auth-token/README\.md$''', +] + +[[rules]] +id = "github-fine-grained-pat" +description = "Found a GitHub Fine-Grained Personal Access Token, risking unauthorized repository access and code manipulation." +regex = '''github_pat_\w{82}''' +entropy = 3 +keywords = ["github_pat_"] + +[[rules]] +id = "github-oauth" +description = "Discovered a GitHub OAuth Access Token, posing a risk of compromised GitHub account integrations and data leaks." +regex = '''gho_[0-9a-zA-Z]{36}''' +entropy = 3 +keywords = ["gho_"] + +[[rules]] +id = "github-pat" +description = "Uncovered a GitHub Personal Access Token, potentially leading to unauthorized repository access and sensitive content exposure." +regex = '''ghp_[0-9a-zA-Z]{36}''' +entropy = 3 +keywords = ["ghp_"] +[[rules.allowlists]] +paths = [ + '''(?:^|/)@octokit/auth-token/README\.md$''', +] + +[[rules]] +id = "github-refresh-token" +description = "Detected a GitHub Refresh Token, which could allow prolonged unauthorized access to GitHub services." +regex = '''ghr_[0-9a-zA-Z]{36}''' +entropy = 3 +keywords = ["ghr_"] + +[[rules]] +id = "gitlab-cicd-job-token" +description = "Identified a GitLab CI/CD Job Token, potential access to projects and some APIs on behalf of a user while the CI job is running." +regex = '''glcbt-[0-9a-zA-Z]{1,5}_[0-9a-zA-Z_-]{20}''' +entropy = 3 +keywords = ["glcbt-"] + +[[rules]] +id = "gitlab-deploy-token" +description = "Identified a GitLab Deploy Token, risking access to repositories, packages and containers with write access." +regex = '''gldt-[0-9a-zA-Z_\-]{20}''' +entropy = 3 +keywords = ["gldt-"] + +[[rules]] +id = "gitlab-feature-flag-client-token" +description = "Identified a GitLab feature flag client token, risks exposing user lists and features flags used by an application." +regex = '''glffct-[0-9a-zA-Z_\-]{20}''' +entropy = 3 +keywords = ["glffct-"] + +[[rules]] +id = "gitlab-feed-token" +description = "Identified a GitLab feed token, risking exposure of user data." +regex = '''glft-[0-9a-zA-Z_\-]{20}''' +entropy = 3 +keywords = ["glft-"] + +[[rules]] +id = "gitlab-incoming-mail-token" +description = "Identified a GitLab incoming mail token, risking manipulation of data sent by mail." +regex = '''glimt-[0-9a-zA-Z_\-]{25}''' +entropy = 3 +keywords = ["glimt-"] + +[[rules]] +id = "gitlab-kubernetes-agent-token" +description = "Identified a GitLab Kubernetes Agent token, risking access to repos and registry of projects connected via agent." +regex = '''glagent-[0-9a-zA-Z_\-]{50}''' +entropy = 3 +keywords = ["glagent-"] + +[[rules]] +id = "gitlab-oauth-app-secret" +description = "Identified a GitLab OIDC Application Secret, risking access to apps using GitLab as authentication provider." +regex = '''gloas-[0-9a-zA-Z_\-]{64}''' +entropy = 3 +keywords = ["gloas-"] + +[[rules]] +id = "gitlab-pat" +description = "Identified a GitLab Personal Access Token, risking unauthorized access to GitLab repositories and codebase exposure." +regex = '''glpat-[\w-]{20}''' +entropy = 3 +keywords = ["glpat-"] + +[[rules]] +id = "gitlab-pat-routable" +description = "Identified a GitLab Personal Access Token (routable), risking unauthorized access to GitLab repositories and codebase exposure." +regex = '''\bglpat-[0-9a-zA-Z_-]{27,300}\.[0-9a-z]{2}[0-9a-z]{7}\b''' +entropy = 4 +keywords = ["glpat-"] + +[[rules]] +id = "gitlab-ptt" +description = "Found a GitLab Pipeline Trigger Token, potentially compromising continuous integration workflows and project security." +regex = '''glptt-[0-9a-f]{40}''' +entropy = 3 +keywords = ["glptt-"] + +[[rules]] +id = "gitlab-rrt" +description = "Discovered a GitLab Runner Registration Token, posing a risk to CI/CD pipeline integrity and unauthorized access." +regex = '''GR1348941[\w-]{20}''' +entropy = 3 +keywords = ["gr1348941"] + +[[rules]] +id = "gitlab-runner-authentication-token" +description = "Discovered a GitLab Runner Authentication Token, posing a risk to CI/CD pipeline integrity and unauthorized access." +regex = '''glrt-[0-9a-zA-Z_\-]{20}''' +entropy = 3 +keywords = ["glrt-"] + +[[rules]] +id = "gitlab-runner-authentication-token-routable" +description = "Discovered a GitLab Runner Authentication Token (Routable), posing a risk to CI/CD pipeline integrity and unauthorized access." +regex = '''\bglrt-t\d_[0-9a-zA-Z_\-]{27,300}\.[0-9a-z]{2}[0-9a-z]{7}\b''' +entropy = 4 +keywords = ["glrt-"] + +[[rules]] +id = "gitlab-scim-token" +description = "Discovered a GitLab SCIM Token, posing a risk to unauthorized access for a organization or instance." +regex = '''glsoat-[0-9a-zA-Z_\-]{20}''' +entropy = 3 +keywords = ["glsoat-"] + +[[rules]] +id = "gitlab-session-cookie" +description = "Discovered a GitLab Session Cookie, posing a risk to unauthorized access to a user account." +regex = '''_gitlab_session=[0-9a-z]{32}''' +entropy = 3 +keywords = ["_gitlab_session="] + +[[rules]] +id = "gitter-access-token" +description = "Uncovered a Gitter Access Token, which may lead to unauthorized access to chat and communication services." +regex = '''(?i)[\w.-]{0,50}?(?:gitter)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9_-]{40})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["gitter"] + +[[rules]] +id = "gocardless-api-token" +description = "Detected a GoCardless API token, potentially risking unauthorized direct debit payment operations and financial data exposure." +regex = '''(?i)[\w.-]{0,50}?(?:gocardless)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}(live_(?i)[a-z0-9\-_=]{40})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = [ + "live_", + "gocardless", +] + +[[rules]] +id = "grafana-api-key" +description = "Identified a Grafana API key, which could compromise monitoring dashboards and sensitive data analytics." +regex = '''(?i)\b(eyJrIjoi[A-Za-z0-9]{70,400}={0,3})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 3 +keywords = ["eyjrijoi"] + +[[rules]] +id = "grafana-cloud-api-token" +description = "Found a Grafana cloud API token, risking unauthorized access to cloud-based monitoring services and data exposure." +regex = '''(?i)\b(glc_[A-Za-z0-9+/]{32,400}={0,3})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 3 +keywords = ["glc_"] + +[[rules]] +id = "grafana-service-account-token" +description = "Discovered a Grafana service account token, posing a risk of compromised monitoring services and data integrity." +regex = '''(?i)\b(glsa_[A-Za-z0-9]{32}_[A-Fa-f0-9]{8})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 3 +keywords = ["glsa_"] + +[[rules]] +id = "harness-api-key" +description = "Identified a Harness Access Token (PAT or SAT), risking unauthorized access to a Harness account." +regex = '''(?:pat|sat)\.[a-zA-Z0-9_-]{22}\.[a-zA-Z0-9]{24}\.[a-zA-Z0-9]{20}''' +keywords = [ + "pat.", + "sat.", +] + +[[rules]] +id = "hashicorp-tf-api-token" +description = "Uncovered a HashiCorp Terraform user/org API token, which may lead to unauthorized infrastructure management and security breaches." +regex = '''(?i)[a-z0-9]{14}\.(?-i:atlasv1)\.[a-z0-9\-_=]{60,70}''' +entropy = 3.5 +keywords = ["atlasv1"] + +[[rules]] +id = "hashicorp-tf-password" +description = "Identified a HashiCorp Terraform password field, risking unauthorized infrastructure configuration and security breaches." +regex = '''(?i)[\w.-]{0,50}?(?:administrator_login_password|password)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}("[a-z0-9=_\-]{8,20}")(?:[\x60'"\s;]|\\[nr]|$)''' +path = '''(?i)\.(?:tf|hcl)$''' +entropy = 2 +keywords = [ + "administrator_login_password", + "password", +] + +[[rules]] +id = "heroku-api-key" +description = "Detected a Heroku API Key, potentially compromising cloud application deployments and operational security." +regex = '''(?i)[\w.-]{0,50}?(?:heroku)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["heroku"] + +[[rules]] +id = "heroku-api-key-v2" +description = "Detected a Heroku API Key, potentially compromising cloud application deployments and operational security." +regex = '''\b((HRKU-AA[0-9a-zA-Z_-]{58}))(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 4 +keywords = ["hrku-aa"] + +[[rules]] +id = "hubspot-api-key" +description = "Found a HubSpot API Token, posing a risk to CRM data integrity and unauthorized marketing operations." +regex = '''(?i)[\w.-]{0,50}?(?:hubspot)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([0-9A-F]{8}-[0-9A-F]{4}-[0-9A-F]{4}-[0-9A-F]{4}-[0-9A-F]{12})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["hubspot"] + +[[rules]] +id = "huggingface-access-token" +description = "Discovered a Hugging Face Access token, which could lead to unauthorized access to AI models and sensitive data." +regex = '''\b(hf_(?i:[a-z]{34}))(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 2 +keywords = ["hf_"] + +[[rules]] +id = "huggingface-organization-api-token" +description = "Uncovered a Hugging Face Organization API token, potentially compromising AI organization accounts and associated data." +regex = '''\b(api_org_(?i:[a-z]{34}))(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 2 +keywords = ["api_org_"] + +[[rules]] +id = "infracost-api-token" +description = "Detected an Infracost API Token, risking unauthorized access to cloud cost estimation tools and financial data." +regex = '''\b(ico-[a-zA-Z0-9]{32})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 3 +keywords = ["ico-"] + +[[rules]] +id = "intercom-api-key" +description = "Identified an Intercom API Token, which could compromise customer communication channels and data privacy." +regex = '''(?i)[\w.-]{0,50}?(?:intercom)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9=_\-]{60})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["intercom"] + +[[rules]] +id = "intra42-client-secret" +description = "Found a Intra42 client secret, which could lead to unauthorized access to the 42School API and sensitive data." +regex = '''\b(s-s4t2(?:ud|af)-(?i)[abcdef0123456789]{64})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 3 +keywords = [ + "intra", + "s-s4t2ud-", + "s-s4t2af-", +] + +[[rules]] +id = "jfrog-api-key" +description = "Found a JFrog API Key, posing a risk of unauthorized access to software artifact repositories and build pipelines." +regex = '''(?i)[\w.-]{0,50}?(?:jfrog|artifactory|bintray|xray)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9]{73})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = [ + "jfrog", + "artifactory", + "bintray", + "xray", +] + +[[rules]] +id = "jfrog-identity-token" +description = "Discovered a JFrog Identity Token, potentially compromising access to JFrog services and sensitive software artifacts." +regex = '''(?i)[\w.-]{0,50}?(?:jfrog|artifactory|bintray|xray)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9]{64})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = [ + "jfrog", + "artifactory", + "bintray", + "xray", +] + +[[rules]] +id = "jwt" +description = "Uncovered a JSON Web Token, which may lead to unauthorized access to web applications and sensitive user data." +regex = '''\b(ey[a-zA-Z0-9]{17,}\.ey[a-zA-Z0-9\/\\_-]{17,}\.(?:[a-zA-Z0-9\/\\_-]{10,}={0,2})?)(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 3 +keywords = ["ey"] + +[[rules]] +id = "jwt-base64" +description = "Detected a Base64-encoded JSON Web Token, posing a risk of exposing encoded authentication and data exchange information." +regex = '''\bZXlK(?:(?PaGJHY2lPaU)|(?PaGNIVWlPaU)|(?PaGNIWWlPaU)|(?PaGRXUWlPaU)|(?PaU5qUWlP)|(?PamNtbDBJanBi)|(?PamRIa2lPaU)|(?PbGNHc2lPbn)|(?PbGJtTWlPaU)|(?PcWEzVWlPaU)|(?PcWQyc2lPb)|(?PcGMzTWlPaU)|(?PcGRpSTZJ)|(?PcmFXUWlP)|(?PclpYbGZiM0J6SWpwY)|(?PcmRIa2lPaUp)|(?PdWIyNWpaU0k2)|(?Pd01tTWlP)|(?Pd01uTWlPaU)|(?Pd2NIUWlPaU)|(?PemRXSWlPaU)|(?PemRuUWlP)|(?PMFlXY2lPaU)|(?PMGVYQWlPaUp)|(?PMWNtd2l)|(?PMWMyVWlPaUp)|(?PMlpYSWlPaU)|(?PMlpYSnphVzl1SWpv)|(?PNElqb2)|(?PNE5XTWlP)|(?PNE5YUWlPaU)|(?PNE5YUWpVekkxTmlJNkl)|(?PNE5YVWlPaU)|(?PNmFYQWlPaU))[a-zA-Z0-9\/\\_+\-\r\n]{40,}={0,2}''' +entropy = 2 +keywords = ["zxlk"] + +[[rules]] +id = "kraken-access-token" +description = "Identified a Kraken Access Token, potentially compromising cryptocurrency trading accounts and financial security." +regex = '''(?i)[\w.-]{0,50}?(?:kraken)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9\/=_\+\-]{80,90})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["kraken"] + +[[rules]] +id = "kubernetes-secret-yaml" +description = "Possible Kubernetes Secret detected, posing a risk of leaking credentials/tokens from your deployments" +regex = '''(?i)(?:\bkind:[ \t]*["']?\bsecret\b["']?(?s:.){0,200}?\bdata:(?s:.){0,100}?\s+([\w.-]+:(?:[ \t]*(?:\||>[-+]?)\s+)?[ \t]*(?:["']?[a-z0-9+/]{10,}={0,3}["']?|\{\{[ \t\w"|$:=,.-]+}}|""|''))|\bdata:(?s:.){0,100}?\s+([\w.-]+:(?:[ \t]*(?:\||>[-+]?)\s+)?[ \t]*(?:["']?[a-z0-9+/]{10,}={0,3}["']?|\{\{[ \t\w"|$:=,.-]+}}|""|''))(?s:.){0,200}?\bkind:[ \t]*["']?\bsecret\b["']?)''' +path = '''(?i)\.ya?ml$''' +keywords = ["secret"] +[[rules.allowlists]] +regexes = [ + '''[\w.-]+:(?:[ \t]*(?:\||>[-+]?)\s+)?[ \t]*(?:\{\{[ \t\w"|$:=,.-]+}}|""|'')''', +] +[[rules.allowlists]] +regexTarget = "match" +regexes = [ + '''(kind:(?s:.)+\n---\n(?s:.)+\bdata:|data:(?s:.)+\n---\n(?s:.)+\bkind:)''', +] + +[[rules]] +id = "kucoin-access-token" +description = "Found a Kucoin Access Token, risking unauthorized access to cryptocurrency exchange services and transactions." +regex = '''(?i)[\w.-]{0,50}?(?:kucoin)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-f0-9]{24})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["kucoin"] + +[[rules]] +id = "kucoin-secret-key" +description = "Discovered a Kucoin Secret Key, which could lead to compromised cryptocurrency operations and financial data breaches." +regex = '''(?i)[\w.-]{0,50}?(?:kucoin)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["kucoin"] + +[[rules]] +id = "launchdarkly-access-token" +description = "Uncovered a Launchdarkly Access Token, potentially compromising feature flag management and application functionality." +regex = '''(?i)[\w.-]{0,50}?(?:launchdarkly)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9=_\-]{40})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["launchdarkly"] + +[[rules]] +id = "linear-api-key" +description = "Detected a Linear API Token, posing a risk to project management tools and sensitive task data." +regex = '''lin_api_(?i)[a-z0-9]{40}''' +entropy = 2 +keywords = ["lin_api_"] + +[[rules]] +id = "linear-client-secret" +description = "Identified a Linear Client Secret, which may compromise secure integrations and sensitive project management data." +regex = '''(?i)[\w.-]{0,50}?(?:linear)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-f0-9]{32})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 2 +keywords = ["linear"] + +[[rules]] +id = "linkedin-client-id" +description = "Found a LinkedIn Client ID, risking unauthorized access to LinkedIn integrations and professional data exposure." +regex = '''(?i)[\w.-]{0,50}?(?:linked[_-]?in)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9]{14})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 2 +keywords = [ + "linkedin", + "linked_in", + "linked-in", +] + +[[rules]] +id = "linkedin-client-secret" +description = "Discovered a LinkedIn Client secret, potentially compromising LinkedIn application integrations and user data." +regex = '''(?i)[\w.-]{0,50}?(?:linked[_-]?in)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9]{16})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 2 +keywords = [ + "linkedin", + "linked_in", + "linked-in", +] + +[[rules]] +id = "lob-api-key" +description = "Uncovered a Lob API Key, which could lead to unauthorized access to mailing and address verification services." +regex = '''(?i)[\w.-]{0,50}?(?:lob)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}((live|test)_[a-f0-9]{35})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = [ + "test_", + "live_", +] + +[[rules]] +id = "lob-pub-api-key" +description = "Detected a Lob Publishable API Key, posing a risk of exposing mail and print service integrations." +regex = '''(?i)[\w.-]{0,50}?(?:lob)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}((test|live)_pub_[a-f0-9]{31})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = [ + "test_pub", + "live_pub", + "_pub", +] + +[[rules]] +id = "looker-client-id" +description = "Found a Looker Client ID, risking unauthorized access to a Looker account and exposing sensitive data." +regex = '''(?i)[\w.-]{0,50}?(?:looker)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9]{20})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["looker"] + +[[rules]] +id = "looker-client-secret" +description = "Found a Looker Client Secret, risking unauthorized access to a Looker account and exposing sensitive data." +regex = '''(?i)[\w.-]{0,50}?(?:looker)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9]{24})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["looker"] + +[[rules]] +id = "mailchimp-api-key" +description = "Identified a Mailchimp API key, potentially compromising email marketing campaigns and subscriber data." +regex = '''(?i)[\w.-]{0,50}?(?:MailchimpSDK.initialize|mailchimp)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-f0-9]{32}-us\d\d)(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["mailchimp"] + +[[rules]] +id = "mailgun-private-api-token" +description = "Found a Mailgun private API token, risking unauthorized email service operations and data breaches." +regex = '''(?i)[\w.-]{0,50}?(?:mailgun)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}(key-[a-f0-9]{32})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["mailgun"] + +[[rules]] +id = "mailgun-pub-key" +description = "Discovered a Mailgun public validation key, which could expose email verification processes and associated data." +regex = '''(?i)[\w.-]{0,50}?(?:mailgun)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}(pubkey-[a-f0-9]{32})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["mailgun"] + +[[rules]] +id = "mailgun-signing-key" +description = "Uncovered a Mailgun webhook signing key, potentially compromising email automation and data integrity." +regex = '''(?i)[\w.-]{0,50}?(?:mailgun)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-h0-9]{32}-[a-h0-9]{8}-[a-h0-9]{8})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["mailgun"] + +[[rules]] +id = "mapbox-api-token" +description = "Detected a MapBox API token, posing a risk to geospatial services and sensitive location data exposure." +regex = '''(?i)[\w.-]{0,50}?(?:mapbox)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}(pk\.[a-z0-9]{60}\.[a-z0-9]{22})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["mapbox"] + +[[rules]] +id = "mattermost-access-token" +description = "Identified a Mattermost Access Token, which may compromise team communication channels and data privacy." +regex = '''(?i)[\w.-]{0,50}?(?:mattermost)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9]{26})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["mattermost"] + +[[rules]] +id = "maxmind-license-key" +description = "Discovered a potential MaxMind license key." +regex = '''\b([A-Za-z0-9]{6}_[A-Za-z0-9]{29}_mmk)(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 4 +keywords = ["_mmk"] + +[[rules]] +id = "messagebird-api-token" +description = "Found a MessageBird API token, risking unauthorized access to communication platforms and message data." +regex = '''(?i)[\w.-]{0,50}?(?:message[_-]?bird)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9]{25})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = [ + "messagebird", + "message-bird", + "message_bird", +] + +[[rules]] +id = "messagebird-client-id" +description = "Discovered a MessageBird client ID, potentially compromising API integrations and sensitive communication data." +regex = '''(?i)[\w.-]{0,50}?(?:message[_-]?bird)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = [ + "messagebird", + "message-bird", + "message_bird", +] + +[[rules]] +id = "microsoft-teams-webhook" +description = "Uncovered a Microsoft Teams Webhook, which could lead to unauthorized access to team collaboration tools and data leaks." +regex = '''https://[a-z0-9]+\.webhook\.office\.com/webhookb2/[a-z0-9]{8}-([a-z0-9]{4}-){3}[a-z0-9]{12}@[a-z0-9]{8}-([a-z0-9]{4}-){3}[a-z0-9]{12}/IncomingWebhook/[a-z0-9]{32}/[a-z0-9]{8}-([a-z0-9]{4}-){3}[a-z0-9]{12}''' +keywords = [ + "webhook.office.com", + "webhookb2", + "incomingwebhook", +] + +[[rules]] +id = "netlify-access-token" +description = "Detected a Netlify Access Token, potentially compromising web hosting services and site management." +regex = '''(?i)[\w.-]{0,50}?(?:netlify)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9=_\-]{40,46})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["netlify"] + +[[rules]] +id = "new-relic-browser-api-token" +description = "Identified a New Relic ingest browser API token, risking unauthorized access to application performance data and analytics." +regex = '''(?i)[\w.-]{0,50}?(?:new-relic|newrelic|new_relic)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}(NRJS-[a-f0-9]{19})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["nrjs-"] + +[[rules]] +id = "new-relic-insert-key" +description = "Discovered a New Relic insight insert key, compromising data injection into the platform." +regex = '''(?i)[\w.-]{0,50}?(?:new-relic|newrelic|new_relic)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}(NRII-[a-z0-9-]{32})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["nrii-"] + +[[rules]] +id = "new-relic-user-api-id" +description = "Found a New Relic user API ID, posing a risk to application monitoring services and data integrity." +regex = '''(?i)[\w.-]{0,50}?(?:new-relic|newrelic|new_relic)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9]{64})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = [ + "new-relic", + "newrelic", + "new_relic", +] + +[[rules]] +id = "new-relic-user-api-key" +description = "Discovered a New Relic user API Key, which could lead to compromised application insights and performance monitoring." +regex = '''(?i)[\w.-]{0,50}?(?:new-relic|newrelic|new_relic)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}(NRAK-[a-z0-9]{27})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["nrak"] + +[[rules]] +id = "notion-api-token" +description = "Notion API token" +regex = '''\b(ntn_[0-9]{11}[A-Za-z0-9]{32}[A-Za-z0-9]{3})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 4 +keywords = ["ntn_"] + +[[rules]] +id = "npm-access-token" +description = "Uncovered an npm access token, potentially compromising package management and code repository access." +regex = '''(?i)\b(npm_[a-z0-9]{36})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 2 +keywords = ["npm_"] + +[[rules]] +id = "nuget-config-password" +description = "Identified a password within a Nuget config file, potentially compromising package management access." +regex = '''(?i)''' +path = '''(?i)nuget\.config$''' +entropy = 1 +keywords = ["|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9=_\-]{32})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = [ + "nytimes", + "new-york-times", + "newyorktimes", +] + +[[rules]] +id = "octopus-deploy-api-key" +description = "Discovered a potential Octopus Deploy API key, risking application deployments and operational security." +regex = '''\b(API-[A-Z0-9]{26})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 3 +keywords = ["api-"] + +[[rules]] +id = "okta-access-token" +description = "Identified an Okta Access Token, which may compromise identity management services and user authentication data." +regex = '''[\w.-]{0,50}?(?i:[\w.-]{0,50}?(?:(?-i:[Oo]kta|OKTA))(?:[ \t\w.-]{0,20})[\s'"]{0,3})(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}(00[\w=\-]{40})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 4 +keywords = ["okta"] + +[[rules]] +id = "openai-api-key" +description = "Found an OpenAI API Key, posing a risk of unauthorized access to AI services and data manipulation." +regex = '''\b(sk-(?:proj|svcacct|admin)-(?:[A-Za-z0-9_-]{74}|[A-Za-z0-9_-]{58})T3BlbkFJ(?:[A-Za-z0-9_-]{74}|[A-Za-z0-9_-]{58})\b|sk-[a-zA-Z0-9]{20}T3BlbkFJ[a-zA-Z0-9]{20})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 3 +keywords = ["t3blbkfj"] + +[[rules]] +id = "openshift-user-token" +description = "Found an OpenShift user token, potentially compromising an OpenShift/Kubernetes cluster." +regex = '''\b(sha256~[\w-]{43})(?:[^\w-]|\z)''' +entropy = 3.5 +keywords = ["sha256~"] + +[[rules]] +id = "perplexity-api-key" +description = "Detected a Perplexity API key, which could lead to unauthorized access to Perplexity AI services and data exposure." +regex = '''\b(pplx-[a-zA-Z0-9]{48})(?:[\x60'"\s;]|\\[nr]|$|\b)''' +entropy = 4 +keywords = ["pplx-"] + +[[rules]] +id = "pkcs12-file" +description = "Found a PKCS #12 file, which commonly contain bundled private keys." +path = '''(?i)(?:^|\/)[^\/]+\.p(?:12|fx)$''' + +[[rules]] +id = "plaid-api-token" +description = "Discovered a Plaid API Token, potentially compromising financial data aggregation and banking services." +regex = '''(?i)[\w.-]{0,50}?(?:plaid)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}(access-(?:sandbox|development|production)-[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["plaid"] + +[[rules]] +id = "plaid-client-id" +description = "Uncovered a Plaid Client ID, which could lead to unauthorized financial service integrations and data breaches." +regex = '''(?i)[\w.-]{0,50}?(?:plaid)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9]{24})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 3.5 +keywords = ["plaid"] + +[[rules]] +id = "plaid-secret-key" +description = "Detected a Plaid Secret key, risking unauthorized access to financial accounts and sensitive transaction data." +regex = '''(?i)[\w.-]{0,50}?(?:plaid)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9]{30})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 3.5 +keywords = ["plaid"] + +[[rules]] +id = "planetscale-api-token" +description = "Identified a PlanetScale API token, potentially compromising database management and operations." +regex = '''\b(pscale_tkn_(?i)[\w=\.-]{32,64})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 3 +keywords = ["pscale_tkn_"] + +[[rules]] +id = "planetscale-oauth-token" +description = "Found a PlanetScale OAuth token, posing a risk to database access control and sensitive data integrity." +regex = '''\b(pscale_oauth_[\w=\.-]{32,64})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 3 +keywords = ["pscale_oauth_"] + +[[rules]] +id = "planetscale-password" +description = "Discovered a PlanetScale password, which could lead to unauthorized database operations and data breaches." +regex = '''(?i)\b(pscale_pw_(?i)[\w=\.-]{32,64})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 3 +keywords = ["pscale_pw_"] + +[[rules]] +id = "postman-api-token" +description = "Uncovered a Postman API token, potentially compromising API testing and development workflows." +regex = '''\b(PMAK-(?i)[a-f0-9]{24}\-[a-f0-9]{34})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 3 +keywords = ["pmak-"] + +[[rules]] +id = "prefect-api-token" +description = "Detected a Prefect API token, risking unauthorized access to workflow management and automation services." +regex = '''\b(pnu_[a-zA-Z0-9]{36})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 2 +keywords = ["pnu_"] + +[[rules]] +id = "private-key" +description = "Identified a Private Key, which may compromise cryptographic security and sensitive data encryption." +regex = '''(?i)-----BEGIN[ A-Z0-9_-]{0,100}PRIVATE KEY(?: BLOCK)?-----[\s\S-]{64,}?KEY(?: BLOCK)?-----''' +keywords = ["-----begin"] + +[[rules]] +id = "privateai-api-token" +description = "Identified a PrivateAI Token, posing a risk of unauthorized access to AI services and data manipulation." +regex = '''[\w.-]{0,50}?(?i:[\w.-]{0,50}?(?:private[_-]?ai)(?:[ \t\w.-]{0,20})[\s'"]{0,3})(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9]{32})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 3 +keywords = [ + "privateai", + "private_ai", + "private-ai", +] + +[[rules]] +id = "pulumi-api-token" +description = "Found a Pulumi API token, posing a risk to infrastructure as code services and cloud resource management." +regex = '''\b(pul-[a-f0-9]{40})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 2 +keywords = ["pul-"] + +[[rules]] +id = "pypi-upload-token" +description = "Discovered a PyPI upload token, potentially compromising Python package distribution and repository integrity." +regex = '''pypi-AgEIcHlwaS5vcmc[\w-]{50,1000}''' +entropy = 3 +keywords = ["pypi-ageichlwas5vcmc"] + +[[rules]] +id = "rapidapi-access-token" +description = "Uncovered a RapidAPI Access Token, which could lead to unauthorized access to various APIs and data services." +regex = '''(?i)[\w.-]{0,50}?(?:rapidapi)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9_-]{50})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["rapidapi"] + +[[rules]] +id = "readme-api-token" +description = "Detected a Readme API token, risking unauthorized documentation management and content exposure." +regex = '''\b(rdme_[a-z0-9]{70})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 2 +keywords = ["rdme_"] + +[[rules]] +id = "rubygems-api-token" +description = "Identified a Rubygem API token, potentially compromising Ruby library distribution and package management." +regex = '''\b(rubygems_[a-f0-9]{48})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 2 +keywords = ["rubygems_"] + +[[rules]] +id = "scalingo-api-token" +description = "Found a Scalingo API token, posing a risk to cloud platform services and application deployment security." +regex = '''\b(tk-us-[\w-]{48})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 2 +keywords = ["tk-us-"] + +[[rules]] +id = "sendbird-access-id" +description = "Discovered a Sendbird Access ID, which could compromise chat and messaging platform integrations." +regex = '''(?i)[\w.-]{0,50}?(?:sendbird)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["sendbird"] + +[[rules]] +id = "sendbird-access-token" +description = "Uncovered a Sendbird Access Token, potentially risking unauthorized access to communication services and user data." +regex = '''(?i)[\w.-]{0,50}?(?:sendbird)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-f0-9]{40})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["sendbird"] + +[[rules]] +id = "sendgrid-api-token" +description = "Detected a SendGrid API token, posing a risk of unauthorized email service operations and data exposure." +regex = '''\b(SG\.(?i)[a-z0-9=_\-\.]{66})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 2 +keywords = ["sg."] + +[[rules]] +id = "sendinblue-api-token" +description = "Identified a Sendinblue API token, which may compromise email marketing services and subscriber data privacy." +regex = '''\b(xkeysib-[a-f0-9]{64}\-(?i)[a-z0-9]{16})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 2 +keywords = ["xkeysib-"] + +[[rules]] +id = "sentry-access-token" +description = "Found a Sentry.io Access Token (old format), risking unauthorized access to error tracking services and sensitive application data." +regex = '''(?i)[\w.-]{0,50}?(?:sentry)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-f0-9]{64})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 3 +keywords = ["sentry"] + +[[rules]] +id = "sentry-org-token" +description = "Found a Sentry.io Organization Token, risking unauthorized access to error tracking services and sensitive application data." +regex = '''\bsntrys_eyJpYXQiO[a-zA-Z0-9+/]{10,200}(?:LCJyZWdpb25fdXJs|InJlZ2lvbl91cmwi|cmVnaW9uX3VybCI6)[a-zA-Z0-9+/]{10,200}={0,2}_[a-zA-Z0-9+/]{43}(?:[^a-zA-Z0-9+/]|\z)''' +entropy = 4.5 +keywords = ["sntrys_eyjpyxqio"] + +[[rules]] +id = "sentry-user-token" +description = "Found a Sentry.io User Token, risking unauthorized access to error tracking services and sensitive application data." +regex = '''\b(sntryu_[a-f0-9]{64})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 3.5 +keywords = ["sntryu_"] + +[[rules]] +id = "settlemint-application-access-token" +description = "Found a Settlemint Application Access Token." +regex = '''\b(sm_aat_[a-zA-Z0-9]{16})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 3 +keywords = ["sm_aat"] + +[[rules]] +id = "settlemint-personal-access-token" +description = "Found a Settlemint Personal Access Token." +regex = '''\b(sm_pat_[a-zA-Z0-9]{16})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 3 +keywords = ["sm_pat"] + +[[rules]] +id = "settlemint-service-access-token" +description = "Found a Settlemint Service Access Token." +regex = '''\b(sm_sat_[a-zA-Z0-9]{16})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 3 +keywords = ["sm_sat"] + +[[rules]] +id = "shippo-api-token" +description = "Discovered a Shippo API token, potentially compromising shipping services and customer order data." +regex = '''\b(shippo_(?:live|test)_[a-fA-F0-9]{40})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 2 +keywords = ["shippo_"] + +[[rules]] +id = "shopify-access-token" +description = "Uncovered a Shopify access token, which could lead to unauthorized e-commerce platform access and data breaches." +regex = '''shpat_[a-fA-F0-9]{32}''' +entropy = 2 +keywords = ["shpat_"] + +[[rules]] +id = "shopify-custom-access-token" +description = "Detected a Shopify custom access token, potentially compromising custom app integrations and e-commerce data security." +regex = '''shpca_[a-fA-F0-9]{32}''' +entropy = 2 +keywords = ["shpca_"] + +[[rules]] +id = "shopify-private-app-access-token" +description = "Identified a Shopify private app access token, risking unauthorized access to private app data and store operations." +regex = '''shppa_[a-fA-F0-9]{32}''' +entropy = 2 +keywords = ["shppa_"] + +[[rules]] +id = "shopify-shared-secret" +description = "Found a Shopify shared secret, posing a risk to application authentication and e-commerce platform security." +regex = '''shpss_[a-fA-F0-9]{32}''' +entropy = 2 +keywords = ["shpss_"] + +[[rules]] +id = "sidekiq-secret" +description = "Discovered a Sidekiq Secret, which could lead to compromised background job processing and application data breaches." +regex = '''(?i)[\w.-]{0,50}?(?:BUNDLE_ENTERPRISE__CONTRIBSYS__COM|BUNDLE_GEMS__CONTRIBSYS__COM)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-f0-9]{8}:[a-f0-9]{8})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = [ + "bundle_enterprise__contribsys__com", + "bundle_gems__contribsys__com", +] + +[[rules]] +id = "sidekiq-sensitive-url" +description = "Uncovered a Sidekiq Sensitive URL, potentially exposing internal job queues and sensitive operation details." +regex = '''(?i)\bhttps?://([a-f0-9]{8}:[a-f0-9]{8})@(?:gems.contribsys.com|enterprise.contribsys.com)(?:[\/|\#|\?|:]|$)''' +keywords = [ + "gems.contribsys.com", + "enterprise.contribsys.com", +] + +[[rules]] +id = "slack-app-token" +description = "Detected a Slack App-level token, risking unauthorized access to Slack applications and workspace data." +regex = '''(?i)xapp-\d-[A-Z0-9]+-\d+-[a-z0-9]+''' +entropy = 2 +keywords = ["xapp"] + +[[rules]] +id = "slack-bot-token" +description = "Identified a Slack Bot token, which may compromise bot integrations and communication channel security." +regex = '''xoxb-[0-9]{10,13}-[0-9]{10,13}[a-zA-Z0-9-]*''' +entropy = 3 +keywords = ["xoxb"] + +[[rules]] +id = "slack-config-access-token" +description = "Found a Slack Configuration access token, posing a risk to workspace configuration and sensitive data access." +regex = '''(?i)xoxe.xox[bp]-\d-[A-Z0-9]{163,166}''' +entropy = 2 +keywords = [ + "xoxe.xoxb-", + "xoxe.xoxp-", +] + +[[rules]] +id = "slack-config-refresh-token" +description = "Discovered a Slack Configuration refresh token, potentially allowing prolonged unauthorized access to configuration settings." +regex = '''(?i)xoxe-\d-[A-Z0-9]{146}''' +entropy = 2 +keywords = ["xoxe-"] + +[[rules]] +id = "slack-legacy-bot-token" +description = "Uncovered a Slack Legacy bot token, which could lead to compromised legacy bot operations and data exposure." +regex = '''xoxb-[0-9]{8,14}-[a-zA-Z0-9]{18,26}''' +entropy = 2 +keywords = ["xoxb"] + +[[rules]] +id = "slack-legacy-token" +description = "Detected a Slack Legacy token, risking unauthorized access to older Slack integrations and user data." +regex = '''xox[os]-\d+-\d+-\d+-[a-fA-F\d]+''' +entropy = 2 +keywords = [ + "xoxo", + "xoxs", +] + +[[rules]] +id = "slack-legacy-workspace-token" +description = "Identified a Slack Legacy Workspace token, potentially compromising access to workspace data and legacy features." +regex = '''xox[ar]-(?:\d-)?[0-9a-zA-Z]{8,48}''' +entropy = 2 +keywords = [ + "xoxa", + "xoxr", +] + +[[rules]] +id = "slack-user-token" +description = "Found a Slack User token, posing a risk of unauthorized user impersonation and data access within Slack workspaces." +regex = '''xox[pe](?:-[0-9]{10,13}){3}-[a-zA-Z0-9-]{28,34}''' +entropy = 2 +keywords = [ + "xoxp-", + "xoxe-", +] + +[[rules]] +id = "slack-webhook-url" +description = "Discovered a Slack Webhook, which could lead to unauthorized message posting and data leakage in Slack channels." +regex = '''(?:https?://)?hooks.slack.com/(?:services|workflows|triggers)/[A-Za-z0-9+/]{43,56}''' +keywords = ["hooks.slack.com"] + +[[rules]] +id = "snyk-api-token" +description = "Uncovered a Snyk API token, potentially compromising software vulnerability scanning and code security." +regex = '''(?i)[\w.-]{0,50}?(?:snyk[_.-]?(?:(?:api|oauth)[_.-]?)?(?:key|token))(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["snyk"] + +[[rules]] +id = "sonar-api-token" +description = "Uncovered a Sonar API token, potentially compromising software vulnerability scanning and code security." +regex = '''(?i)[\w.-]{0,50}?(?:sonar[_.-]?(login|token))(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}((?:squ_|sqp_|sqa_)?[a-z0-9=_\-]{40})(?:[\x60'"\s;]|\\[nr]|$)''' +secretGroup = 2 +keywords = ["sonar"] + +[[rules]] +id = "sourcegraph-access-token" +description = "Sourcegraph is a code search and navigation engine." +regex = '''(?i)\b(\b(sgp_(?:[a-fA-F0-9]{16}|local)_[a-fA-F0-9]{40}|sgp_[a-fA-F0-9]{40}|[a-fA-F0-9]{40})\b)(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 3 +keywords = [ + "sgp_", + "sourcegraph", +] + +[[rules]] +id = "square-access-token" +description = "Detected a Square Access Token, risking unauthorized payment processing and financial transaction exposure." +regex = '''\b((?:EAAA|sq0atp-)[\w-]{22,60})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 2 +keywords = [ + "sq0atp-", + "eaaa", +] + +[[rules]] +id = "squarespace-access-token" +description = "Identified a Squarespace Access Token, which may compromise website management and content control on Squarespace." +regex = '''(?i)[\w.-]{0,50}?(?:squarespace)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["squarespace"] + +[[rules]] +id = "stripe-access-token" +description = "Found a Stripe Access Token, posing a risk to payment processing services and sensitive financial data." +regex = '''\b((?:sk|rk)_(?:test|live|prod)_[a-zA-Z0-9]{10,99})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 2 +keywords = [ + "sk_test", + "sk_live", + "sk_prod", + "rk_test", + "rk_live", + "rk_prod", +] + +[[rules]] +id = "sumologic-access-id" +description = "Discovered a SumoLogic Access ID, potentially compromising log management services and data analytics integrity." +regex = '''[\w.-]{0,50}?(?i:[\w.-]{0,50}?(?:(?-i:[Ss]umo|SUMO))(?:[ \t\w.-]{0,20})[\s'"]{0,3})(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}(su[a-zA-Z0-9]{12})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 3 +keywords = ["sumo"] + +[[rules]] +id = "sumologic-access-token" +description = "Uncovered a SumoLogic Access Token, which could lead to unauthorized access to log data and analytics insights." +regex = '''(?i)[\w.-]{0,50}?(?:(?-i:[Ss]umo|SUMO))(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9]{64})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 3 +keywords = ["sumo"] + +[[rules]] +id = "telegram-bot-api-token" +description = "Detected a Telegram Bot API Token, risking unauthorized bot operations and message interception on Telegram." +regex = '''(?i)[\w.-]{0,50}?(?:telegr)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([0-9]{5,16}:(?-i:A)[a-z0-9_\-]{34})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["telegr"] + +[[rules]] +id = "travisci-access-token" +description = "Identified a Travis CI Access Token, potentially compromising continuous integration services and codebase security." +regex = '''(?i)[\w.-]{0,50}?(?:travis)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9]{22})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["travis"] + +[[rules]] +id = "twilio-api-key" +description = "Found a Twilio API Key, posing a risk to communication services and sensitive customer interaction data." +regex = '''SK[0-9a-fA-F]{32}''' +entropy = 3 +keywords = ["sk"] + +[[rules]] +id = "twitch-api-token" +description = "Discovered a Twitch API token, which could compromise streaming services and account integrations." +regex = '''(?i)[\w.-]{0,50}?(?:twitch)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9]{30})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["twitch"] + +[[rules]] +id = "twitter-access-secret" +description = "Uncovered a Twitter Access Secret, potentially risking unauthorized Twitter integrations and data breaches." +regex = '''(?i)[\w.-]{0,50}?(?:twitter)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9]{45})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["twitter"] + +[[rules]] +id = "twitter-access-token" +description = "Detected a Twitter Access Token, posing a risk of unauthorized account operations and social media data exposure." +regex = '''(?i)[\w.-]{0,50}?(?:twitter)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([0-9]{15,25}-[a-zA-Z0-9]{20,40})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["twitter"] + +[[rules]] +id = "twitter-api-key" +description = "Identified a Twitter API Key, which may compromise Twitter application integrations and user data security." +regex = '''(?i)[\w.-]{0,50}?(?:twitter)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9]{25})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["twitter"] + +[[rules]] +id = "twitter-api-secret" +description = "Found a Twitter API Secret, risking the security of Twitter app integrations and sensitive data access." +regex = '''(?i)[\w.-]{0,50}?(?:twitter)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9]{50})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["twitter"] + +[[rules]] +id = "twitter-bearer-token" +description = "Discovered a Twitter Bearer Token, potentially compromising API access and data retrieval from Twitter." +regex = '''(?i)[\w.-]{0,50}?(?:twitter)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}(A{22}[a-zA-Z0-9%]{80,100})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["twitter"] + +[[rules]] +id = "typeform-api-token" +description = "Uncovered a Typeform API token, which could lead to unauthorized survey management and data collection." +regex = '''(?i)[\w.-]{0,50}?(?:typeform)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}(tfp_[a-z0-9\-_\.=]{59})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["tfp_"] + +[[rules]] +id = "vault-batch-token" +description = "Detected a Vault Batch Token, risking unauthorized access to secret management services and sensitive data." +regex = '''\b(hvb\.[\w-]{138,300})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 4 +keywords = ["hvb."] + +[[rules]] +id = "vault-service-token" +description = "Identified a Vault Service Token, potentially compromising infrastructure security and access to sensitive credentials." +regex = '''\b((?:hvs\.[\w-]{90,120}|s\.(?i:[a-z0-9]{24})))(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 3.5 +keywords = [ + "hvs.", + "s.", +] +[[rules.allowlists]] +regexes = [ + '''s\.[A-Za-z]{24}''', +] + +[[rules]] +id = "yandex-access-token" +description = "Found a Yandex Access Token, posing a risk to Yandex service integrations and user data privacy." +regex = '''(?i)[\w.-]{0,50}?(?:yandex)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}(t1\.[A-Z0-9a-z_-]+[=]{0,2}\.[A-Z0-9a-z_-]{86}[=]{0,2})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["yandex"] + +[[rules]] +id = "yandex-api-key" +description = "Discovered a Yandex API Key, which could lead to unauthorized access to Yandex services and data manipulation." +regex = '''(?i)[\w.-]{0,50}?(?:yandex)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}(AQVN[A-Za-z0-9_\-]{35,38})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["yandex"] + +[[rules]] +id = "yandex-aws-access-token" +description = "Uncovered a Yandex AWS Access Token, potentially compromising cloud resource access and data security on Yandex Cloud." +regex = '''(?i)[\w.-]{0,50}?(?:yandex)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}(YC[a-zA-Z0-9_\-]{38})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["yandex"] + +[[rules]] +id = "zendesk-secret-key" +description = "Detected a Zendesk Secret Key, risking unauthorized access to customer support services and sensitive ticketing data." +regex = '''(?i)[\w.-]{0,50}?(?:zendesk)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9]{40})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["zendesk"] + diff --git a/crates/aprender-review-experiment/examples/replay.rs b/crates/aprender-review-experiment/examples/replay.rs new file mode 100644 index 0000000000..9abb743f07 --- /dev/null +++ b/crates/aprender-review-experiment/examples/replay.rs @@ -0,0 +1,570 @@ +//! `cargo run -p aprender-review-experiment --example replay -- ` +//! +//! PRM-S1: the `review-replay-v1` speed benchmark (contract `review-replay-v1`, +//! library `replay.rs`). An example, not a `[[bin]]` (see `rex.rs`). +//! +//! Both engines decode Qwen3.5 from the replay GGUF. `llama-server` and +//! `--engine llama_cpp` are the llama.cpp runtime. +//! +//! Commands: +//! - `build --version V --candidates F --tokenize-url U --diffs-out DIR --out SET +//! [--per-stratum N]` F is JSONL `{"group":"owner/repo#N","diff":"path"}`, one +//! line per quorum diff (inputs only). Each diff is composed with the review +//! prompt, templated by `U/apply-template` and counted by `U/tokenize` (a +//! `llama-server` on the replay GGUF). The sealed test manifest and its sketch +//! file are required. Writes the set and `DIR/.diff` for the +//! chosen items, and prints counts and the set sha only. +//! - `run --set SET --diffs DIR --url U --engine apr|llama_cpp --engine-version X +//! --apr-tag T --cell C --gguf-sha G --model M --out ROWS [--server-pid P]` +//! replays every item once through the chat-completions endpoint `U` and +//! writes one `review-replay-receipt-v1` row per item to `ROWS`, replacing +//! any earlier file (the rows are write-once goldens, never appended). +//! `--server-pid` adds the serve's VmHWM (peak RSS so far) to each row. +//! `--ids F` (a `prompt-ids` file for this engine) records each item's +//! prompt-ids sha; a `review-replay-v2` set requires it, and sends +//! `enable_thinking: false` to both engines. +//! - `prompt-ids --set SET --diffs DIR --url BASE --engine E --model M --out F` +//! the ids engine `E` prefills for every item's replay request (llama.cpp: +//! `/apply-template` + `/tokenize`; apr: `/v1/chat/prompt-ids`), one JSONL row +//! per item. Prints the sha over the per-item shas. +//! - `ids-diff A B` the pre-run gate: exit 0 only when two `prompt-ids` files +//! hold identical ids for the same items; else prints the first difference. +//! - `sketch --items DIR` writes `test-sketch-v1.txt` (the near-dup sketches +//! `build` needs) from the corpus items in DIR, e.g. the unpacked +//! `items-v1.tar`. Every test-split diff must hash to its corpus +//! `diff_sha256`, or nothing is written. Prints counts only. +//! - `voters DIR` prints `lane seconds` for every +//! `predicate.consultations..duration_seconds` in `DIR/*/*/receipt.intoto.jsonl` +//! (quorum receipts under `evidence/pr-review/`). +//! - `summary --rows F --voter LANE=FILE... [--prev F]` the §9 speed block as +//! JSON (valid YAML). A voter FILE holds one latency in seconds per line. +//! Exit 1 when the run is refused, and the error names the reason. + +use std::collections::BTreeMap; +use std::io::Write as _; +use std::process::ExitCode; + +use aprender_review_experiment::cluster::{parse_sketches, render_sketches, sketch}; +use aprender_review_experiment::contamination::Index; +use aprender_review_experiment::corpus::{parse_manifest, sha256_hex, Item, Split}; +use aprender_review_experiment::harness::{classify, compose, post}; +use aprender_review_experiment::prereg::PROMPT_V1; +use aprender_review_experiment::replay::{ + build, ids_sha256, pins_prompt, replay_request, summarize, BuildError, Candidate, Engine, Row, + Set, PER_STRATUM, ROW_SCHEMA, +}; +use serde_json::Value; + +const SEED: u64 = 4354; +const CORPUS_DIR: &str = "docs/audits/review-corpus"; + +type Flags = BTreeMap>; + +fn main() -> ExitCode { + let a: Vec = std::env::args().skip(1).collect(); + let r = match a.first().map(String::as_str) { + Some("build") => flags(&a[1..]).and_then(|f| cmd_build(&f)), + Some("run") => flags(&a[1..]).and_then(|f| cmd_run(&f)), + Some("sketch") => flags(&a[1..]).and_then(|f| cmd_sketch(&f)), + Some("voters") => a + .get(1) + .ok_or_else(|| "voters DIR".to_string()) + .and_then(|d| cmd_voters(d)), + Some("summary") => flags(&a[1..]).and_then(|f| cmd_summary(&f)), + Some("prompt-ids") => flags(&a[1..]).and_then(|f| cmd_prompt_ids(&f)), + Some("ids-diff") => match (a.get(1), a.get(2)) { + (Some(x), Some(y)) => cmd_ids_diff(x, y), + _ => Err("ids-diff A B".into()), + }, + _ => Err("usage: replay build|run|prompt-ids|ids-diff|sketch|voters|summary (see the example's docs)".into()), + }; + match r { + Ok(()) => ExitCode::SUCCESS, + Err(e) => { + eprintln!("replay: {e}"); + ExitCode::FAILURE + } + } +} + +/// `--key value` pairs; a key may repeat. +fn flags(a: &[String]) -> Result { + let mut f = Flags::new(); + let mut it = a.iter(); + while let Some(k) = it.next() { + let k = k + .strip_prefix("--") + .ok_or_else(|| format!("expected --flag, got {k:?}"))?; + let v = it.next().ok_or_else(|| format!("--{k} needs a value"))?; + f.entry(k.to_string()).or_default().push(v.clone()); + } + Ok(f) +} + +fn need<'a>(f: &'a Flags, k: &str) -> Result<&'a str, String> { + f.get(k) + .and_then(|v| v.last()) + .map(String::as_str) + .ok_or_else(|| format!("--{k} is required")) +} + +fn read(path: &str) -> Result { + std::fs::read_to_string(path).map_err(|e| format!("{path}: {e}")) +} + +fn sealed_index() -> Result { + let manifest = parse_manifest(&read(&format!("{CORPUS_DIR}/test-manifest-v1.txt"))?) + .ok_or("test manifest does not parse")?; + let sketches = parse_sketches(&read(&format!("{CORPUS_DIR}/test-sketch-v1.txt"))?) + .ok_or("test sketch file does not parse")?; + Ok(Index::new(&manifest).with_sketches(sketches)) +} + +fn post_json(url: &str, body: &Value) -> Result { + let reply = post(url, body)?; + if reply.status != 200 { + return Err(format!("{url}: HTTP {}", reply.status)); + } + serde_json::from_str(&reply.body).map_err(|e| format!("{url}: {e}")) +} + +/// Composed-prompt tokens under the serve's chat template. +fn count_tokens(base: &str, diff: &str) -> Result { + let msgs = Value::Array(vec![Value::Object( + [ + ("role".to_string(), Value::from("user")), + ("content".to_string(), Value::from(compose(PROMPT_V1, diff))), + ] + .into_iter() + .collect(), + )]); + let body = Value::Object([("messages".to_string(), msgs)].into_iter().collect()); + let templated = post_json(&format!("{base}/apply-template"), &body)?; + let prompt = templated["prompt"] + .as_str() + .ok_or("apply-template: no prompt")?; + let body = Value::Object( + [ + ("content".to_string(), Value::from(prompt)), + ("add_special".to_string(), Value::from(true)), + ] + .into_iter() + .collect(), + ); + let toks = post_json(&format!("{base}/tokenize"), &body)?; + toks["tokens"] + .as_array() + .map(|t| t.len() as u64) + .ok_or_else(|| "tokenize: no tokens".to_string()) +} + +fn cmd_build(f: &Flags) -> Result<(), String> { + let index = sealed_index()?; + let base = need(f, "tokenize-url")?.trim_end_matches('/'); + let per = match f.get("per-stratum") { + Some(_) => need(f, "per-stratum")? + .parse() + .map_err(|e| format!("--per-stratum: {e}"))?, + None => PER_STRATUM, + }; + let mut cands = Vec::new(); + for (n, line) in read(need(f, "candidates")?)?.lines().enumerate() { + let v: Value = + serde_json::from_str(line).map_err(|e| format!("candidates:{}: {e}", n + 1))?; + let (Some(group), Some(path)) = (v["group"].as_str(), v["diff"].as_str()) else { + return Err(format!("candidates:{}: needs group and diff", n + 1)); + }; + let diff = read(path)?; + cands.push(Candidate { + group: group.to_string(), + diff_sha256: sha256_hex(diff.as_bytes()), + input_tokens: count_tokens(base, &diff)?, + diff, + }); + } + let set = match build(need(f, "version")?, SEED, per, &cands, &index) { + Ok(s) => s, + Err(BuildError::EmptySealedIndex) => return Err("sealed test manifest is empty".into()), + Err(BuildError::Short(s)) => { + let s: Vec = s + .iter() + .map(|x| format!("{} {}/{}", x.stratum.as_str(), x.have, x.need)) + .collect(); + return Err(format!("short strata (never padded): {}", s.join(", "))); + } + }; + let dir = need(f, "diffs-out")?; + std::fs::create_dir_all(dir).map_err(|e| format!("{dir}: {e}"))?; + for it in &set.items { + let c = cands + .iter() + .find(|c| c.diff_sha256 == it.diff_sha256) + .ok_or("lost a diff")?; + let p = format!("{dir}/{}.diff", it.diff_sha256); + std::fs::write(&p, &c.diff).map_err(|e| format!("{p}: {e}"))?; + } + let out = need(f, "out")?; + std::fs::write(out, set.render()).map_err(|e| format!("{out}: {e}"))?; + println!( + "{{\"candidates\":{},\"items\":{},\"set_sha\":\"{}\"}}", + cands.len(), + set.items.len(), + set.sha() + ); + Ok(()) +} + +fn vm_hwm_mb(pid: &str) -> Option { + let s = std::fs::read_to_string(format!("/proc/{pid}/status")).ok()?; + let kb: f64 = s + .lines() + .find_map(|l| l.strip_prefix("VmHWM:"))? + .trim() + .trim_end_matches("kB") + .trim() + .parse() + .ok()?; + Some(kb / 1024.0) +} + +fn cmd_run(f: &Flags) -> Result<(), String> { + let text = read(need(f, "set")?)?; + let set = Set::parse(&text)?; + let set_sha = sha256_hex(text.as_bytes()); + let engine = match need(f, "engine")? { + "apr" => Engine::Apr, + "llama_cpp" => Engine::LlamaCpp, + e => return Err(format!("--engine {e}: want apr|llama_cpp")), + }; + let (url, model, dir) = (need(f, "url")?, need(f, "model")?, need(f, "diffs")?); + let pid = f.get("server-pid").and_then(|v| v.last()); + // A prompt-pinning set runs only on ids this engine was shown to prefill. + let ids = match f.get("ids") { + Some(_) => ids_by_item(need(f, "ids")?, engine)?, + None if pins_prompt(&set.version) => { + return Err(format!( + "{}: --ids (from prompt-ids) is required", + set.version + )) + } + None => BTreeMap::new(), + }; + if pins_prompt(&set.version) { + if let Some(it) = set + .items + .iter() + .find(|it| !ids.contains_key(&it.diff_sha256)) + { + return Err(format!("--ids has no row for {}", it.diff_sha256)); + } + } + let out_path = need(f, "out")?; + // --out is written whole: the s1-v2 row files are declared `audit-golden` + // (single-writer, never union), so a second run must replace them, not append. + let mut out = std::fs::File::create(out_path).map_err(|e| format!("{out_path}: {e}"))?; + for it in &set.items { + let diff = read(&format!("{dir}/{}.diff", it.diff_sha256))?; + if sha256_hex(diff.as_bytes()) != it.diff_sha256 { + return Err(format!( + "{dir}/{}.diff does not hash to its name", + it.diff_sha256 + )); + } + let reply = post(url, &replay_request(&set.version, model, PROMPT_V1, &diff))?; + let p = classify(&reply); + let row = Row { + schema: ROW_SCHEMA.into(), + replay_version: set.version.clone(), + set_sha: set_sha.clone(), + diff_sha256: it.diff_sha256.clone(), + stratum: it.stratum, + engine, + engine_version: need(f, "engine-version")?.into(), + apr_tag: need(f, "apr-tag")?.into(), + cell: need(f, "cell")?.into(), + gguf_sha256: need(f, "gguf-sha")?.into(), + wall_ms: reply.wall_ms, + prompt_ms: p.server.map(|s| s.prompt_ms), + prompt_tps: p.server.and_then(|s| s.prompt_per_second), + decode_tps: p.server.and_then(|s| s.predicted_per_second), + input_tokens: p.tokens.map(|t| t.prompt), + output_tokens: p.tokens.map(|t| t.completion), + peak_rss_mb: pid.and_then(|p| vm_hwm_mb(p)), + verdict: p.verdict, + prompt_ids_sha256: ids.get(it.diff_sha256.as_str()).cloned(), + }; + let line = serde_json::to_string(&row).map_err(|e| e.to_string())?; + writeln!(out, "{line}").map_err(|e| format!("{out_path}: {e}"))?; + } + println!("{{\"rows\":{},\"set_sha\":\"{set_sha}\"}}", set.items.len()); + Ok(()) +} + +/// Prints `lane seconds` for every `predicate.consultations..duration_seconds` +/// found in one `receipt.intoto.jsonl` file's lines. Returns how many it printed. +fn print_consultation_latencies(text: &str) -> u64 { + let mut n = 0; + for line in text.lines() { + let Ok(v) = serde_json::from_str::(line) else { + continue; + }; + let Some(c) = v["predicate"]["consultations"].as_object() else { + continue; + }; + for (lane, body) in c { + if let Some(s) = body["duration_seconds"].as_f64() { + println!("{lane} {s}"); + n += 1; + } + } + } + n +} + +/// Walks one PR directory's head subdirectories, printing consultation +/// latencies from each `receipt.intoto.jsonl` found. Returns the total printed. +fn count_voters_in_pr(pr_path: &std::path::Path) -> u64 { + let mut n = 0; + for head in std::fs::read_dir(pr_path).into_iter().flatten().flatten() { + let Ok(text) = std::fs::read_to_string(head.path().join("receipt.intoto.jsonl")) else { + continue; + }; + n += print_consultation_latencies(&text); + } + n +} + +/// The ids `engine` prefills for one item's replay request. +/// +/// llama.cpp: `/apply-template` renders the request (it honours +/// `chat_template_kwargs`), and `/tokenize` encodes the result with specials, +/// as its chat path does. apr: `/v1/chat/prompt-ids` answers from the chat path. +// `json!` expands to an `unwrap` of an infallible `to_value` (as in `harness::request_body`). +#[allow(clippy::disallowed_methods)] +fn prompt_ids(base: &str, engine: Engine, body: &Value) -> Result, String> { + let (reply, key) = match engine { + Engine::Apr => ( + post_json(&format!("{base}/v1/chat/prompt-ids"), body)?, + "prompt_ids", + ), + Engine::LlamaCpp => { + let templated = post_json(&format!("{base}/apply-template"), body)?; + let prompt = templated["prompt"] + .as_str() + .ok_or("apply-template: no prompt")?; + let tok = serde_json::json!({ "content": prompt, "add_special": true }); + (post_json(&format!("{base}/tokenize"), &tok)?, "tokens") + } + }; + reply[key] + .as_array() + .ok_or_else(|| format!("{base}: no {key}"))? + .iter() + .map(|t| { + t.as_u64() + .ok_or_else(|| format!("{base}: {key} holds a non-id")) + }) + .collect() +} + +fn engine_of(s: &str) -> Result { + match s { + "apr" => Ok(Engine::Apr), + "llama_cpp" => Ok(Engine::LlamaCpp), + e => Err(format!("--engine {e}: want apr|llama_cpp")), + } +} + +/// `prompt-ids --set SET --diffs DIR --url BASE --engine E --model M --out F`: +/// one JSONL row per item with the ids `E` prefills for its replay request and +/// their sha. Prints the item count and the sha over the per-item shas. +#[allow(clippy::disallowed_methods)] // `json!`, as above. +fn cmd_prompt_ids(f: &Flags) -> Result<(), String> { + let set = Set::parse(&read(need(f, "set")?)?)?; + let engine = engine_of(need(f, "engine")?)?; + let (base, model, dir) = ( + need(f, "url")?.trim_end_matches('/'), + need(f, "model")?, + need(f, "diffs")?, + ); + let out_path = need(f, "out")?; + let mut out = std::fs::File::create(out_path).map_err(|e| format!("{out_path}: {e}"))?; + let mut all = String::new(); + for it in &set.items { + let diff = read(&format!("{dir}/{}.diff", it.diff_sha256))?; + let body = replay_request(&set.version, model, PROMPT_V1, &diff); + let ids = prompt_ids(base, engine, &body)?; + let sha = ids_sha256(&ids); + let row = serde_json::json!({ + "diff_sha256": it.diff_sha256, "engine": engine, + "n_tokens": ids.len(), "ids_sha256": sha, "ids": ids, + }); + writeln!(out, "{row}").map_err(|e| format!("{out_path}: {e}"))?; + all.push_str(&sha); + all.push('\n'); + } + println!( + "{{\"items\":{},\"ids_sha\":\"{}\"}}", + set.items.len(), + sha256_hex(all.as_bytes()) + ); + Ok(()) +} + +/// A `prompt-ids` file as `diff_sha256 -> (ids_sha256, ids)`. +fn ids_file(path: &str) -> Result)>, String> { + let mut m = BTreeMap::new(); + for (n, line) in read(path)?.lines().enumerate() { + let v: Value = serde_json::from_str(line).map_err(|e| format!("{path}:{}: {e}", n + 1))?; + let bad = || format!("{path}:{}: not a prompt-ids row", n + 1); + let engine = engine_of(v["engine"].as_str().ok_or_else(bad)?)?; + let ids: Vec = v["ids"] + .as_array() + .ok_or_else(bad)? + .iter() + .filter_map(Value::as_u64) + .collect(); + let sha = v["ids_sha256"].as_str().ok_or_else(bad)?.to_string(); + if ids_sha256(&ids) != sha { + return Err(format!("{path}:{}: ids do not hash to ids_sha256", n + 1)); + } + let key = v["diff_sha256"].as_str().ok_or_else(bad)?.to_string(); + m.insert(key, (engine, sha, ids)); + } + Ok(m) +} + +fn ids_by_item(path: &str, engine: Engine) -> Result, String> { + ids_file(path)? + .into_iter() + .map(|(k, (e, sha, _))| { + if e == engine { + Ok((k, sha)) + } else { + Err(format!( + "{path}: ids are from {e:?}, this run is {engine:?}" + )) + } + }) + .collect() +} + +/// `ids-diff A B`: the pre-run gate. Exit 0 only when both files cover the +/// same items and every item's ids are identical; otherwise print the first +/// differing item, position and a few ids either side. +#[allow(clippy::disallowed_methods)] // `json!`, as above. +fn cmd_ids_diff(a: &str, b: &str) -> Result<(), String> { + let (ma, mb) = (ids_file(a)?, ids_file(b)?); + if !ma.keys().eq(mb.keys()) { + return Err(format!("{a} and {b} cover different items")); + } + let mut differ = Vec::new(); + for (k, (_, sa, ia)) in &ma { + let (_, sb, ib) = &mb[k]; + if sa != sb { + let pos = ia + .iter() + .zip(ib) + .position(|(x, y)| x != y) + .unwrap_or(ia.len().min(ib.len())); + let win = |v: &[u64]| v[pos.saturating_sub(3)..(pos + 4).min(v.len())].to_vec(); + differ.push(serde_json::json!({ + "diff_sha256": k, "at": pos, "n": [ia.len(), ib.len()], + "a": win(ia), "b": win(ib), + })); + } + } + println!( + "{}", + serde_json::json!({ + "items": ma.len(), "differ": differ.len(), + "first": differ.first(), + }) + ); + if differ.is_empty() { + Ok(()) + } else { + Err(format!( + "{} of {} items prefill different ids", + differ.len(), + ma.len() + )) + } +} + +fn cmd_voters(dir: &str) -> Result<(), String> { + let mut n = 0; + for pr in std::fs::read_dir(dir) + .map_err(|e| format!("{dir}: {e}"))? + .flatten() + { + n += count_voters_in_pr(&pr.path()); + } + if n == 0 { + return Err(format!("{dir}: no consultation latencies")); + } + Ok(()) +} + +fn rows_of(path: &str) -> Result, String> { + read(path)? + .lines() + .enumerate() + .map(|(i, l)| serde_json::from_str(l).map_err(|e| format!("{path}:{}: {e}", i + 1))) + .collect() +} + +fn cmd_summary(f: &Flags) -> Result<(), String> { + let rows = rows_of(need(f, "rows")?)?; + let prev = match f.get("prev") { + Some(_) => Some(rows_of(need(f, "prev")?)?), + None => None, + }; + let mut voters = Vec::new(); + for v in f.get("voter").into_iter().flatten() { + let (lane, path) = v + .split_once('=') + .ok_or_else(|| format!("--voter {v}: want LANE=FILE"))?; + let xs = read(path)? + .split_whitespace() + .map(|x| x.parse::().map_err(|e| format!("{path}: {x}: {e}"))) + .collect::, _>>()?; + voters.push((lane.to_string(), xs)); + } + let speed = + summarize(&rows, &voters, prev.as_deref(), SEED).map_err(|e| format!("refused: {e:?}"))?; + println!( + "{}", + serde_json::to_string_pretty(&speed).map_err(|e| e.to_string())? + ); + Ok(()) +} + +/// `sketch`: the test items' near-dup sketches, written only when every +/// test-split diff in DIR is byte-identical to the one the corpus sealed. +fn cmd_sketch(f: &Flags) -> Result<(), String> { + let dir = need(f, "items")?; + let mut out: Vec<(String, Vec)> = Vec::new(); + let mut tests = 0usize; + for line in read(&format!("{CORPUS_DIR}/corpus-v1.jsonl"))?.lines() { + let item: Item = serde_json::from_str(line).map_err(|e| e.to_string())?; + if item.split != Split::Test { + continue; + } + tests += 1; + let diff = read(&format!("{dir}/{}.diff", item.id))?; + if sha256_hex(diff.as_bytes()) != item.diff_sha256 { + return Err(format!("{}: diff sha differs from the corpus", item.id)); + } + if let Some(s) = sketch(&diff) { + out.push((item.id, s)); + } + } + if tests == 0 { + return Err("the corpus has no test items".into()); + } + let path = format!("{CORPUS_DIR}/test-sketch-v1.txt"); + std::fs::write(&path, render_sketches(&out)).map_err(|e| format!("{path}: {e}"))?; + println!("test_items {tests} sketched {}", out.len()); + Ok(()) +} diff --git a/crates/aprender-review-experiment/examples/rex.rs b/crates/aprender-review-experiment/examples/rex.rs index dfe56cad54..972dbd5203 100644 --- a/crates/aprender-review-experiment/examples/rex.rs +++ b/crates/aprender-review-experiment/examples/rex.rs @@ -3,12 +3,16 @@ //! An example, not a `[[bin]]`: the workspace binary register only shrinks //! (monorepo_invariants::test_no_unauthorized_binaries). //! +//! The lanes run Qwen3.5 (4B primary, 9B control). `llama` in a flag or a +//! cell name (`--llama-receipts`, `--llama-cell`) is the llama.cpp runtime. +//! //! Commands: //! - `prereg` print the prereg lock the tree implies //! - `prereg-check` exit 1 unless the committed lock matches the tree //! - `corpus-build PRS_JSON CUTOFF ITEMS_DIR MUTANTS_JSON...` //! build corpus v1 (REX-02): writes `ITEMS_DIR/.diff`, and in the repo -//! `docs/audits/review-corpus/corpus-v1.jsonl` + `test-manifest-v1.txt`. +//! `docs/audits/review-corpus/corpus-v1.jsonl` + `test-manifest-v1.txt` + +//! `test-sketch-v1.txt` (PRA-001 T7 near-dup sketches of the test items). //! Prints counts only; it never prints a test item. //! - `review --url U --cell C --host H --backend B --arm A --model-id M //! --weights-sha W --apr-tag T --apr-sha S --items DIR --out DIR --split dev|test @@ -20,24 +24,89 @@ //! - `score --receipts F --cell C --arm A --split dev|test [--rerun] //! [--pubkey P]` the §2.3 metrics as JSON. Without `--pubkey` the result //! is labelled `unsigned` (exploratory); with it, `minisign -V` must pass. +//! `--pilot` (with `--split dev`) adds the REX-05 projection: warm/cold +//! wall-clock spread, projected test runtime, and the §2.2 sample-size rule. //! - `admit --cell C|all --why NoDeclaredExecutor --model-id M --weights-sha W //! --apr-tag T --apr-sha S --out FILE` append `NotRun` admission rows (REX-04); -//! `admit --cell C --removed-by R ...` appends a `Refused` row. +//! `admit --cell C --removed-by R ...` appends a `Refused` row; +//! `admit --cell C --parity-receipt R --threshold T --threshold-basis B +//! --min-positions N ...` appends an `Admitted` row derived from the receipt +//! (FALSIFY-RCA-004..006), or writes nothing, exits 1 and says why. //! - `admission-check --file F` every §2.1 cell resolved exactly once; prints the //! summary JSON. Exit 1 if inadmissible, 10 if admissible but S-7 (no cell admitted). +//! - `ledger REPO OUT_JSONL QUORUM_RECEIPT...` REX-07: write `review-ledger-v2` +//! rows from quorum receipts and print shadow coverage. Exit 10 unless every +//! receipt carries an uncounted shadow row that leaves the width alone. +//! - `ladder REPORT_JSON [--current RUNG]` PRM-09: the lane's next rung +//! (shadow/tripwire/tie-breaker/vote) from a `prm-001-report-v1` report, one +//! rung from `--current` (default shadow), with every reason it stopped below +//! vote. Exit 0 whatever the rung, 2 on usage; the gates read `.mode`. +//! - `ratchet record --file F --tag T --cell C --receipts R [--llama-receipts L +//! --llama-cell LC]` REX-10: append one `review-lane-perf-ratchet-v1` entry +//! from the tag's warm, non-rerun receipts on the cell (and llama.cpp's p95 on +//! the same items). A tag with no timings is refused: it did not run. +//! - `ratchet check --file F` the andon JSON (arming/green/red). Exit 10 on RED: +//! a paired Harrell–Davis p95 rise over the best earlier tag, or a violation. +//! - `challenge --ledger F --test-version V --id ID --tier B1a --weights-sha W +//! --prompt-sha P [--adapter-sha A] --initial-weights-sha W0 --initial-prompt-sha P0 +//! --cell C --pubkey K --challenger-receipts R --challenger-arm A +//! --champion-receipts R --champion-arm A [--h1 holds|fails] [--h2 holds|fails] +//! [--contamination-hits N] [--p95-s X] [--budget-p95-s Y]` REX-11: the §5.4 +//! decision on the sealed test split, appended to the +//! `review-champion-challenger-v1` ledger. Both receipt files must verify under +//! `--pubkey` (unsigned data cannot promote, R-1); an omitted gate fails. +//! Exit 0 promoted, 10 rejected, 11 refused (no evaluation spent). +//! - `b2 status [--refusals F] [--accepted verb=receipt,...]` REX-12: one +//! `review-b2-loop-v1` row per B2 tier (ready, verb_refused, unledgered). +//! Exit 11 while any row is NotRun. +//! - `b2 teacher-receipt --logits F --teacher-sha W --k K` the teacher-logit +//! dataset receipt (sha, count, 0 test hashes) or every reason it is refused. +//! - `lane-kappa ROWS --split val [--shadow qwen-shadow] (--min-n N | --manifest M +//! --candidate ROWS2)` PRM-C13 (was PRA-001 T13) (contract lane-independence-v1): κ_err of the +//! shadow lane against every counted lane on matured gold rows of the split, as +//! JSON; a thin pair prints `insufficient`, never a number. With `--candidate` +//! the gate runs H7 (δ-free, `stats::h7_holds`) on the candidate rows: the +//! voters and min_n come only from the manifest's `lane_independence` block. Exit 12 (andon, S-14) on a refused candidate, an unusable manifest, +//! or `--candidate` without `--manifest`. +//! - `workload ROWS --min-n N` PRM-S2 (contract workload-profile-v1): per-lane +//! input/output token p50/p95 over agent-trace-v1 rows, §0.2 decided per lane +//! and the replay strata weights over distinct diffs, as the receipt's +//! `workload` block. A falsified §0.2 is a measured result (exit 0); an +//! undecided one (no lane at min_n) is exit 13 (andon, S-14). +//! - `sparse-logits --chat F --store DIR --model-sha M --tokenizer-sha T --apr-tag V +//! --backend B --temperature X [--k K]` PRM-C11 (contract sparse-logits-v1): one +//! `apr serve` chat-completion body into a sparse-logits-v1 blob in the local CAS +//! under DIR; prints the capture (`logits_sha`, sizes) as JSON for the +//! agent-trace-v1 row. Exit 1 on a refused body (nothing is written), 2 on usage. +//! - `pool-admit ROWS --pool F --quarantine Q` PRM-C6 backfill (contract +//! trace-admission-secret-v1): every agent-trace-v1 row through the pool gate +//! (sealed-test refusal, G-PROV, both secret scanners); admitted rows to F, +//! quarantined rows byte-identical to Q, the per-verdict and per-rule counts as +//! JSON. Never overwrites F or Q (exit 1). An empty sealed index is exit 12 +//! (andon) and writes nothing. +//! - `datacard --root DIR` PRM-C14 (contract trace-datacard-v1): writes the +//! Croissant card, datasheet, manifest and G16 weekly receipt for the store under +//! DIR (idempotent; refuses hand-edited files, exit 1) and prints each ISO week's +//! G16 verdict. Any unmeasured week, or none, is exit 13 (andon): G16 stays `[U]`. +use aprender_review_experiment::b2; use aprender_review_experiment::build_corpus::{ choose, g_candidates, is_green, p_candidates, r_candidates, seal, Mutant, Pr, PER_CLASS, }; +use aprender_review_experiment::champion; +use aprender_review_experiment::cluster::{parse_sketches, render_sketches, sketch}; +use aprender_review_experiment::contamination::Index; use aprender_review_experiment::corpus::{ - assign_splits, corpus_version, render_manifest, sha256_hex, Item, Split, + assign_splits, corpus_version, parse_manifest, render_manifest, sha256_hex, Item, Split, }; use aprender_review_experiment::harness::{ classify, post, request_body, rerun_subset, run_order, utc_now, Run, }; +use aprender_review_experiment::pilot::{project, Projection}; use aprender_review_experiment::prereg; -use aprender_review_experiment::receipt::{Arm, Expect, NotRun, Receipt}; -use aprender_review_experiment::score::{collect, score}; +use aprender_review_experiment::ratchet; +use aprender_review_experiment::receipt::{admissible, Arm, Expect, NotRun, Receipt}; +use aprender_review_experiment::score::{collect, score, Ratio, Scored}; use std::collections::BTreeMap; use std::process::ExitCode; @@ -76,7 +145,7 @@ fn to_exit_code(result: Result<(), String>, label: &str) -> ExitCode { /// Prints the top-level usage line and exits 2 (unknown/missing command). fn usage() -> ExitCode { eprintln!( - "Usage: rex (see the example docs)" + "Usage: rex (see the example docs)" ); ExitCode::from(2) } @@ -87,6 +156,16 @@ fn main() -> ExitCode { Some("prereg") => prereg_cmd(false), Some("prereg-check") => prereg_cmd(true), Some("admission-check") => admission_check(&args[1..]), + Some("ledger") => ledger_cmd(&args[1..]), + Some("ladder") => ladder_cmd(&args[1..]), + Some("ratchet") => ratchet_cmd(&args[1..]), + Some("challenge") => challenge_cmd(&args[1..]), + Some("b2") => b2_cmd(&args[1..]), + Some("lane-kappa") => lane_kappa_cmd(&args[1..]), + Some("workload") => workload_cmd(&args[1..]), + Some("sparse-logits") => sparse_logits_cmd(&args[1..]), + Some("pool-admit") => pool_admit_cmd(&args[1..]), + Some("datacard") => datacard_cmd(&args[1..]), Some(c @ ("review" | "not-run" | "score" | "admit")) => run_flagged_cmd(c, &args[1..]), Some("corpus-build") if args.len() >= 5 => run_corpus_build_cmd(&args[1..]), _ => usage(), @@ -104,11 +183,11 @@ fn prereg_cmd(check: bool) -> ExitCode { } let bad = prereg::verify(prereg::LOCK, &c); if bad.is_empty() { - println!("rex-prereg-v1 OK prereg_sha={}", c.prereg_sha()); + println!("{} OK prereg_sha={}", prereg::SCHEME, c.prereg_sha()); return ExitCode::SUCCESS; } for b in &bad { - eprintln!("rex-prereg-v1 DRIFT {b}"); + eprintln!("{} DRIFT {b}", prereg::SCHEME); } ExitCode::from(1) } @@ -167,6 +246,16 @@ fn write_outputs(all: &[(Item, String)], items_dir: &str) -> Result<(), String> .or_default() += 1; } let manifest = render_manifest(&seal(all)); + let sketches: Vec<(String, Vec)> = all + .iter() + .filter(|(i, _)| i.split == Split::Test) + .filter_map(|(i, d)| Some((i.id.clone(), sketch(d)?))) + .collect(); + std::fs::write( + format!("{CORPUS_DIR}/test-sketch-v1.txt"), + render_sketches(&sketches), + ) + .map_err(|e| e.to_string())?; std::fs::write(format!("{CORPUS_DIR}/corpus-v1.jsonl"), &jsonl).map_err(|e| e.to_string())?; std::fs::write(format!("{CORPUS_DIR}/test-manifest-v1.txt"), &manifest) .map_err(|e| e.to_string())?; @@ -356,6 +445,53 @@ struct Report<'a> { corpus_version: &'a str, rejected_rows: usize, score: aprender_review_experiment::score::LaneScore, + #[serde(skip_serializing_if = "Option::is_none")] + pilot: Option, +} + +/// The `--pilot` projection for `score_cmd`: warm/cold wall-clock spread and +/// the §2.2 sample-size rule, or `None` when `--pilot` was not passed. +fn build_pilot( + f: &Flags, + recall: Ratio, + items: &[Item], + cell: &str, + arm: Arm, + lines: &str, + expect: Expect<'_>, + rows: &[Scored], +) -> Result, String> { + if !f.contains_key("pilot") { + return Ok(None); + } + if split_of(f)? != Split::Dev { + return Err("--pilot projects from dev items: pass --split dev".into()); + } + let ran: std::collections::BTreeSet<&str> = rows + .iter() + .filter(|s| s.verdict.executed()) + .map(|s| s.id.as_str()) + .collect(); + let (mut warm, mut cold) = (Vec::new(), Vec::new()); + for r in lines.lines().filter_map(|l| admissible(l, expect).ok()) { + let mine = r.cell == cell && r.arm == arm && !r.rerun; + if let (true, Some(t)) = (mine && ran.contains(r.item_id.as_str()), r.timings) { + if r.cold { + cold.push(t.wall_ms); + } else { + warm.push(t.wall_ms); + } + } + } + let test: Vec<&Item> = items.iter().filter(|i| i.split == Split::Test).collect(); + let defects = test.iter().filter(|i| i.class.is_defect()).count() as u64; + Ok(Some(project( + recall, + &warm, + &cold, + defects, + test.len() as u64, + ))) } fn score_cmd(f: &Flags) -> Result<(), String> { @@ -363,14 +499,7 @@ fn score_cmd(f: &Flags) -> Result<(), String> { let path = need(f, "receipts")?; let signed = match f.get("pubkey") { Some(pk) => { - let ok = std::process::Command::new("minisign") - .args(["-Vqm", path, "-p", pk]) - .status() - .map_err(|e| format!("minisign: {e}"))? - .success(); - if !ok { - return Err(format!("{path}: signature does not verify under {pk}")); - } + verify_signed(path, pk)?; "signed" } None => "unsigned (exploratory)", @@ -397,6 +526,8 @@ fn score_cmd(f: &Flags) -> Result<(), String> { |r| r.cell == cell && r.arm == arm && r.rerun == rerun, |p| std::fs::read_to_string(root.join(p)).ok(), ); + let sc = score(&rows); + let pilot = build_pilot(f, sc.recall, &items, &cell, arm, &lines, expect, &rows)?; let out = Report { cell: &cell, arm, @@ -404,7 +535,8 @@ fn score_cmd(f: &Flags) -> Result<(), String> { prereg_sha: &prereg_sha, corpus_version: &cv, rejected_rows: rejected.len(), - score: score(&rows), + score: sc, + pilot, }; println!( "{}", @@ -418,20 +550,43 @@ fn score_cmd(f: &Flags) -> Result<(), String> { /// REX-04: append admission rows for one cell or all six. fn admit(f: &Flags) -> Result<(), String> { - use aprender_review_experiment::admission::{Row, Status, CELLS, SCHEME}; + use aprender_review_experiment::admission::{ + parity_from_receipt, Expect, Row, Status, CELLS, SCHEME, + }; let prereg_sha = prereg::locked_prereg_sha() .ok_or("no locked prereg sha")? .to_string(); - let status = match f.get("removed-by") { - Some(r) => Status::Refused { + let which = need(f, "cell")?; + let status = match (f.get("parity-receipt"), f.get("removed-by")) { + (Some(path), _) => { + if which == "all" { + return Err("--parity-receipt admits one --cell, not all".into()); + } + let num = |k: &str| -> Result { + need(f, k)? + .parse() + .map_err(|_| format!("--{k}: not a number")) + }; + let bytes = std::fs::read(path).map_err(|e| format!("{path}: {e}"))?; + let x = Expect { + apr_sha256: need(f, "apr-sha")?, + weights_sha256: need(f, "weights-sha")?, + threshold: num("threshold")?, + threshold_basis: need(f, "threshold-basis")?, + min_positions: num("min-positions")? as usize, + }; + let parity = parity_from_receipt(&bytes, &x) + .map_err(|e| format!("{which}: {path} does not admit: {}", e.join("; ")))?; + Status::Admitted { parity } + } + (None, Some(r)) => Status::Refused { removed_by: r.clone(), }, - None => Status::NotRun { + (None, None) => Status::NotRun { reason: serde_json::from_value(serde_json::Value::String(need(f, "why")?.into())) .map_err(|_| "--why NoDeclaredExecutor|ServeError|…")?, }, }; - let which = need(f, "cell")?; let cells: Vec<_> = CELLS .iter() .filter(|c| which == "all" || c.cell == which) @@ -501,3 +656,646 @@ fn admission_check(a: &[String]) -> ExitCode { } } } + +fn ledger_cmd(a: &[String]) -> ExitCode { + let (Some(repo), Some(out), receipts) = (a.first(), a.get(1), a.get(2..).unwrap_or(&[])) else { + eprintln!("usage: rex ledger REPO OUT_JSONL QUORUM_RECEIPT..."); + return ExitCode::from(2); + }; + let mut read = Vec::new(); + for p in receipts { + match std::fs::read_to_string(p) { + Ok(t) => read.push((p.clone(), t)), + Err(e) => { + eprintln!("rex ledger: {p}: {e}"); + return ExitCode::from(1); + } + } + } + let (rows, c) = aprender_review_experiment::ledger::build(&read, repo); + let lines: Result = rows + .iter() + .map(|r| serde_json::to_string(r).map(|j| j + "\n")) + .collect(); + match lines + .map_err(|e| e.to_string()) + .and_then(|l| std::fs::write(out, l).map_err(|e| format!("{out}: {e}"))) + { + Ok(()) => {} + Err(e) => { + eprintln!("rex ledger: {e}"); + return ExitCode::from(1); + } + } + match serde_json::to_string(&c) { + Ok(j) => println!("{j}"), + Err(e) => eprintln!("rex ledger: {e}"), + } + if c.holds() { + ExitCode::SUCCESS + } else { + eprintln!( + "REX-07: {} of {} quorums carry a shadow row, {} violations", + c.carried, + c.quorums, + c.violations.len() + ); + ExitCode::from(10) + } +} + +fn ladder_cmd(a: &[String]) -> ExitCode { + use aprender_review_experiment::ladder::{decide, Mode}; + let usage = || { + eprintln!("usage: rex ladder REPORT_JSON [--current shadow|tripwire|tie-breaker|vote]"); + ExitCode::from(2) + }; + let (path, current) = match a { + [p] => (p, Mode::Shadow), + [p, f, r] if f == "--current" => { + match serde_json::from_value(serde_json::Value::String(r.clone())) { + Ok(m) => (p, m), + Err(_) => return usage(), + } + } + _ => return usage(), + }; + let Some(lock) = prereg::locked_prereg_sha() else { + eprintln!("rex ladder: no locked prereg sha"); + return ExitCode::from(1); + }; + let text = match std::fs::read_to_string(path) { + Ok(t) => t, + Err(e) => { + eprintln!("rex ladder: {path}: {e}"); + return ExitCode::from(1); + } + }; + let d = decide(&text, lock, current); + match serde_json::to_string(&d) { + Ok(j) => println!("{j}"), + Err(e) => eprintln!("rex ladder: {e}"), + } + ExitCode::SUCCESS +} + +/// Exit code for a lane-independence andon (S-14). +const ANDON_KAPPA: u8 = 12; + +fn lane_kappa_cmd(a: &[String]) -> ExitCode { + use aprender_review_experiment::kappa_probe::{gate, parse_manifest, parse_rows, probe}; + const USAGE: &str = "usage: rex lane-kappa ROWS --split val [--shadow L] (--min-n N | --manifest M --candidate ROWS2)"; + let (Some(path), Ok(f)) = (a.first(), flags(a.get(1..).unwrap_or_default())) else { + eprintln!("{USAGE}"); + return ExitCode::from(2); + }; + let Some(split) = f.get("split") else { + eprintln!("{USAGE}"); + return ExitCode::from(2); + }; + let shadow = f.get("shadow").map_or("qwen-shadow", String::as_str); + let load = |p: &str| { + std::fs::read_to_string(p) + .map_err(|e| format!("{p}: {e}")) + .and_then(|t| parse_rows(&t).map_err(|e| format!("{p}: {e}"))) + }; + let andon = |why: String| { + eprintln!("rex lane-kappa: ANDON {why}"); + ExitCode::from(ANDON_KAPPA) + }; + let manifest = match f.get("manifest").map(|m| { + std::fs::read_to_string(m) + .map_err(|e| format!("{m}: {e}")) + .and_then(|t| parse_manifest(&t)) + }) { + None => None, + Some(Ok(m)) => Some(m), + Some(Err(e)) => return andon(format!("S-14 manifest: {e}")), + }; + if f.contains_key("candidate") && manifest.is_none() { + return andon( + "S-14 --candidate needs --manifest: the voters and min_n are not pre-registered".into(), + ); + } + let min_n = match (&manifest, f.get("min-n").map(|n| n.parse::())) { + (Some(m), _) => m.min_n, + (None, Some(Ok(n))) if n > 0 => n, + _ => { + eprintln!("{USAGE}"); + return ExitCode::from(2); + } + }; + let baseline = match load(path) { + Ok(r) => probe(&r, split, shadow, min_n), + Err(e) => { + eprintln!("rex lane-kappa: {e}"); + return ExitCode::from(1); + } + }; + let mut out = serde_json::Map::new(); + out.insert("baseline".into(), to_json(&baseline)); + let mut code = ExitCode::SUCCESS; + if let (Some(m), Some(c)) = (&manifest, f.get("candidate")) { + let rows = match load(c) { + Ok(r) => r, + Err(e) => { + eprintln!("rex lane-kappa: {e}"); + return ExitCode::from(1); + } + }; + let candidate = probe(&rows, split, shadow, min_n); + let v = gate(&rows, split, shadow, m); + if !v.pass { + eprintln!("rex lane-kappa: ANDON {}", v.refusals.join("; ")); + code = ExitCode::from(ANDON_KAPPA); + } + out.insert("candidate".into(), to_json(&candidate)); + out.insert("manifest".into(), to_json(m)); + out.insert("gate".into(), to_json(&v)); + } + println!("{}", serde_json::Value::Object(out)); + code +} + +fn to_json(x: &T) -> serde_json::Value { + serde_json::to_value(x).unwrap_or(serde_json::Value::Null) +} + +/// Warm, non-rerun, admissible timings of one cell (and tag, when given). +fn warm_timings( + path: &str, + cell: &str, + tag: Option<&str>, +) -> Result<(Vec, String), String> { + let (_, prereg_sha, cv) = corpus()?; + let text = std::fs::read_to_string(path).map_err(|e| format!("{path}: {e}"))?; + let expect = Expect { + prereg_sha: &prereg_sha, + corpus_version: &cv, + }; + let rows = text + .lines() + .filter_map(|l| admissible(l, expect).ok()) + .filter(|r| { + r.cell == cell + && tag.is_none_or(|t| r.apr_tag == t) + && !r.cold + && !r.rerun + && r.timings.is_some() + }) + .collect(); + Ok((rows, sha256_hex(text.as_bytes()))) +} + +fn samples(rows: &[Receipt]) -> Vec> { + rows.iter() + .filter_map(|r| { + r.timings.map(|t| ratchet::Sample { + item_id: &r.item_id, + apr_sha256: &r.apr_sha256, + wall_ms: t.wall_ms, + }) + }) + .collect() +} + +fn ratchet_entries(file: &str) -> Result, String> { + match std::fs::read_to_string(file) { + Ok(t) => t + .lines() + .filter(|l| !l.trim().is_empty()) + .map(|l| serde_json::from_str(l).map_err(|e| format!("{file}: {e}"))) + .collect(), + Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(Vec::new()), + Err(e) => Err(format!("{file}: {e}")), + } +} + +fn ratchet_record(f: &Flags) -> Result<(), String> { + let (file, tag, cell) = (need(f, "file")?, need(f, "tag")?, need(f, "cell")?); + let (rows, sha) = warm_timings(need(f, "receipts")?, cell, Some(tag))?; + let llama = match f.get("llama-receipts") { + Some(p) => warm_timings(p, need(f, "llama-cell")?, None)?.0, + None => Vec::new(), + }; + let entry = ratchet::record(tag, cell, &samples(&rows), &samples(&llama), &sha)?; + let mut all = ratchet_entries(file)?; + all.push(entry.clone()); + let line = serde_json::to_string(&entry).map_err(|e| e.to_string())?; + let mut out = std::fs::OpenOptions::new() + .create(true) + .append(true) + .open(file) + .map_err(|e| format!("{file}: {e}"))?; + std::io::Write::write_all(&mut out, format!("{line}\n").as_bytes()) + .map_err(|e| e.to_string())?; + println!( + "recorded {tag} on {cell}: {} items, p95 {:.0} ms; andon {:?}", + entry.items.len(), + entry.p95_ms, + ratchet::check(&all).andon + ); + Ok(()) +} + +fn ratchet_cmd(a: &[String]) -> ExitCode { + let run = |sub: &str| -> Result { + let f = flags(&a[1..])?; + if sub == "record" { + return ratchet_record(&f).map(|()| ExitCode::SUCCESS); + } + let v = ratchet::check(&ratchet_entries(need(&f, "file")?)?); + println!( + "{}", + serde_json::to_string_pretty(&v).map_err(|e| e.to_string())? + ); + Ok(if v.andon == ratchet::Andon::Red { + ExitCode::from(10) + } else { + ExitCode::SUCCESS + }) + }; + match a.first().map(String::as_str) { + Some(s @ ("record" | "check")) => run(s).unwrap_or_else(|e| { + eprintln!("rex ratchet {s}: {e}"); + ExitCode::from(1) + }), + _ => { + eprintln!("usage: rex ratchet record|check --file F ... (see the example docs)"); + ExitCode::from(2) + } + } +} + +/// REX-12: `b2 status` (exit 11 while any B2 row is NotRun) and +/// `b2 teacher-receipt`. +fn b2_cmd(a: &[String]) -> ExitCode { + let run = |sub: &str| -> Result { + let f = flags(&a[1..])?; + if sub == "status" { + let path = f + .get("refusals") + .map_or("evidence/verbs/refusals.json", String::as_str); + let text = std::fs::read_to_string(path).map_err(|e| format!("{path}: {e}"))?; + let ledger = b2::parse_ledger(&text).map_err(|e| format!("{path}: {e}"))?; + let mut accepted = BTreeMap::new(); + for kv in f.get("accepted").map_or("", String::as_str).split(',') { + if kv.is_empty() { + continue; + } + let (verb, receipt) = kv + .split_once('=') + .ok_or_else(|| format!("--accepted {kv}: verb=receipt"))?; + accepted.insert(verb.to_string(), receipt.to_string()); + } + let rows = b2::status(&ledger, &accepted)?; + for r in &rows { + println!("{}", serde_json::to_string(r).map_err(|e| e.to_string())?); + } + let ready = rows + .iter() + .all(|r| matches!(r.state, b2::State::Ready { .. })); + return Ok(if ready { + ExitCode::SUCCESS + } else { + ExitCode::from(11) + }); + } + let (items, _, _) = corpus()?; + let index = sealed_index()?; + let path = need(&f, "logits")?; + let jsonl = std::fs::read_to_string(path).map_err(|e| format!("{path}: {e}"))?; + let k = need(&f, "k")?.parse().map_err(|e| format!("--k: {e}"))?; + match b2::teacher_receipt(&jsonl, need(&f, "teacher-sha")?, k, &items, &index) { + Ok(r) => { + println!( + "{}", + serde_json::to_string_pretty(&r).map_err(|e| e.to_string())? + ); + Ok(ExitCode::SUCCESS) + } + Err(e) => Err(e.join("\n")), + } + }; + match a.first().map(String::as_str) { + Some(s @ ("status" | "teacher-receipt")) => run(s).unwrap_or_else(|e| { + eprintln!("rex b2 {s}: {e}"); + ExitCode::from(1) + }), + _ => { + eprintln!("usage: rex b2 status|teacher-receipt ... (see the example docs)"); + ExitCode::from(2) + } + } +} + +/// `minisign -V` must pass on `path` under `pk`. +fn verify_signed(path: &str, pk: &str) -> Result<(), String> { + let ok = std::process::Command::new("minisign") + .args(["-Vqm", path, "-p", pk]) + .status() + .map_err(|e| format!("minisign: {e}"))? + .success(); + if ok { + Ok(()) + } else { + Err(format!("{path}: signature does not verify under {pk}")) + } +} + +/// One lane's scored rows on the sealed test split (first runs only). +fn test_rows(path: &str, pk: &str, cell: &str, arm: Arm) -> Result, String> { + verify_signed(path, pk)?; + let (items, prereg_sha, cv) = corpus()?; + let lines = std::fs::read_to_string(path).map_err(|e| format!("{path}: {e}"))?; + let root = std::path::Path::new(path) + .parent() + .map(std::path::Path::to_path_buf) + .unwrap_or_default(); + let expect = Expect { + prereg_sha: &prereg_sha, + corpus_version: &cv, + }; + let (rows, rejected) = collect( + &lines, + expect, + &ordered(&items, Split::Test, false), + |r| r.cell == cell && r.arm == arm && !r.rerun, + |p| std::fs::read_to_string(root.join(p)).ok(), + ); + if !rejected.is_empty() { + eprintln!("{path}: {} rejected rows", rejected.len()); + } + Ok(rows) +} + +fn opt_bool(f: &Flags, k: &str) -> Result, String> { + match f.get(k).map(String::as_str) { + None => Ok(None), + Some("holds") => Ok(Some(true)), + Some("fails") => Ok(Some(false)), + Some(v) => Err(format!("--{k} {v}: holds|fails")), + } +} + +fn opt_num(f: &Flags, k: &str) -> Result, String> { + f.get(k) + .map(|v| v.parse().map_err(|_| format!("--{k} {v}: not a number"))) + .transpose() +} + +fn challenge(f: &Flags) -> Result { + let file = need(f, "ledger")?; + let ledger: Vec = match std::fs::read_to_string(file) { + Ok(t) => t + .lines() + .filter(|l| !l.trim().is_empty()) + .map(|l| serde_json::from_str(l).map_err(|e| format!("{file}: {e}"))) + .collect::>()?, + Err(e) if e.kind() == std::io::ErrorKind::NotFound => Vec::new(), + Err(e) => return Err(format!("{file}: {e}")), + }; + let pin = |w: &str, p: &str, a: &str| -> Result { + Ok(champion::Pin { + weights_sha256: need(f, w)?.into(), + prompt_sha256: need(f, p)?.into(), + adapter_sha256: f.get(a).cloned(), + }) + }; + let initial = pin( + "initial-weights-sha", + "initial-prompt-sha", + "initial-adapter-sha", + )?; + let (pk, cell) = (need(f, "pubkey")?, need(f, "cell")?); + let ch = test_rows( + need(f, "challenger-receipts")?, + pk, + cell, + arm_of(need(f, "challenger-arm")?)?, + )?; + let cm = test_rows( + need(f, "champion-receipts")?, + pk, + cell, + arm_of(need(f, "champion-arm")?)?, + )?; + let e = champion::Evidence { + id: need(f, "id")?, + tier: need(f, "tier")?, + pin: pin("weights-sha", "prompt-sha", "adapter-sha")?, + test_version: need(f, "test-version")?, + challenger: &ch, + champion: &cm, + h1_holds: opt_bool(f, "h1")?, + h2_holds: opt_bool(f, "h2")?, + contamination_hits: opt_num(f, "contamination-hits")?, + p95_s: opt_num(f, "p95-s")?, + queue_budget_p95_s: opt_num(f, "budget-p95-s")?, + }; + let d = champion::decide(&ledger, &initial, &e); + let line = serde_json::to_string(&d).map_err(|e| e.to_string())?; + let mut out = std::fs::OpenOptions::new() + .create(true) + .append(true) + .open(file) + .map_err(|e| format!("{file}: {e}"))?; + std::io::Write::write_all(&mut out, format!("{line}\n").as_bytes()) + .map_err(|e| e.to_string())?; + println!( + "{}", + serde_json::to_string_pretty(&d).map_err(|e| e.to_string())? + ); + Ok(d.outcome) +} + +fn challenge_cmd(a: &[String]) -> ExitCode { + match flags(a).and_then(|f| challenge(&f)) { + Ok(champion::Outcome::Promoted) => ExitCode::SUCCESS, + Ok(champion::Outcome::Rejected) => ExitCode::from(10), + Ok(champion::Outcome::Refused) => ExitCode::from(11), + Err(e) => { + eprintln!("rex challenge: {e}"); + ExitCode::from(1) + } + } +} + +const ANDON_WORKLOAD: u8 = 13; + +fn workload_cmd(a: &[String]) -> ExitCode { + use aprender_review_experiment::workload::{profile, render}; + const USAGE: &str = "usage: rex workload ROWS --min-n N"; + let (Some(path), Ok(f)) = (a.first(), flags(a.get(1..).unwrap_or_default())) else { + eprintln!("{USAGE}"); + return ExitCode::from(2); + }; + let Some(Ok(min_n)) = f.get("min-n").map(|n| n.parse::()) else { + eprintln!("{USAGE}"); + return ExitCode::from(2); + }; + if min_n == 0 { + eprintln!("{USAGE}"); + return ExitCode::from(2); + } + let rows = match std::fs::read_to_string(path) { + Ok(t) => t, + Err(e) => { + eprintln!("rex workload: {path}: {e}"); + return ExitCode::from(1); + } + }; + let p = profile(&rows, min_n); + println!("{}", render(&p)); + if p.claim.is_none() { + eprintln!("rex workload: ANDON {}", p.refusals.join("; ")); + return ExitCode::from(ANDON_WORKLOAD); + } + ExitCode::SUCCESS +} + +fn sparse_logits_cmd(a: &[String]) -> ExitCode { + use aprender_review_experiment::sparse_logits::{capture, Header, Mode, DEFAULT_K, SCHEME}; + const USAGE: &str = "usage: rex sparse-logits --chat F --store DIR --model-sha M --tokenizer-sha T --apr-tag V --backend B --temperature X [--k K]"; + let header = |f: &Flags| -> Result { + let k = match f.get("k") { + Some(k) => k.parse::().map_err(|e| format!("--k: {e}"))?, + None => DEFAULT_K, + }; + Ok(Header { + schema: SCHEME.into(), + mode: Mode::Topk, + model_sha256: need(f, "model-sha")?.into(), + tokenizer_sha256: need(f, "tokenizer-sha")?.into(), + apr_tag: need(f, "apr-tag")?.into(), + backend: need(f, "backend")?.into(), + temperature_of_record: need(f, "temperature")? + .parse() + .map_err(|e| format!("--temperature: {e}"))?, + k, + n_tokens: 0, + }) + }; + let parsed = flags(a).and_then(|f| { + let h = header(&f)?; + Ok(( + h, + need(&f, "chat")?.to_string(), + need(&f, "store")?.to_string(), + )) + }); + let (h, chat, store) = match parsed { + Ok(p) => p, + Err(e) => { + eprintln!("rex sparse-logits: {e}\n{USAGE}"); + return ExitCode::from(2); + } + }; + match read_json::(&chat) + .and_then(|body| capture(std::path::Path::new(&store), &h, &body)) + { + Ok(c) => { + println!("{}", serde_json::to_string(&c).unwrap_or_default()); + ExitCode::SUCCESS + } + Err(e) => { + eprintln!("rex sparse-logits: {e}"); + ExitCode::from(1) + } + } +} + +/// The sealed-test index: the test manifest, plus its cluster sketches when present. +fn sealed_index() -> Result { + let manifest = std::fs::read_to_string(format!("{CORPUS_DIR}/test-manifest-v1.txt")) + .map_err(|e| e.to_string())?; + let sealed = parse_manifest(&manifest).ok_or("test manifest does not parse")?; + Ok( + match std::fs::read_to_string(format!("{CORPUS_DIR}/test-sketch-v1.txt")) { + Ok(t) => Index::new(&sealed) + .with_sketches(parse_sketches(&t).ok_or("test sketch file does not parse")?), + Err(_) => { + eprintln!( + "no test-sketch-v1.txt: exact-hash contamination only (no cluster check)" + ); + Index::new(&sealed) + } + }, + ) +} + +fn pool_admit_cmd(a: &[String]) -> ExitCode { + use aprender_review_experiment::pool::admit; + const USAGE: &str = "usage: rex pool-admit ROWS --pool F --quarantine Q"; + let (Some(rows_path), Ok(f)) = (a.first(), flags(a.get(1..).unwrap_or_default())) else { + eprintln!("{USAGE}"); + return ExitCode::from(2); + }; + let (Ok(pool), Ok(quarantine)) = (need(&f, "pool"), need(&f, "quarantine")) else { + eprintln!("{USAGE}"); + return ExitCode::from(2); + }; + let run = || -> Result { + for out in [pool, quarantine] { + if std::path::Path::new(out).exists() { + return Err(format!("{out} exists: a backfill never overwrites")); + } + } + let rows = std::fs::read_to_string(rows_path).map_err(|e| format!("{rows_path}: {e}"))?; + let adm = admit(&rows, &sealed_index()?); + let receipt = adm.receipt(); + println!( + "{}", + serde_json::to_string(&receipt).map_err(|e| e.to_string())? + ); + if receipt.unchecked > 0 { + eprintln!("rex pool-admit: ANDON sealed index is empty; nothing written"); + return Ok(ExitCode::from(12)); + } + let lines = |v: Vec<&str>| v.iter().map(|r| format!("{r}\n")).collect::(); + std::fs::write(pool, lines(adm.pool())).map_err(|e| format!("{pool}: {e}"))?; + std::fs::write(quarantine, lines(adm.quarantine())) + .map_err(|e| format!("{quarantine}: {e}"))?; + Ok(ExitCode::SUCCESS) + }; + run().unwrap_or_else(|e| { + eprintln!("rex pool-admit: {e}"); + ExitCode::from(1) + }) +} + +fn datacard_cmd(a: &[String]) -> ExitCode { + use aprender_review_experiment::datacard::{g16_decided, write_snapshot, CardMeta}; + let Ok(f) = flags(a) else { + eprintln!("usage: rex datacard --root DIR"); + return ExitCode::from(2); + }; + let Ok(root) = need(&f, "root") else { + eprintln!("usage: rex datacard --root DIR"); + return ExitCode::from(2); + }; + let run = || -> Result { + let dir = write_snapshot(std::path::Path::new(root), &CardMeta::default())?; + println!("{}", dir.display()); + let p = dir.join("weekly.json"); + let text = std::fs::read_to_string(&p).map_err(|e| format!("{}: {e}", p.display()))?; + let weekly: serde_json::Value = + serde_json::from_str(&text).map_err(|e| format!("{}: {e}", p.display()))?; + match g16_decided(&weekly) { + Ok(weeks) => { + for (week, g) in weeks { + println!("{week} g16={g}"); + } + Ok(ExitCode::SUCCESS) + } + Err(e) => { + eprintln!("rex datacard: ANDON G16 stays [U]: {e}"); + Ok(ExitCode::from(13)) + } + } + }; + run().unwrap_or_else(|e| { + eprintln!("rex datacard: {e}"); + ExitCode::from(1) + }) +} diff --git a/crates/aprender-review-experiment/src/admission.rs b/crates/aprender-review-experiment/src/admission.rs index bd82ed3bb8..f29e6c82ce 100644 --- a/crates/aprender-review-experiment/src/admission.rs +++ b/crates/aprender-review-experiment/src/admission.rs @@ -15,6 +15,10 @@ pub const SCHEME: &str = "rex-cell-admission-v1"; /// GPU with CPU only; a llama.cpp comparison is recorded under this name. pub const LLAMA_CPP: &str = "llama.cpp@d1d3c3396"; +/// The interim oracle of §4.1 rule 1 until llama.cpp is a declared input: the +/// released `apr parity` GPU-vs-CPU comparison (plus H1 in the analysis). +pub const APR_GPU_CPU: &str = "apr-parity-gpu-cpu"; + /// One §2.1 device cell. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub struct Cell { @@ -106,23 +110,19 @@ pub struct Summary { } fn row_errors(r: &Row, prereg_sha: &str) -> Vec { + let mut e = identity_errors(r); + e.extend(provenance_errors(r, prereg_sha)); + e.extend(status_errors(r)); + e +} + +/// The schema, and the cell against its §2.1 declaration. +fn identity_errors(r: &Row) -> Vec { let mut e = Vec::new(); let id = &r.cell; if r.schema != SCHEME { e.push(format!("{id}: schema {}", r.schema)); } - cell_errors(r, &mut e); - field_errors(r, &mut e); - if r.prereg_sha != prereg_sha { - e.push(format!("{id}: prereg_sha differs from the lock")); - } - status_errors(id, &r.status, &mut e); - e -} - -/// The row names a §2.1 cell, on that cell's declared host and backend. -fn cell_errors(r: &Row, e: &mut Vec) { - let id = &r.cell; match CELLS.iter().find(|c| c.cell == r.cell) { None => e.push(format!("{id}: not a §2.1 cell")), Some(c) if c.host != r.host || c.backend != r.backend => e.push(format!( @@ -131,34 +131,40 @@ fn cell_errors(r: &Row, e: &mut Vec) { )), Some(_) => {} } + e } -/// Provenance fields are present, and the two digests are sha256s. -fn field_errors(r: &Row, e: &mut Vec) { +/// The binary, the weights and the pre-registration the row was measured on. +fn provenance_errors(r: &Row, prereg_sha: &str) -> Vec { let id = &r.cell; - for (k, v) in [ + let missing = [ ("apr_tag", &r.apr_tag), ("model_id", &r.model_id), ("at", &r.at), - ] { - if v.trim().is_empty() || v == "unknown" { - e.push(format!("{id}: {k} missing")); - } - } - for (k, v) in [ + ] + .into_iter() + .filter(|(_, v)| v.trim().is_empty() || *v == "unknown") + .map(|(k, _)| format!("{id}: {k} missing")); + let not_sha = [ ("apr_sha256", &r.apr_sha256), ("weights_sha256", &r.weights_sha256), - ] { - if !is_hex64(v) { - e.push(format!("{id}: {k} is not a sha256")); - } + ] + .into_iter() + .filter(|(_, v)| !is_hex64(v)) + .map(|(k, _)| format!("{id}: {k} is not a sha256")); + let mut e: Vec = missing.chain(not_sha).collect(); + if r.prereg_sha != prereg_sha { + e.push(format!("{id}: prereg_sha differs from the lock")); } + e } -/// An Admitted row carries its parity evidence; a Refused row names its remover. -fn status_errors(id: &str, status: &Status, e: &mut Vec) { - match status { +/// What an Admitted or Refused status must carry. +fn status_errors(r: &Row) -> Vec { + let id = &r.cell; + match &r.status { Status::Admitted { parity: p } => { + let mut e = Vec::new(); if p.oracle.trim().is_empty() || p.threshold_basis.trim().is_empty() { e.push(format!("{id}: Admitted without oracle or threshold basis")); } @@ -171,11 +177,12 @@ fn status_errors(id: &str, status: &Status, e: &mut Vec) { p.cosine, p.threshold )); } + e } Status::Refused { removed_by } if removed_by.trim().is_empty() => { - e.push(format!("{id}: Refused without removed_by")); + vec![format!("{id}: Refused without removed_by")] } - _ => {} + _ => Vec::new(), } } @@ -225,6 +232,161 @@ pub fn check(lines: &str, prereg_sha: &str) -> Result> { } } +/// What a parity receipt must match before it admits a cell: the row's own +/// binary and weights, the declared threshold and its basis, and the evidence +/// floor (`min_positions` of `evidence/parity/thresholds.yaml`). +#[derive(Debug, Clone, Copy)] +pub struct Expect<'a> { + pub apr_sha256: &'a str, + pub weights_sha256: &'a str, + pub threshold: f64, + pub threshold_basis: &'a str, + pub min_positions: usize, +} + +/// What a receipt boils down to, whichever of the two shapes it was. +type ReceiptFields<'v> = (String, Vec>, Option<&'v str>, Option<&'v str>); + +/// Parse an `apr-parity-oracle/v1` receipt: oracle, per-position cosines and +/// the producer's weights sha (this shape carries no binary sha). +fn parse_oracle_v1_receipt<'v>(v: &'v serde_json::Value, e: &mut Vec) -> ReceiptFields<'v> { + if v["verdict"].as_str() != Some("GREEN") { + e.push(format!("verdict {} is not GREEN", v["verdict"])); + } + let per = v["per_position"].as_array().map_or(&[][..], Vec::as_slice); + ( + v["oracle"].as_str().unwrap_or_default().to_string(), + per.iter().map(|p| p["cosine"].as_f64()).collect(), + v["subject"]["producer"]["model_sha256"].as_str(), + None, + ) +} + +/// Parse an `apr-review-serve` parity receipt: exit/verdict/parity checks, +/// the comparator (normalized to `APR_GPU_CPU` for an `apr-cpu*` comparator), +/// weights sha and binary sha. +fn parse_review_serve_receipt<'v>( + v: &'v serde_json::Value, + e: &mut Vec, +) -> ReceiptFields<'v> { + if v["exit"].as_i64() != Some(0) { + e.push(format!("exit {} is not 0", v["exit"])); + } + if v["verdict"].as_str() != Some("pass") { + e.push(format!("verdict {} is not pass", v["verdict"])); + } + let p = &v["parity"]; + if p["failed"].as_u64() != Some(0) || p["parity"].as_bool() != Some(true) { + e.push(format!( + "parity failed {} (parity {})", + p["failed"], p["parity"] + )); + } + let comparator = v["comparator"].as_str().unwrap_or_default(); + let oracle = if comparator.starts_with("apr-cpu") { + APR_GPU_CPU.to_string() + } else { + comparator.to_string() + }; + let metrics = p["metrics"].as_array().map_or(&[][..], Vec::as_slice); + ( + oracle, + metrics + .iter() + .map(|m| m["cosine_similarity"].as_f64()) + .collect(), + v["weights_sha256"].as_str(), + Some(v["binary_sha256"].as_str().unwrap_or_default()), + ) +} + +/// The cross-checks common to both receipt shapes: declared oracle, weights +/// sha, binary sha (when the shape carries one), position count and the +/// row's own threshold declaration. +fn check_receipt_fields(fields: &ReceiptFields<'_>, x: &Expect<'_>, e: &mut Vec) { + let (oracle, cos, weights, binary) = fields; + if oracle != LLAMA_CPP && oracle != APR_GPU_CPU { + e.push(format!("oracle {oracle:?} is not a declared parity.oracle")); + } + if *weights != Some(x.weights_sha256) { + e.push(format!( + "weights sha {weights:?} is not the row's {}", + x.weights_sha256 + )); + } + if binary.is_some_and(|b| b != x.apr_sha256) { + e.push(format!( + "binary sha {binary:?} is not the row's {}", + x.apr_sha256 + )); + } + if cos.len() < x.min_positions { + e.push(format!( + "{} positions < min_positions {}", + cos.len(), + x.min_positions + )); + } + if !x.threshold.is_finite() { + e.push(format!("threshold {} is not finite", x.threshold)); + } + if x.threshold_basis.trim().is_empty() { + e.push("no threshold basis declared".into()); + } +} + +/// The minimum cosine across positions, recording a "no numeric cosine" +/// error for each position that did not carry one (JSON has no NaN or +/// infinity, so a present cosine is always finite). +fn min_cosine(cos: &[Option], e: &mut Vec) -> f64 { + let mut min = f64::INFINITY; + for (i, c) in cos.iter().enumerate() { + match c { + Some(c) => min = min.min(*c), + None => e.push(format!("position {i}: no numeric cosine")), + } + } + min +} + +/// Derive an `Admitted` parity block from the bytes of a parity receipt. +/// +/// # Errors +/// Every reason the receipt cannot admit the cell. +pub fn parity_from_receipt(receipt: &[u8], x: &Expect<'_>) -> Result> { + use serde_json::Value; + let v: Value = + serde_json::from_slice(receipt).map_err(|e| vec![format!("receipt is not JSON: {e}")])?; + let mut e = Vec::new(); + // (oracle, cosines, weights sha, binary sha) per receipt shape + let fields: ReceiptFields<'_> = if v["schema"].as_str() == Some("apr-parity-oracle/v1") { + parse_oracle_v1_receipt(&v, &mut e) + } else if v.get("parity").is_some() && v.get("comparator").is_some() { + parse_review_serve_receipt(&v, &mut e) + } else { + return Err(vec![ + "not a parity receipt (apr-parity-oracle/v1 or apr-review-serve parity)".into(), + ]); + }; + check_receipt_fields(&fields, x, &mut e); + let (oracle, cos, ..) = &fields; + let min = min_cosine(cos, &mut e); + if min < x.threshold { + e.push(format!("min cosine {min} below threshold {}", x.threshold)); + } + if e.is_empty() { + Ok(Parity { + oracle: oracle.clone(), + cosine: min, + threshold: x.threshold, + threshold_basis: x.threshold_basis.to_string(), + receipt_sha256: crate::corpus::sha256_hex(receipt), + }) + } else { + Err(e) + } +} + #[cfg(test)] #[path = "admission_tests.rs"] mod tests; diff --git a/crates/aprender-review-experiment/src/admission_tests.rs b/crates/aprender-review-experiment/src/admission_tests.rs index 4be7a2f1a2..33a94946a8 100644 --- a/crates/aprender-review-experiment/src/admission_tests.rs +++ b/crates/aprender-review-experiment/src/admission_tests.rs @@ -1,3 +1,6 @@ +// `json!` expands to an `unwrap` of an infallible `to_value`. +#![allow(clippy::disallowed_methods)] + use super::*; const PREREG: &str = "ef51087dc79bab0ad160e8a14f5b13e2ea43986b30c05dafc63c84c2dc21cdc0"; @@ -152,3 +155,185 @@ fn identity_and_cell_declaration_are_enforced() { } assert!(check("{not json}\n", PREREG).is_err()); } + +/// A `apr-review-serve parity` receipt (the declared C4 oneshot's shape) with +/// `n` positions; position `low_at` carries cosine `low`, the rest 0.9999. +fn serve_receipt(n: usize, low_at: usize, low: f64) -> serde_json::Value { + let metrics: Vec<_> = (0..n) + .map(|p| { + let c = if p == low_at { low } else { 0.9999 }; + serde_json::json!({"position": p, "cosine_similarity": c, "verdict": "Pass"}) + }) + .collect(); + serde_json::json!({ + "host": "gx10-a5b5", "apr_tag": "v0.69.3", + "binary_sha256": h('a'), "weights_sha256": h('b'), + "comparator": "apr-cpu (apr parity: GPU vs CPU; no llama.cpp comparator in this apr, aprender#3576)", + "exit": 0, "verdict": "pass", + "parity": {"tokens": n, "passed": n, "failed": 0, "parity": true, "metrics": metrics} + }) +} + +/// An `apr-parity-oracle/v1` receipt (`apr parity-oracle`, #4444) with `n` +/// positions; its top-level `cosine` claims 0.9999 whatever the positions say. +fn oracle_receipt(n: usize, low_at: usize, low: f64) -> serde_json::Value { + let per: Vec<_> = (0..n) + .map(|p| { + let c = if p == low_at { low } else { 0.9999 }; + serde_json::json!({"pos": p, "cosine": c}) + }) + .collect(); + serde_json::json!({ + "schema": "apr-parity-oracle/v1", "oracle": LLAMA_CPP, "verdict": "GREEN", + "cosine": 0.9999, "threshold": 0.5, "threshold_basis": "self-declared", + "n_positions": n, + "subject": {"producer": {"model_sha256": h('b')}}, + "per_position": per + }) +} + +const BASIS: &str = "evidence/parity/thresholds.yaml default.min_cosine"; + +fn expect() -> Expect<'static> { + Expect { + apr_sha256: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + weights_sha256: "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + threshold: 0.98, + threshold_basis: BASIS, + min_positions: 64, + } +} + +fn bytes(v: &serde_json::Value) -> Vec { + serde_json::to_vec_pretty(v).expect("serialize receipt") +} + +/// FALSIFY-RCA-004: an Admitted parity block is derived from the receipt — +/// the minimum cosine over its positions (never a summary field), the sha256 +/// of its bytes, the oracle it measured against, and the declared threshold +/// and basis (never the receipt's own). +#[test] +fn falsify_rca_004_admitted_is_derived_from_the_receipt() { + let b = bytes(&serve_receipt(64, 17, 0.985)); + let p = parity_from_receipt(&b, &expect()).expect("admits"); + assert_eq!(p.oracle, APR_GPU_CPU); + assert_eq!(p.cosine, 0.985); + assert_eq!(p.threshold, 0.98); + assert_eq!(p.threshold_basis, BASIS); + assert_eq!(p.receipt_sha256, crate::corpus::sha256_hex(&b)); + + let o = bytes(&oracle_receipt(78, 40, 0.981)); + let q = parity_from_receipt(&o, &expect()).expect("admits"); + assert_eq!(q.oracle, LLAMA_CPP); + assert_eq!(q.cosine, 0.981, "the summary `cosine` must not be read"); + assert_eq!( + q.threshold, 0.98, + "the receipt's own threshold must not be read" + ); + assert_eq!(q.threshold_basis, BASIS); + + // exactly min_positions and exactly the threshold admit + assert!(parity_from_receipt(&bytes(&serve_receipt(64, 0, 0.98)), &expect()).is_ok()); + + // the derived block is a well-formed Admitted row + let mut rows: Vec = CELLS.iter().map(|c| row(c, not_run())).collect(); + rows[3] = row(&CELLS[3], Status::Admitted { parity: p }); + let s = check(&file(&rows), PREREG).expect("admissible"); + assert_eq!((s.admitted, s.s7), (vec!["C4".to_string()], false)); +} + +/// FALSIFY-RCA-005: a receipt with too few positions, a failing or +/// non-zero-exit run, a position below threshold, a missing or non-finite +/// cosine, or no declared basis does not admit, and says why. +#[test] +fn falsify_rca_005_a_short_or_failing_receipt_does_not_admit() { + let refuse = |v: &serde_json::Value, x: &Expect<'_>, why: &str| { + let e = parity_from_receipt(&bytes(v), x).expect_err(why); + assert!(e.iter().any(|m| m.contains(why)), "{why}: {e:?}"); + }; + let x = expect(); + // the declared gx10 oneshot measured 7 positions: under the 64 floor + refuse(&serve_receipt(7, 0, 0.9997), &x, "min_positions"); + refuse(&serve_receipt(63, 0, 0.99), &x, "min_positions"); + refuse(&oracle_receipt(63, 0, 0.99), &x, "min_positions"); + refuse(&serve_receipt(64, 5, 0.979_999), &x, "below threshold"); + refuse(&oracle_receipt(78, 5, 0.97), &x, "below threshold"); + let mut v = serve_receipt(64, 0, 0.99); + v["exit"] = 1.into(); + refuse(&v, &x, "exit"); + let mut v = serve_receipt(64, 0, 0.99); + v["verdict"] = "fail".into(); + refuse(&v, &x, "verdict"); + let mut v = oracle_receipt(64, 0, 0.99); + v["verdict"] = "RED".into(); + refuse(&v, &x, "verdict"); + let mut v = serve_receipt(64, 0, 0.99); + v["parity"]["failed"] = 1.into(); + refuse(&v, &x, "failed"); + let mut v = serve_receipt(64, 0, 0.99); + v["parity"]["parity"] = false.into(); + refuse(&v, &x, "failed"); + let mut v = serve_receipt(64, 0, 0.99); + v["parity"]["metrics"][3] = serde_json::json!({"position": 3}); + refuse(&v, &x, "cosine"); + let mut v = oracle_receipt(64, 0, 0.99); + v["per_position"][3]["cosine"] = "NaN".into(); + refuse(&v, &x, "cosine"); + let mut v = oracle_receipt(64, 0, 0.99); + v["oracle"] = "ollama".into(); + refuse(&v, &x, "oracle"); + let mut v = serve_receipt(64, 0, 0.99); + v["comparator"] = "hf-transformers".into(); + refuse(&v, &x, "oracle"); + refuse( + &serve_receipt(64, 0, 0.99), + &Expect { + threshold_basis: " ", + ..x + }, + "basis", + ); + refuse( + &serve_receipt(64, 0, 0.99), + &Expect { + threshold: f64::NAN, + ..x + }, + "threshold", + ); + assert!(parity_from_receipt(b"{not json", &x).is_err()); + assert!(parity_from_receipt(b"{\"schema\": \"other\"}", &x).is_err()); + // a half-shaped receipt (parity block, no comparator, or the reverse) is + // not a parity receipt at all, not an oracle mismatch + for key in ["comparator", "parity"] { + let mut v = serve_receipt(64, 0, 0.99); + v.as_object_mut().expect("object").remove(key); + refuse(&v, &x, "not a parity receipt"); + } +} + +/// FALSIFY-RCA-006: the receipt must be of the row's own binary and weights — +/// a passing receipt of another build or another model does not admit. +#[test] +fn falsify_rca_006_the_receipt_must_be_the_rows_binary_and_weights() { + let x = expect(); + let wrong = "c".repeat(64); + let mut v = serve_receipt(64, 0, 0.99); + v["binary_sha256"] = wrong.clone().into(); + let e = parity_from_receipt(&bytes(&v), &x).expect_err("binary"); + assert!(e.iter().any(|m| m.contains("binary")), "{e:?}"); + let mut v = serve_receipt(64, 0, 0.99); + v["weights_sha256"] = wrong.clone().into(); + let e = parity_from_receipt(&bytes(&v), &x).expect_err("weights"); + assert!(e.iter().any(|m| m.contains("weights")), "{e:?}"); + let mut v = oracle_receipt(64, 0, 0.99); + v["subject"]["producer"]["model_sha256"] = wrong.into(); + let e = parity_from_receipt(&bytes(&v), &x).expect_err("weights"); + assert!(e.iter().any(|m| m.contains("weights")), "{e:?}"); + let mut v = serve_receipt(64, 0, 0.99); + v.as_object_mut().expect("object").remove("binary_sha256"); + assert!( + parity_from_receipt(&bytes(&v), &x).is_err(), + "a missing sha must not match" + ); +} diff --git a/crates/aprender-review-experiment/src/b2.rs b/crates/aprender-review-experiment/src/b2.rs new file mode 100644 index 0000000000..427bca374d --- /dev/null +++ b/crates/aprender-review-experiment/src/b2.rs @@ -0,0 +1,251 @@ +//! REX-12 B2 loop (§5.3 B2a..B2d), `review-b2-loop-v1`. +//! +//! - **Row status.** Each B2 tier waits on one `apr` verb accepting qwen35. +//! A tier is `Ready` only on a positive acceptance receipt for its verb. A +//! verb with a `refusal-receipt-v1` row in the refusal ledger reads +//! `VerbRefused{removed_by}`. A verb with neither is `Unledgered`: nobody +//! measured it, so it is NotRun too. Absence of a refusal is not +//! acceptance; the ledger is a bucket, not a census. +//! - **Teacher dataset.** The 27B teacher writes top-k logits at m = 1 over +//! the train pool (every non-test item). [`teacher_receipt`] refuses the +//! dataset unless every row is a known non-test item, appears once, carries +//! exactly k finite logits per position, and the file text contains no +//! sealed diff sha or hunk. The receipt carries sha, count and 0 test hits. + +use std::collections::{BTreeMap, BTreeSet}; + +use serde::{Deserialize, Serialize}; + +use crate::contamination::Index; +use crate::corpus::{sha256_hex, Item, Split}; + +pub const SCHEME: &str = "review-b2-loop-v1"; +pub const LEDGER_SCHEMA: &str = "apr-refusal-ledger/v1"; + +/// The §5.3 B2 tiers and the verb each waits on. +pub const TIERS: [(&str, &str); 4] = [ + ("B2a", "distill"), + ("B2b", "finetune"), + ("B2c", "merge"), + ("B2d", "quantize"), +]; + +/// One `refusal-receipt-v1` row. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Refusal { + pub verb: String, + pub reason: String, + pub exit_code: i32, + pub removed_by: String, +} + +/// `evidence/verbs/refusals.json`. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct Ledger { + pub schema: String, + #[serde(default)] + pub note: String, + pub refusals: Vec, +} + +/// `refusal-receipt-v1`: `v.`, `never` or `unscheduled`. +#[must_use] +pub fn valid_removed_by(s: &str) -> bool { + if matches!(s, "never" | "unscheduled") { + return true; + } + let Some(v) = s.strip_prefix('v') else { + return false; + }; + let parts: Vec<&str> = v.split('.').collect(); + parts.len() == 2 + && parts + .iter() + .all(|p| !p.is_empty() && p.bytes().all(|b| b.is_ascii_digit())) +} + +/// Parse and validate a refusal ledger; one bad row rejects the file. +pub fn parse_ledger(text: &str) -> Result { + let l: Ledger = serde_json::from_str(text).map_err(|e| e.to_string())?; + if l.schema != LEDGER_SCHEMA { + return Err(format!("schema {:?} is not {LEDGER_SCHEMA}", l.schema)); + } + let mut seen = BTreeSet::new(); + for r in &l.refusals { + if !valid_removed_by(&r.removed_by) { + return Err(format!( + "{}: removed_by {:?} is not v., never or unscheduled", + r.verb, r.removed_by + )); + } + if r.reason.trim().is_empty() || r.exit_code == 0 { + return Err(format!( + "{}: a refusal needs a reason and a non-zero exit", + r.verb + )); + } + if !seen.insert(r.verb.as_str()) { + return Err(format!("{}: two refusal rows", r.verb)); + } + } + Ok(l) +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "state", rename_all = "snake_case")] +pub enum State { + /// The verb accepted qwen35: the tier's challengers go to §5.4. + Ready { + acceptance: String, + }, + VerbRefused { + removed_by: String, + exit_code: i32, + }, + /// No refusal row and no acceptance: unmeasured, so NotRun. + Unledgered, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct Row { + pub schema: String, + pub tier: String, + pub verb: String, + #[serde(flatten)] + pub state: State, +} + +/// The B2 rows. `accepted` maps a verb to its acceptance receipt (a sha or a +/// path); a verb that is both accepted and refused is an error, since one of +/// the two receipts is stale. +pub fn status(ledger: &Ledger, accepted: &BTreeMap) -> Result, String> { + let refused: BTreeMap<&str, &Refusal> = ledger + .refusals + .iter() + .map(|r| (r.verb.as_str(), r)) + .collect(); + TIERS + .iter() + .map(|&(tier, verb)| { + let state = match (accepted.get(verb), refused.get(verb)) { + (Some(_), Some(_)) => { + return Err(format!( + "{verb}: both accepted and refused; one receipt is stale" + )) + } + (Some(a), None) => State::Ready { + acceptance: a.clone(), + }, + (None, Some(r)) => State::VerbRefused { + removed_by: r.removed_by.clone(), + exit_code: r.exit_code, + }, + (None, None) => State::Unledgered, + }; + Ok(Row { + schema: SCHEME.into(), + tier: tier.into(), + verb: verb.into(), + state, + }) + }) + .collect() +} + +/// One teacher row: an item's top-k `(token, logit)` pairs per position. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct TeacherRow { + pub item_id: String, + pub top_k: Vec>, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct TeacherReceipt { + pub schema: String, + pub teacher_weights_sha256: String, + pub k: usize, + /// Samples per item (§5.3 B2a: m = 1). + pub m: usize, + pub count: usize, + pub sha256: String, + pub sealed_items_indexed: usize, + pub test_hits: usize, +} + +/// Receipt a teacher-logit JSONL, or list every reason it is refused. +pub fn teacher_receipt( + jsonl: &str, + teacher_weights_sha256: &str, + k: usize, + items: &[Item], + sealed: &Index, +) -> Result> { + let mut e = Vec::new(); + if sealed.is_empty() { + e.push("no sealed manifest indexed: 0 test hits would prove nothing".into()); + } + if k == 0 { + e.push("k = 0".into()); + } + let split: BTreeMap<&str, Split> = items.iter().map(|i| (i.id.as_str(), i.split)).collect(); + let mut seen = BTreeSet::new(); + for (n, line) in jsonl + .lines() + .enumerate() + .filter(|(_, l)| !l.trim().is_empty()) + { + let row: TeacherRow = match serde_json::from_str(line) { + Ok(r) => r, + Err(x) => { + e.push(format!("line {}: {x}", n + 1)); + continue; + } + }; + let id = row.item_id.as_str(); + match split.get(id) { + None => e.push(format!("{id}: not a corpus item")), + Some(Split::Test) => e.push(format!("{id}: a sealed test item (R-2)")), + Some(Split::Dev) => {} + } + if !seen.insert(row.item_id.clone()) { + e.push(format!("{id}: more than one row (m = 1)")); + } + if row.top_k.is_empty() { + e.push(format!("{id}: no positions")); + } + if let Some(p) = row + .top_k + .iter() + .position(|pos| pos.len() != k || pos.iter().any(|(_, x)| !x.is_finite())) + { + e.push(format!("{id}: position {p} is not {k} finite logits")); + } + } + if seen.is_empty() { + e.push("no teacher rows".into()); + } + let hits = sealed.scan(jsonl, "teacher"); + if !hits.is_empty() { + e.push(format!("{} sealed test leak(s): {hits:?}", hits.len())); + } + if !e.is_empty() { + return Err(e); + } + Ok(TeacherReceipt { + schema: SCHEME.into(), + teacher_weights_sha256: teacher_weights_sha256.into(), + k, + m: 1, + count: seen.len(), + sha256: sha256_hex(jsonl.as_bytes()), + sealed_items_indexed: sealed.len(), + test_hits: 0, + }) +} + +#[cfg(test)] +#[path = "b2_tests.rs"] +mod tests; diff --git a/crates/aprender-review-experiment/src/b2_tests.rs b/crates/aprender-review-experiment/src/b2_tests.rs new file mode 100644 index 0000000000..37fd589381 --- /dev/null +++ b/crates/aprender-review-experiment/src/b2_tests.rs @@ -0,0 +1,146 @@ +use super::*; +use crate::corpus::{hunk_fingerprints, Class, Sealed}; + +const IN_TREE: &str = include_str!("../../../evidence/verbs/refusals.json"); + +fn ledger(rows: &str) -> String { + format!(r#"{{"schema":"apr-refusal-ledger/v1","refusals":[{rows}]}}"#) +} + +fn refusal(verb: &str, removed_by: &str) -> String { + format!( + r#"{{"verb":"{verb}","reason":"refuses qwen35","exit_code":8,"removed_by":"{removed_by}"}}"# + ) +} + +#[test] +fn falsify_rxb_001_no_b2_row_is_ready_without_an_acceptance_receipt() { + // The in-tree ledger at HEAD: every B2 row is NotRun, none silently Ready. + let l = parse_ledger(IN_TREE).expect("in-tree ledger parses"); + let rows = status(&l, &BTreeMap::new()).expect("status"); + assert_eq!(rows.len(), 4); + assert!(rows.iter().all(|r| !matches!(r.state, State::Ready { .. }))); + + let l = parse_ledger(&ledger(&refusal("distill", "v0.75"))).expect("parses"); + let acc = BTreeMap::from([("finetune".to_string(), "sha-ft".to_string())]); + let rows = status(&l, &acc).expect("status"); + let by: BTreeMap<&str, &State> = rows.iter().map(|r| (r.tier.as_str(), &r.state)).collect(); + assert_eq!( + by["B2a"], + &State::VerbRefused { + removed_by: "v0.75".into(), + exit_code: 8 + } + ); + assert_eq!( + by["B2b"], + &State::Ready { + acceptance: "sha-ft".into() + } + ); + assert_eq!( + by["B2c"], + &State::Unledgered, + "no refusal row is not acceptance" + ); + + let both = BTreeMap::from([("distill".to_string(), "sha".to_string())]); + assert!(status(&l, &both).is_err(), "accepted and refused at once"); +} + +#[test] +fn falsify_rxb_002_the_ledger_is_refusal_receipt_v1_or_rejected() { + for good in ["v0.75", "v1.0", "never", "unscheduled"] { + assert!(valid_removed_by(good), "{good}"); + } + for bad in [ + "tbd", "soon", "0.70", "v0.70.1", "v0.", "v.7", "vx.y", "", "9f3c2a1", + ] { + assert!(!valid_removed_by(bad), "{bad}"); + assert!( + parse_ledger(&ledger(&refusal("merge", bad))).is_err(), + "{bad}" + ); + } + let dup = format!( + "{},{}", + refusal("merge", "never"), + refusal("merge", "v0.75") + ); + assert!( + parse_ledger(&ledger(&dup)).is_err(), + "two rows for one verb" + ); + assert!(parse_ledger(&ledger(&refusal("merge", "v0.75")).replace(":8", ":0")).is_err()); + assert!(parse_ledger(&ledger("").replace("ledger/v1", "ledger/v2")).is_err()); +} + +fn diff(body: &str) -> String { + format!("--- a/src/a.rs\n+++ b/src/a.rs\n@@ -10,5 +10,5 @@\n fn f(x: u32) -> bool {{\n{body} }}\n \n") +} + +fn fixture() -> (Vec, Index, String) { + let mk = |id: &str, split: Split, d: &str| { + let mut i = Item::new(id.into(), Class::R, "src".into(), d); + i.split = split; + i + }; + let test_diff = diff("- z * 2\n+ z * 3\n"); + let items = vec![ + mk("D01", Split::Dev, &diff("- a\n+ b\n")), + mk("D02", Split::Dev, &diff("- c\n+ d\n")), + mk("T01", Split::Test, &test_diff), + ]; + let ix = Index::new(&[Sealed { + id: "T01".into(), + diff_sha256: sha256_hex(test_diff.as_bytes()), + hunks: hunk_fingerprints(&test_diff), + }]); + (items, ix, test_diff) +} + +fn row(id: &str, k: usize) -> String { + let pos: Vec<(u32, f32)> = (0..k).map(|t| (t as u32, -(t as f32))).collect(); + serde_json::to_string(&TeacherRow { + item_id: id.into(), + top_k: vec![pos.clone(), pos], + }) + .expect("serializes") +} + +#[test] +fn falsify_rxb_003_the_teacher_dataset_is_receipted_with_zero_test_hashes() { + let (items, ix, test_diff) = fixture(); + let good = format!("{}\n{}\n", row("D01", 4), row("D02", 4)); + let r = teacher_receipt(&good, "w27b", 4, &items, &ix).expect("receipted"); + assert_eq!((r.count, r.m, r.k, r.test_hits), (2, 1, 4, 0)); + assert_eq!(r.sha256, sha256_hex(good.as_bytes())); + assert_eq!(r.sealed_items_indexed, 1); + + let refused = |text: &str, why: &str| { + let e = teacher_receipt(text, "w27b", 4, &items, &ix).expect_err(why); + assert!(e.iter().any(|x| x.contains(why)), "{why}: {e:?}"); + }; + refused( + &format!("{good}{}\n", row("T01", 4)), + "T01: a sealed test item", + ); + refused(&format!("{good}{}\n", row("D01", 4)), "more than one row"); + refused(&format!("{good}{}\n", row("X9", 4)), "not a corpus item"); + refused(&format!("{}\n", row("D01", 3)), "not 4 finite logits"); + refused(&row("D01", 4).replace("-1.0", "NaN"), "line 1"); + // A sealed diff carried in any string field is a hit. + let smuggled = format!( + "{good}{}\n", + format!( + r#"{{"item_id":"D02","top_k":[],"note":{}}}"#, + serde_json::to_string(&test_diff).expect("serializes") + ) + ); + refused(&smuggled, "sealed test leak"); + let leak = format!(r#"{good}"{}""#, sha256_hex(test_diff.as_bytes())); + refused(&leak, "sealed test leak"); + refused("", "no teacher rows"); + let e = teacher_receipt(&good, "w27b", 4, &items, &Index::default()).expect_err("empty"); + assert!(e[0].contains("prove nothing"), "{e:?}"); +} diff --git a/crates/aprender-review-experiment/src/champion.rs b/crates/aprender-review-experiment/src/champion.rs new file mode 100644 index 0000000000..432cea6e72 --- /dev/null +++ b/crates/aprender-review-experiment/src/champion.rs @@ -0,0 +1,217 @@ +//! REX-11 champion/challenger promotion (§5.4), `review-champion-challenger-v1`. +//! +//! A challenger (any §5.3 candidate: a prompt version, retrieval few-shot, a +//! B2 checkpoint) replaces the champion only if all five §5.4 gates hold on +//! the current sealed test version: +//! 1. per-item correctness improves: McNemar exact, paired, one-sided, α; +//! 2. the precision point estimate is not lower than the champion's; +//! 3. H1 (invariance) and H2 (determinism) hold on the primary cell; +//! 4. the contamination contract is green: 0 test hashes in its training data; +//! 5. its p95 on the primary cell meets the §4 queue budget. +//! +//! Every gate fails closed: an unmeasured gate is a failed gate. Each decision +//! consumes one of the test version's [`EVALUATIONS_PER_VERSION`] evaluations +//! (§2.2); once they are spent the next challenger is **refused**, consumes +//! nothing, and the test set must rotate. A challenger that never ran is +//! refused too (R-6): it is not an evaluation. Demotion (§5.4): two +//! consecutive gold-labelled escapes where the champion said PASS and a voter +//! said FAIL trigger an HRQ and re-evaluation. + +use serde::{Deserialize, Serialize}; + +use crate::score::{correctness_pairs, score, Scored}; +use crate::stats::{discordant, mcnemar_exact_one_sided, ALPHA}; + +pub const SCHEME: &str = "review-champion-challenger-v1"; +/// §2.2 `[A]`: promotion decisions a sealed test version supports. +pub const EVALUATIONS_PER_VERSION: usize = 20; + +/// What serves the lane: promotion is a forjar pin bump of these shas. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Pin { + pub weights_sha256: String, + pub prompt_sha256: String, + pub adapter_sha256: Option, +} + +/// Everything §5.4 reads about one challenger. `None` is "not measured". +#[derive(Debug, Clone)] +pub struct Evidence<'a> { + pub id: &'a str, + /// The §5.3 tier (B1a, B1b, B2a…). + pub tier: &'a str, + pub pin: Pin, + pub test_version: &'a str, + pub challenger: &'a [Scored], + pub champion: &'a [Scored], + pub h1_holds: Option, + pub h2_holds: Option, + pub contamination_hits: Option, + pub p95_s: Option, + /// §4 step 5: the slowest voting lane's measured p95 on the same items. + pub queue_budget_p95_s: Option, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "lowercase")] +pub enum Outcome { + Promoted, + Rejected, + /// Not an evaluation: nothing consumed. + Refused, +} + +/// One ledger row. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Decision { + pub schema: String, + pub test_version: String, + pub challenger: String, + pub tier: String, + pub outcome: Outcome, + /// The test version's evaluations used after this row. + pub evaluations_used: usize, + pub champion_before: Pin, + pub pin: Pin, + pub mcnemar_b: u64, + pub mcnemar_c: u64, + pub mcnemar_p: Option, + pub precision_champion: Option, + pub precision_challenger: Option, + pub precision_delta: Option, + pub reasons: Vec, +} + +/// The serving champion: the pin of the last promotion, else the initial one. +#[must_use] +pub fn champion<'a>(ledger: &'a [Decision], initial: &'a Pin) -> &'a Pin { + ledger + .iter() + .rev() + .find(|d| d.outcome == Outcome::Promoted) + .map_or(initial, |d| &d.pin) +} + +/// Evaluations the ledger has spent on `test_version`. +#[must_use] +pub fn evaluations_used(ledger: &[Decision], test_version: &str) -> usize { + ledger + .iter() + .filter(|d| d.test_version == test_version && d.outcome != Outcome::Refused) + .count() +} + +fn gate(reasons: &mut Vec, ok: bool, why: impl FnOnce() -> String) { + if !ok { + reasons.push(why()); + } +} + +/// Decide one challenger against the ledger's champion. Pure: the caller +/// appends the returned row. +#[must_use] +pub fn decide(ledger: &[Decision], initial: &Pin, e: &Evidence<'_>) -> Decision { + let used = evaluations_used(ledger, e.test_version); + let champ = champion(ledger, initial).clone(); + let (sc, sh) = (score(e.challenger), score(e.champion)); + let (b, c) = discordant(&correctness_pairs(e.challenger, e.champion)); + let mut d = Decision { + schema: SCHEME.into(), + test_version: e.test_version.into(), + challenger: e.id.into(), + tier: e.tier.into(), + outcome: Outcome::Refused, + evaluations_used: used, + champion_before: champ.clone(), + pin: e.pin.clone(), + mcnemar_b: b, + mcnemar_c: c, + mcnemar_p: None, + precision_champion: sh.precision.value(), + precision_challenger: sc.precision.value(), + precision_delta: sc + .precision + .value() + .zip(sh.precision.value()) + .map(|(a, b)| a - b), + reasons: Vec::new(), + }; + let ran = e.challenger.iter().any(|s| s.verdict.executed()); + let refusals = [ + (used >= EVALUATIONS_PER_VERSION).then(|| { + format!("test version {} has spent its {EVALUATIONS_PER_VERSION} evaluations: rotate it (§2.2)", e.test_version) + }), + (!ran).then(|| "the challenger has no executed row on the test split (R-6)".to_string()), + e.champion.is_empty().then(|| "no champion rows on this test version".to_string()), + (e.pin == champ).then(|| "the challenger is the champion".to_string()), + ]; + d.reasons = refusals.into_iter().flatten().collect(); + if !d.reasons.is_empty() { + return d; + } + let p = mcnemar_exact_one_sided(b, c); + d.mcnemar_p = Some(p); + d.evaluations_used = used + 1; + let r = &mut d.reasons; + gate(r, p < ALPHA, || { + format!("1: McNemar one-sided p = {p:.4} (b={b}, c={c}) is not < {ALPHA}") + }); + gate(r, d.precision_delta.is_some_and(|x| x >= 0.0), || { + format!( + "2: precision {:?} is lower than the champion's {:?}, or unmeasured", + d.precision_challenger, d.precision_champion + ) + }); + gate(r, e.h1_holds == Some(true), || { + format!("3: H1 invariance {:?}", e.h1_holds) + }); + gate(r, e.h2_holds == Some(true), || { + format!("3: H2 determinism {:?}", e.h2_holds) + }); + gate(r, e.contamination_hits == Some(0), || { + format!( + "4: contamination hits {:?} (must be 0)", + e.contamination_hits + ) + }); + let fits = matches!((e.p95_s, e.queue_budget_p95_s), + (Some(p), Some(q)) if p.is_finite() && q.is_finite() && p <= q); + gate(r, fits, || { + format!( + "5: p95 {:?} s vs queue budget {:?} s", + e.p95_s, e.queue_budget_p95_s + ) + }); + d.outcome = if d.reasons.is_empty() { + Outcome::Promoted + } else { + Outcome::Rejected + }; + d +} + +/// A gold-labelled escape joined back to the PR that let it through (§5.2). +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct Escape { + pub pr: String, + pub gold: bool, + pub champion_passed: bool, + pub a_voter_failed: bool, +} + +/// §5.4 demotion: the first pair of consecutive gold escapes where the +/// champion said PASS and a voter said FAIL, if any. Silver rows are not +/// evidence and neither break nor extend a run. +#[must_use] +pub fn demotion(escapes: &[Escape]) -> Option<(String, String)> { + let gold: Vec<&Escape> = escapes.iter().filter(|e| e.gold).collect(); + gold.windows(2) + .find(|w| w.iter().all(|e| e.champion_passed && e.a_voter_failed)) + .map(|w| (w[0].pr.clone(), w[1].pr.clone())) +} + +#[cfg(test)] +#[path = "champion_tests.rs"] +mod tests; diff --git a/crates/aprender-review-experiment/src/champion_tests.rs b/crates/aprender-review-experiment/src/champion_tests.rs new file mode 100644 index 0000000000..bb4e37efd5 --- /dev/null +++ b/crates/aprender-review-experiment/src/champion_tests.rs @@ -0,0 +1,241 @@ +use super::*; +use crate::receipt::{NotRun, Verdict}; + +/// 60 test items: 0..30 defects, 30..60 good. +fn rows(verdict: impl Fn(usize) -> Verdict) -> Vec { + (0..60) + .map(|i| { + let v = verdict(i); + Scored { + id: format!("t{i:02}"), + class: if i < 30 { + crate::corpus::Class::P + } else { + crate::corpus::Class::G + }, + defect: i < 30, + verdict: v, + localized: false, + output_sha: v.executed().then(|| format!("o{i}")), + } + }) + .collect() +} + +/// The right answer on item `i`, or the wrong one. +fn answer(i: usize, right: bool) -> Verdict { + if (i < 30) == right { + Verdict::Fail + } else { + Verdict::Pass + } +} + +fn pin(w: &str) -> Pin { + Pin { + weights_sha256: w.into(), + prompt_sha256: "p0".into(), + adapter_sha256: None, + } +} + +/// Champion: flags defects 0..10 only, never a good item (40 correct, precision 1). +fn champ() -> Vec { + rows(|i| answer(i, !(10..30).contains(&i))) +} + +/// Challenger: flags every defect, no good item (60 correct; b = 20, c = 0). +fn better() -> Vec { + rows(|i| answer(i, true)) +} + +fn evidence<'a>(ch: &'a [Scored], cm: &'a [Scored]) -> Evidence<'a> { + Evidence { + id: "prompt-v2", + tier: "B1a", + pin: Pin { + prompt_sha256: "p2".into(), + ..pin("w0") + }, + test_version: "test-v1", + challenger: ch, + champion: cm, + h1_holds: Some(true), + h2_holds: Some(true), + contamination_hits: Some(0), + p95_s: Some(40.0), + queue_budget_p95_s: Some(60.0), + } +} + +#[test] +fn falsify_rxc_001_all_five_gates_promote_and_each_one_alone_rejects() { + let (ch, cm) = (better(), champ()); + let d = decide(&[], &pin("w0"), &evidence(&ch, &cm)); + assert_eq!(d.outcome, Outcome::Promoted, "{:?}", d.reasons); + assert_eq!((d.mcnemar_b, d.mcnemar_c, d.evaluations_used), (20, 0, 1)); + + let same = champ(); + let mut flips: Vec<(&str, Evidence<'_>)> = Vec::new(); + flips.push(( + "1", + Evidence { + challenger: &same, + ..evidence(&ch, &cm) + }, + )); + flips.push(( + "3: H1", + Evidence { + h1_holds: None, + ..evidence(&ch, &cm) + }, + )); + flips.push(( + "3: H2", + Evidence { + h2_holds: Some(false), + ..evidence(&ch, &cm) + }, + )); + flips.push(( + "4:", + Evidence { + contamination_hits: Some(1), + ..evidence(&ch, &cm) + }, + )); + flips.push(( + "4:", + Evidence { + contamination_hits: None, + ..evidence(&ch, &cm) + }, + )); + flips.push(( + "5:", + Evidence { + p95_s: Some(61.0), + ..evidence(&ch, &cm) + }, + )); + flips.push(( + "5:", + Evidence { + queue_budget_p95_s: None, + ..evidence(&ch, &cm) + }, + )); + for (gate, e) in flips { + let d = decide(&[], &pin("w0"), &e); + assert_eq!(d.outcome, Outcome::Rejected, "gate {gate}"); + assert_eq!(d.reasons.len(), 1, "gate {gate}: {:?}", d.reasons); + assert!(d.reasons[0].starts_with(gate), "{:?}", d.reasons); + assert_eq!( + d.evaluations_used, 1, + "a rejection still spends an evaluation" + ); + } +} + +#[test] +fn falsify_rxc_001_more_correct_but_less_precise_is_rejected() { + // Every defect flagged, plus 5 false FAILs: b = 20, c = 5, p ≈ 0.002. + let (ch, cm) = (rows(|i| answer(i, !(30..35).contains(&i))), champ()); + let d = decide(&[], &pin("w0"), &evidence(&ch, &cm)); + assert!(d.mcnemar_p.is_some_and(|p| p < ALPHA), "{:?}", d.mcnemar_p); + assert_eq!(d.outcome, Outcome::Rejected); + assert_eq!(d.reasons.len(), 1); + assert!(d.reasons[0].starts_with("2:"), "{:?}", d.reasons); + assert!(d.precision_delta.is_some_and(|x| x < 0.0)); +} + +#[test] +fn falsify_rxc_002_the_budget_counts_and_refusals_spend_nothing() { + let (ch, cm) = (better(), champ()); + let mut ledger = Vec::new(); + for n in 0..EVALUATIONS_PER_VERSION { + let e = Evidence { + h1_holds: Some(false), + ..evidence(&ch, &cm) + }; + let d = decide(&ledger, &pin("w0"), &e); + assert_eq!(d.evaluations_used, n + 1); + ledger.push(d); + } + let d = decide(&ledger, &pin("w0"), &evidence(&ch, &cm)); + assert_eq!(d.outcome, Outcome::Refused, "the 21st evaluation"); + assert_eq!(d.evaluations_used, EVALUATIONS_PER_VERSION); + let fresh = decide( + &ledger, + &pin("w0"), + &Evidence { + test_version: "test-v2", + ..evidence(&ch, &cm) + }, + ); + assert_eq!( + fresh.outcome, + Outcome::Promoted, + "a rotated version has its own budget" + ); + + let never = rows(|_| Verdict::NotRun(NotRun::NoDeclaredExecutor)); + let d = decide(&[], &pin("w0"), &evidence(&never, &cm)); + assert_eq!( + (d.outcome, d.evaluations_used), + (Outcome::Refused, 0), + "R-6" + ); + let d = decide( + &[], + &pin("w0"), + &Evidence { + pin: pin("w0"), + ..evidence(&ch, &cm) + }, + ); + assert_eq!( + d.outcome, + Outcome::Refused, + "the champion cannot challenge itself" + ); +} + +#[test] +fn falsify_rxc_002_promotion_moves_the_pin_and_rollback_is_the_previous_sha() { + let (ch, cm) = (better(), champ()); + let d = decide(&[], &pin("w0"), &evidence(&ch, &cm)); + let ledger = vec![d]; + assert_eq!(champion(&ledger, &pin("w0")).prompt_sha256, "p2"); + assert_eq!(ledger[0].champion_before, pin("w0"), "rollback target"); +} + +#[test] +fn falsify_rxc_003_two_consecutive_gold_escapes_demote() { + let esc = |pr: &str, gold: bool, hit: bool| Escape { + pr: pr.into(), + gold, + champion_passed: hit, + a_voter_failed: hit, + }; + assert_eq!( + demotion(&[ + esc("#1", true, true), + esc("#2", false, false), + esc("#3", true, true) + ]), + Some(("#1".into(), "#3".into())), + "a silver row neither breaks nor extends the run" + ); + assert_eq!( + demotion(&[ + esc("#1", true, true), + esc("#2", true, false), + esc("#3", true, true) + ]), + None, + "a gold escape the champion caught breaks the run" + ); + assert_eq!(demotion(&[esc("#1", true, true)]), None); +} diff --git a/crates/aprender-review-experiment/src/cluster.rs b/crates/aprender-review-experiment/src/cluster.rs new file mode 100644 index 0000000000..53a82b1c36 --- /dev/null +++ b/crates/aprender-review-experiment/src/cluster.rs @@ -0,0 +1,197 @@ +//! Near-duplicate cluster check for sealed test items (PRA-001 T7, gate G-CON; +//! contract `review-corpus-contamination-v1`, FALSIFY-RCC-004..006). +//! +//! The exact checks in [`crate::contamination`] (diff sha, hunk sha) miss a +//! sealed diff that was re-indented or had an identifier renamed. This module +//! catches it with a bottom-k MinHash sketch over normalised token shingles: +//! +//! - **whitespace**: tokens are taken with all whitespace dropped, and blank +//! lines produce no tokens, so re-indenting or re-spacing changes nothing; +//! - **renames**: every identifier becomes `$` (the MOSS normalisation), so any +//! rename, even a partial one, yields the same token stream; keywords and +//! literals are kept, so `x += 1` and `x += 2` still differ. +//! +//! A sealed item's sketch is the `SKETCH` smallest shingle hashes. A text leaks +//! the item when at least `CONTAINMENT` of that sketch is among the text's own +//! shingle hashes: an unbiased estimate of how much of the item the text holds, +//! so a sealed diff embedded in a longer prompt is still seen. + +use std::collections::HashSet; + +/// Tokens per shingle. +pub const SHINGLE: usize = 12; +/// Hashes kept per sealed item (bottom-k). +pub const SKETCH: usize = 64; +/// Items with fewer distinct shingles are too generic to cluster on. +pub const MIN_SHINGLES: usize = 32; +/// Fraction of a sealed sketch that must appear in a text for a hit. +pub const CONTAINMENT: f64 = 0.7; +/// Header line of a rendered sketch file. +pub const SKETCH_SCHEME: &str = "review-corpus-v1 cluster-sketch shingle=12 k=64"; + +const HEADERS: &[&str] = &["diff --git", "index ", "--- ", "+++ ", "@@"]; + +const KEYWORDS: &[&str] = &[ + "as", "async", "await", "break", "const", "continue", "crate", "dyn", "else", "enum", "extern", + "false", "fn", "for", "if", "impl", "in", "let", "loop", "match", "mod", "move", "mut", "pub", + "ref", "return", "self", "Self", "static", "struct", "super", "trait", "true", "type", + "unsafe", "use", "where", "while", +]; + +/// Normalised tokens of one hunk body (or any text): whitespace dropped, a +/// `+`/`-` line marker kept as a token, identifiers replaced by `$`. +#[must_use] +pub fn tokens(body: &str) -> Vec { + let mut out = Vec::new(); + for line in body.lines() { + let (marker, rest) = line_marker(line); + if rest.trim().is_empty() { + continue; + } + out.extend(marker.map(String::from)); + push_line_tokens(rest, &mut out); + } + out +} + +/// The line's `+`/`-` marker (kept as a token) or a leading context space +/// (dropped), and the remainder of the line after either. +fn line_marker(line: &str) -> (Option, &str) { + match line.chars().next() { + Some(c @ ('+' | '-')) => (Some(c), &line[1..]), + Some(' ') => (None, &line[1..]), + _ => (None, line), + } +} + +/// Append one line's tokens: whitespace dropped, identifier runs normalised +/// via [`ident`], every other character kept as its own token. +fn push_line_tokens(rest: &str, out: &mut Vec) { + let mut chars = rest.chars().peekable(); + while let Some(c) = chars.next() { + if c.is_whitespace() { + continue; + } + if c.is_alphanumeric() || c == '_' { + out.push(ident(take_ident(c, &mut chars))); + } else { + out.push(c.to_string()); + } + } +} + +/// Consume a run of identifier characters starting at `first`, returning the +/// whole run as a `String`. +fn take_ident(first: char, chars: &mut std::iter::Peekable>) -> String { + let mut w = String::from(first); + while let Some(&n) = chars.peek() { + if n.is_alphanumeric() || n == '_' { + w.push(n); + chars.next(); + } else { + break; + } + } + w +} + +fn ident(w: String) -> String { + let lead = w.chars().next().unwrap_or('0'); + if lead.is_ascii_digit() || KEYWORDS.contains(&w.as_str()) { + w + } else { + "$".into() + } +} + +fn mix(mut z: u64) -> u64 { + z = (z ^ (z >> 30)).wrapping_mul(0xbf58_476d_1ce4_e5b9); + z = (z ^ (z >> 27)).wrapping_mul(0x94d0_49bb_1331_11eb); + z ^ (z >> 31) +} + +fn hash(shingle: &[String]) -> u64 { + let mut h: u64 = 0xcbf2_9ce4_8422_2325; + for t in shingle { + for b in t.bytes().chain([0xff]) { + h = (h ^ u64::from(b)).wrapping_mul(0x0100_0000_01b3); + } + } + mix(h) +} + +/// Distinct shingle hashes of a text. Diff header lines (`diff --git`, `index`, +/// `---`, `+++`, `@@`) are skipped, so a re-based or re-pathed copy matches; +/// every other line is tokenised, so a copy whose line markers were mangled (a +/// context space eaten by re-indenting) or that sits inside a prompt is seen. +#[must_use] +pub fn shingles(text: &str) -> HashSet { + let body: Vec<&str> = text + .lines() + .filter(|l| !HEADERS.iter().any(|h| l.starts_with(h))) + .collect(); + tokens(&body.join("\n")) + .windows(SHINGLE) + .map(hash) + .collect() +} + +/// Bottom-k sketch of a sealed diff, ascending; `None` below `MIN_SHINGLES`. +#[must_use] +pub fn sketch(diff: &str) -> Option> { + let mut s: Vec = shingles(diff).into_iter().collect(); + if s.len() < MIN_SHINGLES { + return None; + } + s.sort_unstable(); + s.truncate(SKETCH); + Some(s) +} + +/// Fraction of `sketch` present in `set` (0 for an empty sketch). +#[must_use] +pub fn containment(sketch: &[u64], set: &HashSet) -> f64 { + if sketch.is_empty() { + return 0.0; + } + let n = sketch.iter().filter(|h| set.contains(h)).count(); + n as f64 / sketch.len() as f64 +} + +/// Render sealed sketches: the scheme line, then `id hex,hex,…` sorted by id. +#[must_use] +pub fn render_sketches(items: &[(String, Vec)]) -> String { + let mut rows: Vec<&(String, Vec)> = items.iter().collect(); + rows.sort_by(|a, b| a.0.cmp(&b.0)); + let mut out = format!("{SKETCH_SCHEME}\n"); + for (id, s) in rows { + let hex: Vec = s.iter().map(|h| format!("{h:016x}")).collect(); + out.push_str(&format!("{id} {}\n", hex.join(","))); + } + out +} + +/// Parse a sketch file. `None` if the scheme line is missing or any row is +/// malformed: a sketch file that cannot be read clusters nothing. +#[must_use] +pub fn parse_sketches(text: &str) -> Option)>> { + let mut lines = text.lines(); + if lines.next()? != SKETCH_SCHEME { + return None; + } + lines + .filter(|l| !l.is_empty()) + .map(|l| { + let (id, hex) = l.split_once(' ')?; + let s = hex + .split(',') + .map(|h| u64::from_str_radix(h, 16).ok()) + .collect::>>()?; + Some((id.to_string(), s)) + }) + .collect() +} + +#[cfg(test)] +#[path = "cluster_tests.rs"] +mod tests; diff --git a/crates/aprender-review-experiment/src/cluster_tests.rs b/crates/aprender-review-experiment/src/cluster_tests.rs new file mode 100644 index 0000000000..1c97765236 --- /dev/null +++ b/crates/aprender-review-experiment/src/cluster_tests.rs @@ -0,0 +1,140 @@ +// `json!` expands to an `unwrap` of an infallible `to_value`. +#![allow(clippy::disallowed_methods)] + +use super::*; +use crate::contamination::{Hit, Index}; +use crate::corpus::{hunk_fingerprints, sha256_hex, Sealed}; +use crate::pool::{admit, local_tagged, Verdict}; + +const SEALED: &str = "--- a/crates/x/src/sum.rs\n+++ b/crates/x/src/sum.rs\n@@ -40,12 +40,14 @@ impl Ledger {\n pub fn total(&self, rows: &[Row]) -> u64 {\n let mut total = 0u64;\n for row in rows {\n- if row.amount > 0 {\n- total += row.amount;\n+ if row.amount > 0 && !row.voided {\n+ total = total.saturating_add(row.amount);\n }\n }\n+ debug_assert!(total >= self.floor);\n total\n }\n"; + +/// The same change as `SEALED`, re-indented with tabs, re-spaced, with blank +/// lines added, the local `total` renamed to `acc` (the method keeps its name, +/// so the rename is partial) and `row` to `r`, and re-based. +fn perturbed() -> String { + SEALED + .replace("@@ -40,12 +40,14 @@ impl Ledger {", "@@ -212,12 +219,14 @@") + .replace(" ", "\t") + .replace(" > 0", ">0") + .replace(" }\n", " }\n \n") + .replace("total", "acc") + .replace("row", "r") + .replace("fn acc", "fn total") +} + +const UNRELATED: &[&str] = &[ + "--- a/crates/y/src/parse.rs\n+++ b/crates/y/src/parse.rs\n@@ -8,9 +8,11 @@\n pub fn parse(s: &str) -> Result {\n let mut it = s.split(':');\n- let key = it.next().ok_or(Error::Empty)?;\n+ let key = it.next().ok_or(Error::Empty)?.trim();\n+ if key.is_empty() {\n+ return Err(Error::Empty);\n+ }\n let value = it.next().unwrap_or_default();\n Ok(Header { key: key.into(), value: value.into() })\n }\n", + "--- a/crates/z/src/cache.rs\n+++ b/crates/z/src/cache.rs\n@@ -30,8 +30,10 @@\n impl Cache {\n pub fn get(&mut self, k: &Key) -> Option<&Value> {\n- self.hits += 1;\n- self.map.get(k)\n+ let v = self.map.get(k);\n+ if v.is_some() { self.hits += 1 } else { self.misses += 1 }\n+ v\n }\n pub fn clear(&mut self) { self.map.clear(); }\n }\n", + "--- a/crates/w/src/sum.rs\n+++ b/crates/w/src/sum.rs\n@@ -40,12 +40,12 @@ impl Ledger {\n pub fn count(&self, rows: &[Row]) -> usize {\n let mut n = 0usize;\n for row in rows {\n- if row.kind == Kind::Debit {\n+ if matches!(row.kind, Kind::Debit | Kind::Fee) {\n n += 1;\n }\n }\n n\n }\n", +]; + +fn sealed_index() -> Index { + Index::new(&[Sealed { + id: "R042".into(), + diff_sha256: sha256_hex(SEALED.as_bytes()), + hunks: hunk_fingerprints(SEALED), + }]) +} + +fn sketches() -> Vec<(String, Vec)> { + vec![( + "R042".into(), + sketch(SEALED).expect("SEALED is above MIN_SHINGLES"), + )] +} + +/// FALSIFY-RCC-004 (G-CON): a sealed item with whitespace and rename +/// perturbation, inserted into a candidate pool, passes the exact-hash checks +/// and is caught once the cluster check lands: a `cluster` hit, and the row is +/// refused, bare and inside a JSONL field beside other prompt text. +#[test] +fn falsify_rcc_004_perturbed_sealed_item_is_a_cluster_hit() { + let p = perturbed(); + assert_ne!(p, SEALED); + assert!( + sealed_index().scan(&p, "pool").is_empty(), + "the perturbation must defeat the exact checks, or this test proves nothing" + ); + let ix = sealed_index().with_sketches(sketches()); + assert_eq!( + ix.scan(&p, "pool"), + vec![Hit { + item: "R042".into(), + file: "pool".into(), + how: "cluster" + }] + ); + let row = serde_json::json!({"lane": "sonnet-5", "input": format!("Review this diff.\n\n{p}\nBe terse.")}) + .to_string(); + let clean = local_tagged(serde_json::json!({"lane": "qwen", "input": UNRELATED[0]})); + let a = admit(&format!("{row}\n{clean}\n"), &ix); + assert_eq!( + a.verdicts[0].1, + Verdict::Refused { + items: vec!["R042".into()] + } + ); + assert_eq!(a.pool(), vec![clean.as_str()]); + // a backport that also gained one line is still the same cluster + let backport = p.replace("\t let mut", "+\t let _guard = lock();\n\t let mut"); + assert_ne!(backport, p); + assert_eq!(ix.scan(&backport, "pool").len(), 1); +} + +/// FALSIFY-RCC-005: unrelated diffs, including one over the same file shape +/// with the same loop skeleton, are not clustered with the sealed item, and a +/// diff too short to fingerprint has no sketch. +#[test] +fn falsify_rcc_005_unrelated_diffs_are_not_clustered() { + let ix = sealed_index().with_sketches(sketches()); + let s = &sketches()[0].1; + for u in UNRELATED { + let c = containment(s, &shingles(u)); + assert!(c < CONTAINMENT, "containment {c} for {u}"); + assert!(ix.scan(u, "pool").is_empty(), "{u}"); + } + let short = "@@ -1,4 +1,4 @@\n fn f(x: u32) -> bool {\n- x > 1\n+ x >= 1\n }\n"; + assert_eq!(sketch(short), None); +} + +/// FALSIFY-RCC-006: the sketch file round-trips, and a file that is not a +/// sketch file (wrong scheme, bad or missing hex) parses to `None`. +#[test] +fn falsify_rcc_006_sketch_file_round_trips_and_fails_closed() { + let items = sketches(); + assert_eq!(parse_sketches(&render_sketches(&items)), Some(items)); + for bad in [ + "", + "review-corpus-v1 test-manifest\nR1 00ff\n", + &format!("{SKETCH_SCHEME}\nR1 zz\n"), + &format!("{SKETCH_SCHEME}\nR1\n"), + &format!("{SKETCH_SCHEME}\nR1 \n"), + ] { + assert_eq!(parse_sketches(bad), None, "{bad:?}"); + } +} + +#[test] +fn tokens_case_table() { + let t = |s: &str| tokens(s).join(" "); + // whitespace and blank lines vanish; a +/- marker is a token + assert_eq!(t("+ x >= 1;\n \n"), "+ $ > = 1 ;"); + assert_eq!(t("+x>=1;"), "+ $ > = 1 ;"); + assert_eq!(t("+x;\n+\n-\t\n+y;"), t("+x;\n+y;")); + // any rename gives the same stream; keywords and literals stay + assert_eq!( + t(" let mut total = 0u64;\n+total += n;"), + t(" let mut acc = 0u64;\n+acc += n;") + ); + assert_eq!(t(" let mut total = 0u64;"), "let mut $ = 0u64 ;"); + assert_ne!(t("+x += 1;"), t("+x += 2;")); + assert_ne!(t("+x += 1;"), t("-x += 1;")); +} + +#[test] +fn sketch_is_bottom_k_and_perturbation_invariant() { + let s = sketch(SEALED).expect("sketch"); + assert!(s.len() <= SKETCH && s.len() >= MIN_SHINGLES); + assert!(s.windows(2).all(|w| w[0] < w[1])); + assert_eq!(sketch(&perturbed()), Some(s)); +} diff --git a/crates/aprender-review-experiment/src/contamination.rs b/crates/aprender-review-experiment/src/contamination.rs index dc47776ede..742ab6613f 100644 --- a/crates/aprender-review-experiment/src/contamination.rs +++ b/crates/aprender-review-experiment/src/contamination.rs @@ -6,7 +6,11 @@ //! stripped, so a re-based copy still matches). JSON string values are scanned //! after unescaping, so a diff stored in a JSONL `"diff"` field is seen as the //! diff it is, not as one escaped line. +//! +//! With [`Index::with_sketches`], a re-spaced, renamed or lightly edited copy +//! is a `cluster` hit too (PRA-001 T7, [`crate::cluster`]). +use crate::cluster::{containment, shingles, CONTAINMENT}; use crate::corpus::{hunk_fingerprints, Sealed}; use std::collections::BTreeMap; @@ -22,11 +26,13 @@ pub struct Hit { pub how: &'static str, } -/// Index of the sealed items: diff sha → id, hunk fingerprint → id. +/// Index of the sealed items: diff sha → id, hunk fingerprint → id, and +/// (optionally) each item's near-dup sketch from [`crate::cluster`]. #[derive(Debug, Default)] pub struct Index { shas: BTreeMap, hunks: BTreeMap, + sketches: Vec<(String, Vec)>, } impl Index { @@ -42,6 +48,14 @@ impl Index { ix } + /// Add sealed near-dup sketches (`id`, bottom-k hashes), so a re-spaced or + /// renamed copy of a sealed diff is a `cluster` hit. + #[must_use] + pub fn with_sketches(mut self, sketches: Vec<(String, Vec)>) -> Self { + self.sketches = sketches; + self + } + /// Number of sealed diff shas indexed (0 means the scan proves nothing). #[must_use] pub fn len(&self) -> usize { @@ -65,6 +79,15 @@ impl Index { hits.push(hit(id, file, "hunk")); } } + if self.sketches.is_empty() { + return; + } + let set = shingles(text); + for (id, s) in &self.sketches { + if containment(s, &set) >= CONTAINMENT { + hits.push(hit(id, file, "cluster")); + } + } } /// Every leak of a sealed item into `text` (the contents of `file`). diff --git a/crates/aprender-review-experiment/src/datacard.rs b/crates/aprender-review-experiment/src/datacard.rs new file mode 100644 index 0000000000..0f6a0988af --- /dev/null +++ b/crates/aprender-review-experiment/src/datacard.rs @@ -0,0 +1,1038 @@ +//! PRA-001 T14: `trace-datacard-v1` — the Croissant 1.0 + RAI card and the +//! Datasheet for one snapshot of the `agent-trace-v1` index (spec §2.11). +//! +//! A snapshot is the index's bytes: every `index/agent-trace-v1/YYYY/MM/DD.jsonl` +//! under the traces root, in path order, bound by a `sha256sum`-format manifest. +//! The card is generated from that snapshot and never hand-edited: +//! [`write_snapshot`] refuses to overwrite a card that differs from its +//! regeneration. The card carries aggregate counts only — no row value +//! (finding text, diff, sha, PR body) ever reaches it (PRIVATE, §6). +//! +//! The official Croissant validator (`mlcroissant`) is Python, which S-PY +//! forbids, so [`validate`] is a structural Croissant 1.0 + RAI 1.0 checker +//! in Rust: context, dataset type, conformance, resources, record set, fields, +//! references and the RAI fields the spec requires. + +use std::collections::{BTreeMap, BTreeSet}; +use std::fs; +use std::io::Write as _; +use std::path::{Path, PathBuf}; + +use serde_json::{json, Map, Value}; + +pub use crate::corpus::sha256_hex; + +pub const SCHEME: &str = "trace-datacard-v1"; +/// The index, relative to the traces root (`/almacen/traces`). +pub const INDEX_DIR: &str = "index/agent-trace-v1"; +/// Where cards land, relative to the traces root: `datacard//`. +pub const DATACARD_DIR: &str = "datacard"; +pub const ROW_SCHEMA: &str = "agent-trace-v1"; +pub const CROISSANT: &str = "http://mlcommons.org/croissant/1.0"; +pub const CROISSANT_RAI: &str = "http://mlcommons.org/croissant/RAI/1.0"; + +/// The RAI fields §2.11 requires (collection, labelling, limitations, use, PII). +pub const RAI_FIELDS: [&str; 10] = [ + "rai:dataCollection", + "rai:dataCollectionType", + "rai:dataCollectionTimeframe", + "rai:dataAnnotationProtocol", + "rai:dataPreprocessingProtocol", + "rai:dataUseCases", + "rai:dataLimitations", + "rai:dataBiases", + "rai:personalSensitiveInformation", + "rai:dataReleaseMaintenancePlan", +]; + +const DATA_TYPES: [&str; 8] = [ + "sc:Text", + "sc:Integer", + "sc:Float", + "sc:Boolean", + "sc:Date", + "sc:DateTime", + "sc:URL", + "sc:Number", +]; + +/// Composition columns counted per snapshot: (name, JSON path). +const COUNTED: [(&str, &[&str]); 7] = [ + ("provider", &["provider"]), + ("lane", &["lane"]), + ("label_tier", &["label_tier"]), + ("outcome", &["outcome"]), + ("split", &["split_guard", "split"]), + ("parse_status", &["parse_status"]), + ("secret_scan", &["secret_scan", "status"]), +]; + +/// The record-set fields: (name, jsonPath, dataType). Row-level free text +/// (`findings[].text`) is deliberately not described as a field. +const FIELDS: [(&str, &str, &str); 24] = [ + ("trace_id", "$.trace_id", "sc:Text"), + ("quorum_id", "$.quorum_id", "sc:Text"), + ("round", "$.round", "sc:Integer"), + ("lane", "$.lane", "sc:Text"), + ("counted", "$.counted", "sc:Boolean"), + ("provider", "$.provider", "sc:Text"), + ("model_id", "$.model_id", "sc:Text"), + ("access_channel", "$.access_channel", "sc:Text"), + ("repo", "$.repo", "sc:Text"), + ("pr", "$.pr", "sc:Integer"), + ("head_sha", "$.head_sha", "sc:Text"), + ("prompt_version", "$.prompt_version", "sc:Text"), + ("lane_state", "$.lane_state", "sc:Text"), + ("verdict", "$.verdict", "sc:Text"), + ("parse_status", "$.parse_status", "sc:Text"), + ("label_tier", "$.label_tier", "sc:Text"), + ("outcome", "$.outcome", "sc:Text"), + ("split", "$.split_guard.split", "sc:Text"), + ("group_key", "$.split_guard.group_key", "sc:Text"), + ("secret_scan_status", "$.secret_scan.status", "sc:Text"), + ("trace_status", "$.trace_status", "sc:Text"), + ("agreed", "$.agreed", "sc:Boolean"), + ("lane_disagreement", "$.lane_disagreement", "sc:Integer"), + ("dedup_cluster", "$.dedup_cluster", "sc:Text"), +]; + +/// Card identity that is not derived from the index. +#[derive(Debug, Clone)] +pub struct CardMeta { + pub name: String, + pub license: String, + pub url: String, +} + +impl Default for CardMeta { + fn default() -> Self { + Self { + name: "agent-trace-v1".into(), + license: "LicenseRef-paiml-private: not for public release; \ + gold rows with provider local only may be released (spec §2.11)" + .into(), + url: "https://github.com/paiml/aprender/blob/main/docs/specifications/\ + research-design-pr-agent-qwen-3.5.md" + .into(), + } + } +} + +/// One index file: its path relative to the traces root, sha256 and row count. +#[derive(Debug, Clone)] +pub struct IndexFile { + pub rel: String, + pub sha256: String, + pub bytes: u64, + pub rows: u64, +} + +/// One snapshot of the index: its files and aggregate counts, nothing else. +#[derive(Debug)] +pub struct Snapshot { + /// First 16 hex of the manifest's sha256. + pub id: String, + pub rows: u64, + /// Rows a public release may carry: gold, provider local, secret scan clean. + pub public_eligible: u64, + pub files: Vec, + counts: BTreeMap<(String, String), u64>, + /// Per index day: the G16 yield inputs. Shas only, never row content. + days: BTreeMap, +} + +/// One index day's yield: rows, distinct quorums, index bytes, blob shas. +#[derive(Debug, Default)] +struct DayYield { + rows: u64, + quorums: BTreeSet, + index_bytes: u64, + blobs: BTreeSet, +} + +impl Snapshot { + /// Read every index file under `root`. A bad line or a foreign schema is an + /// error naming `file:line` — never a skipped row. + pub fn read(root: &Path) -> Result { + let mut paths = Vec::new(); + walk(&root.join(INDEX_DIR), &mut paths)?; + paths.sort(); + if paths.is_empty() { + return Err(format!( + "no index files under {}", + root.join(INDEX_DIR).display() + )); + } + let mut snap = Snapshot { + id: String::new(), + rows: 0, + public_eligible: 0, + files: Vec::new(), + counts: BTreeMap::new(), + days: BTreeMap::new(), + }; + for p in paths { + snap.add_file(root, &p)?; + } + snap.id = sha256_hex(snap.manifest().as_bytes())[..16].to_string(); + Ok(snap) + } + + fn add_file(&mut self, root: &Path, p: &Path) -> Result<(), String> { + let rel = p + .strip_prefix(root) + .map_err(|e| format!("{}: {e}", p.display()))? + .to_string_lossy() + .into_owned(); + let day = day_of(&rel)?; + let bytes = fs::read(p).map_err(|e| format!("{rel}: {e}"))?; + let text = std::str::from_utf8(&bytes).map_err(|e| format!("{rel}: not utf-8: {e}"))?; + let mut rows = 0; + for (n, line) in text.lines().enumerate() { + if line.trim().is_empty() { + continue; + } + let row: Value = serde_json::from_str(line) + .map_err(|e| format!("{rel}:{}: bad json: {e}", n + 1))?; + let schema = row + .get("schema") + .and_then(Value::as_str) + .unwrap_or(""); + if schema != ROW_SCHEMA { + return Err(format!( + "{rel}:{}: schema {schema:?} is not {ROW_SCHEMA}", + n + 1 + )); + } + self.add_row(&row); + self.add_yield(&day, &row); + rows += 1; + } + self.days.entry(day).or_default().index_bytes += bytes.len() as u64; + self.files.push(IndexFile { + rel, + sha256: sha256_hex(&bytes), + bytes: bytes.len() as u64, + rows, + }); + Ok(()) + } + + fn add_row(&mut self, row: &Value) { + self.rows += 1; + for (col, path) in COUNTED { + let v = path + .iter() + .try_fold(row, |v, k| v.get(k)) + .and_then(Value::as_str) + .unwrap_or(""); + *self.counts.entry((col.into(), v.into())).or_default() += 1; + } + if public_eligible(row) { + self.public_eligible += 1; + } + } + + fn add_yield(&mut self, day: &str, row: &Value) { + let d = self.days.entry(day.to_string()).or_default(); + d.rows += 1; + if let Some(q) = row.get("quorum_id").and_then(Value::as_str) { + d.quorums.insert(q.to_string()); + } + for k in BLOB_SHAS { + if let Some(h) = row.get(k).and_then(Value::as_str).filter(|h| is_sha256(h)) { + d.blobs.insert(h.to_string()); + } + } + } + + /// `sha256sum` format, one line per index file, in path order. + pub fn manifest(&self) -> String { + self.files + .iter() + .map(|f| format!("{} {}\n", f.sha256, f.rel)) + .collect() + } + + pub fn count(&self, col: &str, val: &str) -> u64 { + self.counts + .get(&(col.to_string(), val.to_string())) + .copied() + .unwrap_or(0) + } + + /// (value, count) for one composition column, value order. + pub fn column(&self, col: &str) -> Vec<(&str, u64)> { + self.counts + .iter() + .filter(|((c, _), _)| c == col) + .map(|((_, v), n)| (v.as_str(), *n)) + .collect() + } + + /// First and last index day, `YYYY-MM-DD`. + pub fn timeframe(&self) -> (String, String) { + let days: Vec = self + .files + .iter() + .filter_map(|f| day_of(&f.rel).ok()) + .collect(); + ( + days.first().cloned().unwrap_or_default(), + days.last().cloned().unwrap_or_default(), + ) + } +} + +/// Row columns that address a CAS blob (`blobs/sha256/ab/cd/.zst`). +const BLOB_SHAS: [&str; 3] = ["input_sha", "output_sha", "logits_sha"]; + +fn is_sha256(h: &str) -> bool { + h.len() == 64 && h.bytes().all(|b| b.is_ascii_hexdigit()) +} + +/// §2.11: HF publication is gold-and-local-only; anthropic/google rows never, +/// and a row with any secret-scan status but `clean` never. +fn public_eligible(row: &Value) -> bool { + let s = |k: &str| row.get(k).and_then(Value::as_str); + s("label_tier") == Some("gold") + && s("provider") == Some("local") + && row.pointer("/secret_scan/status").and_then(Value::as_str) == Some("clean") +} + +fn walk(dir: &Path, out: &mut Vec) -> Result<(), String> { + if !dir.exists() { + return Ok(()); + } + for e in fs::read_dir(dir).map_err(|e| format!("{}: {e}", dir.display()))? { + let p = e.map_err(|e| format!("{}: {e}", dir.display()))?.path(); + if p.is_dir() { + walk(&p, out)?; + } else if p.extension().is_some_and(|x| x == "jsonl") { + out.push(p); + } + } + Ok(()) +} + +/// `index/agent-trace-v1/YYYY/MM/DD.jsonl` -> `YYYY-MM-DD`. +fn day_of(rel: &str) -> Result { + let tail = rel + .strip_prefix(INDEX_DIR) + .and_then(|t| t.strip_prefix('/')) + .and_then(|t| t.strip_suffix(".jsonl")) + .ok_or_else(|| format!("{rel}: not under {INDEX_DIR}/"))?; + let parts: Vec<&str> = tail.split('/').collect(); + let ok = parts.len() == 3 + && [4, 2, 2] + .iter() + .zip(&parts) + .all(|(n, p)| p.len() == *n && p.bytes().all(|b| b.is_ascii_digit())); + if !ok { + return Err(format!("{rel}: want {INDEX_DIR}/YYYY/MM/DD.jsonl")); + } + Ok(parts.join("-")) +} + +// `json!` expands to an `unwrap` of an infallible `to_value` on strings, hence the scoped allow. +#[allow(clippy::disallowed_methods)] +fn context() -> Value { + json!({ + "@language": "en", "@vocab": "https://schema.org/", + "citeAs": "cr:citeAs", "column": "cr:column", "conformsTo": "dct:conformsTo", + "cr": "http://mlcommons.org/croissant/", "rai": "http://mlcommons.org/croissant/RAI/", + "data": {"@id": "cr:data", "@type": "@json"}, + "dataType": {"@id": "cr:dataType", "@type": "@vocab"}, + "dct": "http://purl.org/dc/terms/", + "examples": {"@id": "cr:examples", "@type": "@json"}, + "extract": "cr:extract", "field": "cr:field", "fileProperty": "cr:fileProperty", + "fileObject": "cr:fileObject", "fileSet": "cr:fileSet", "format": "cr:format", + "includes": "cr:includes", "isLiveDataset": "cr:isLiveDataset", + "jsonPath": "cr:jsonPath", "key": "cr:key", "md5": "cr:md5", + "parentField": "cr:parentField", "path": "cr:path", "recordSet": "cr:recordSet", + "references": "cr:references", "regex": "cr:regex", "repeated": "cr:repeated", + "replace": "cr:replace", "sc": "https://schema.org/", "separator": "cr:separator", + "source": "cr:source", "subField": "cr:subField", "transform": "cr:transform", + }) +} + +// `json!` expands to an `unwrap` of an infallible `to_value` on strings, hence the scoped allow. +#[allow(clippy::disallowed_methods)] +fn fields() -> Vec { + FIELDS + .iter() + .map(|(name, path, ty)| { + json!({ + "@type": "cr:Field", + "@id": format!("agent-trace/{name}"), + "name": name, + "dataType": ty, + "source": {"fileSet": {"@id": "index-days"}, "extract": {"jsonPath": path}}, + }) + }) + .collect() +} + +fn composition(s: &Snapshot) -> String { + COUNTED + .iter() + .map(|(col, _)| { + let parts: Vec = s + .column(col) + .iter() + .map(|(v, n)| format!("{v}={n}")) + .collect(); + format!("{col}: {}", parts.join(", ")) + }) + .collect::>() + .join("; ") +} + +/// The Croissant 1.0 + RAI 1.0 card for one snapshot. Deterministic: the same +/// index bytes give the same card bytes (no clock is read). +#[allow(clippy::disallowed_methods)] // `json!` over strings, as above +pub fn croissant(s: &Snapshot, m: &CardMeta) -> Value { + let (first, last) = s.timeframe(); + let mut card = Map::new(); + card.insert("@context".into(), context()); + card.insert("@type".into(), json!("sc:Dataset")); + card.insert("conformsTo".into(), json!([CROISSANT, CROISSANT_RAI])); + card.insert("name".into(), json!(m.name)); + card.insert( + "description".into(), + json!(format!( + "PR-review lane traces (agent-trace-v1), snapshot {} of {} rows in {} daily \ + index files, {first}..{last}. Composition — {}.", + s.id, + s.rows, + s.files.len(), + composition(s) + )), + ); + card.insert("license".into(), json!(m.license)); + card.insert("url".into(), json!(m.url)); + card.insert("version".into(), json!(format!("0.0.0+{}", s.id))); + card.insert("datePublished".into(), json!(last)); + card.insert("isLiveDataset".into(), json!(true)); + card.insert( + "distribution".into(), + json!([ + { + "@type": "cr:FileObject", + "@id": "index-manifest", + "name": "index-manifest", + "description": "sha256sum manifest of every index file in this snapshot", + "contentUrl": "manifest.sha256", + "encodingFormat": "text/plain", + "sha256": sha256_hex(s.manifest().as_bytes()), + }, + { + "@type": "cr:FileSet", + "@id": "index-days", + "name": "index-days", + "description": "daily agent-trace-v1 index files under the traces root", + "encodingFormat": "application/jsonlines", + "includes": format!("{INDEX_DIR}/*/*/*.jsonl"), + }, + ]), + ); + card.insert( + "recordSet".into(), + json!([{ + "@type": "cr:RecordSet", + "@id": "agent-trace", + "name": "agent-trace", + "description": "one row per review lane per quorum round", + "key": {"@id": "agent-trace/trace_id"}, + "field": fields(), + }]), + ); + for (k, v) in rai(s, &first, &last) { + card.insert(k.into(), v); + } + Value::Object(card) +} + +// `json!` expands to an `unwrap` of an infallible `to_value` on strings, hence the scoped allow. +#[allow(clippy::disallowed_methods)] +fn rai(s: &Snapshot, first: &str, last: &str) -> [(&'static str, Value); 10] { + [ + ( + "rai:dataCollection", + json!( + "Captured at dispatch by the quorum harness: every review lane's full input and \ + output is written to the almacen CAS and indexed as one agent-trace-v1 row per \ + lane per round (spec §2.2). Rows are appended, never edited." + ), + ), + ( + "rai:dataCollectionType", + json!(["Machine-generated", "Software Collection"]), + ), + ("rai:dataCollectionTimeframe", json!([first, last])), + ( + "rai:dataAnnotationProtocol", + json!(format!( + "label_tier: silver = quorum-derived verdict; gold = the PR's matured outcome \ + (merged, reverted within 14 days, regression escape, closed unmerged); pending \ + until outcome_matured_at; quarantined on a secret-scan hit or parse failure. \ + This snapshot: {}.", + counts_of(s, "label_tier") + )), + ), + ( + "rai:dataPreprocessingProtocol", + json!( + "Rows are deduplicated by dedup_cluster and split by split_guard.group_key \ + (per-PR grouping, so no PR crosses train/val/test/sealed). Secret scanners run \ + before a row is indexed; this card is generated from the index by a Rust \ + binary and never hand-edited." + ), + ), + ( + "rai:dataUseCases", + json!( + "Training and evaluating a local PR-review model (Qwen 3.5) against the \ + quorum lanes; measuring lane agreement and outcome prediction. Not for \ + ranking or evaluating people." + ), + ), + ( + "rai:dataLimitations", + json!(format!( + "Single organisation's repositories and review conventions; silver labels \ + inherit quorum-lane bias; outcomes mature over 14 days so recent rows are \ + pending. Split composition: {}.", + counts_of(s, "split") + )), + ), + ( + "rai:dataBiases", + json!(format!( + "Lane and provider mix is set by quorum policy, not sampled: {}; {}.", + counts_of(s, "provider"), + counts_of(s, "lane") + )), + ), + ( + "rai:personalSensitiveInformation", + json!(format!( + "Rows may contain source code, commit authorship and review text. Every row is \ + secret-scanned; a hit sets secret_scan.status and the row is quarantined and \ + never released. Transcripts and trace blobs never enter a public repo, issue \ + or PR. Secret-scan status: {}.", + counts_of(s, "secret_scan") + )), + ), + ( + "rai:dataReleaseMaintenancePlan", + json!(format!( + "Private. A card is regenerated per index snapshot under datacard//. \ + Public (HF) release is gold rows with provider local only — provider anthropic \ + and google rows are excluded from any public release. Public-eligible rows in \ + this snapshot: {} of {}.", + s.public_eligible, s.rows + )), + ), + ] +} + +fn counts_of(s: &Snapshot, col: &str) -> String { + let parts: Vec = s + .column(col) + .iter() + .map(|(v, n)| format!("{v}={n}")) + .collect(); + format!("{col} {}", parts.join(", ")) +} + +/// Structural Croissant 1.0 + RAI 1.0 check. Empty means valid. +pub fn validate(card: &Value) -> Vec { + let mut errs = Vec::new(); + let Some(obj) = card.as_object() else { + return vec!["card is not a JSON object".into()]; + }; + check_context(obj, &mut errs); + check_dataset(obj, &mut errs); + let ids = check_ids(card, &mut errs); + check_distribution(obj, &ids, &mut errs); + check_record_sets(obj, &ids, &mut errs); + for k in RAI_FIELDS { + if !nonempty(obj.get(k)) { + errs.push(format!("{k} is missing or empty (RAI 1.0)")); + } + } + errs +} + +fn nonempty(v: Option<&Value>) -> bool { + match v { + Some(Value::String(s)) => !s.trim().is_empty(), + Some(Value::Array(a)) => !a.is_empty() && a.iter().all(|x| nonempty(Some(x))), + _ => false, + } +} + +fn check_context(obj: &Map, errs: &mut Vec) { + let want = context(); + let Some(ctx) = obj.get("@context").and_then(Value::as_object) else { + errs.push("@context is missing or not an object".into()); + return; + }; + for (k, v) in want.as_object().into_iter().flatten() { + match ctx.get(k) { + None => errs.push(format!("@context missing key {k}")), + Some(got) if got != v => errs.push(format!("@context key {k} is {got}, want {v}")), + Some(_) => {} + } + } +} + +fn check_dataset(obj: &Map, errs: &mut Vec) { + let s = |k: &str| obj.get(k).and_then(Value::as_str).unwrap_or(""); + if s("@type") != "sc:Dataset" { + errs.push(format!("@type is {:?}, want sc:Dataset", obj.get("@type"))); + } + let conforms: Vec<&str> = match obj.get("conformsTo") { + Some(Value::String(c)) => vec![c.as_str()], + Some(Value::Array(a)) => a.iter().filter_map(Value::as_str).collect(), + _ => Vec::new(), + }; + for want in [CROISSANT, CROISSANT_RAI] { + if !conforms.contains(&want) { + errs.push(format!("conformsTo does not list {want}")); + } + } + for k in ["name", "description", "license", "url", "datePublished"] { + if s(k).trim().is_empty() { + errs.push(format!("{k} is missing or empty")); + } + } +} + +/// Every `@id` defined by a resource, record set or field; duplicates are errors. +fn check_ids(card: &Value, errs: &mut Vec) -> BTreeSet { + let mut ids = BTreeSet::new(); + let defs = card["distribution"] + .as_array() + .into_iter() + .flatten() + .chain(card["recordSet"].as_array().into_iter().flatten()) + .chain( + card["recordSet"] + .as_array() + .into_iter() + .flatten() + .flat_map(|r| r["field"].as_array().into_iter().flatten()), + ); + for d in defs { + match d.get("@id").and_then(Value::as_str) { + None => errs.push(format!("{} has no @id", d["@type"])), + Some(id) if !ids.insert(id.to_string()) => errs.push(format!("duplicate @id {id}")), + Some(_) => {} + } + } + ids +} + +fn check_distribution(obj: &Map, ids: &BTreeSet, errs: &mut Vec) { + let dist = obj.get("distribution").and_then(Value::as_array); + if dist.is_none_or(|d| d.is_empty()) { + errs.push("distribution is missing or empty".into()); + } + for r in dist.into_iter().flatten() { + check_one_distribution(r, ids, errs); + } +} + +/// One `distribution[]` entry: `@type`-specific field checks, then its +/// `containedIn` reference (if any) against the known `@id` set. +fn check_one_distribution(r: &Value, ids: &BTreeSet, errs: &mut Vec) { + let id = r["@id"].as_str().unwrap_or("?"); + let has = |k: &str| { + r.get(k) + .and_then(Value::as_str) + .is_some_and(|s| !s.is_empty()) + }; + match r["@type"].as_str() { + Some("cr:FileObject") => check_file_object(r, id, &has, errs), + Some("cr:FileSet") => check_file_set(id, &has, errs), + t => errs.push(format!( + "distribution {id}: @type {t:?} is not cr:FileObject/cr:FileSet" + )), + } + if let Some(c) = r.get("containedIn") { + check_ref(c, ids, &format!("{id}.containedIn"), errs); + } +} + +/// A `cr:FileObject` distribution entry: required fields, and a well-formed +/// sha256. +fn check_file_object(r: &Value, id: &str, has: &impl Fn(&str) -> bool, errs: &mut Vec) { + for k in ["contentUrl", "encodingFormat"] { + if !has(k) { + errs.push(format!("FileObject {id}: {k} is missing")); + } + } + let sha = r["sha256"].as_str().unwrap_or(""); + if sha.len() != 64 || !sha.bytes().all(|b| b.is_ascii_hexdigit()) { + errs.push(format!("FileObject {id}: sha256 is missing or not 64 hex")); + } +} + +/// A `cr:FileSet` distribution entry: required fields. +fn check_file_set(id: &str, has: &impl Fn(&str) -> bool, errs: &mut Vec) { + for k in ["includes", "encodingFormat"] { + if !has(k) { + errs.push(format!("FileSet {id}: {k} is missing")); + } + } +} + +fn check_ref(v: &Value, ids: &BTreeSet, at: &str, errs: &mut Vec) { + match v.get("@id").and_then(Value::as_str) { + Some(id) if ids.contains(id) => {} + other => errs.push(format!("{at}: unknown resource {other:?}")), + } +} + +fn check_record_sets(obj: &Map, ids: &BTreeSet, errs: &mut Vec) { + let sets = obj.get("recordSet").and_then(Value::as_array); + if sets.is_none_or(|s| s.is_empty()) { + errs.push("recordSet is missing or empty".into()); + } + for rs in sets.into_iter().flatten() { + let rid = rs["@id"].as_str().unwrap_or("?"); + if rs["@type"] != "cr:RecordSet" { + errs.push(format!("recordSet {rid}: @type is not cr:RecordSet")); + } + if let Some(k) = rs.get("key") { + check_ref(k, ids, &format!("{rid}.key"), errs); + } + let fields = rs["field"].as_array(); + if fields.is_none_or(|f| f.is_empty()) { + errs.push(format!("recordSet {rid}: field is missing or empty")); + } + for f in fields.into_iter().flatten() { + check_field(f, ids, errs); + } + } +} + +fn check_field(f: &Value, ids: &BTreeSet, errs: &mut Vec) { + let fid = f["@id"].as_str().unwrap_or("?"); + if f["@type"] != "cr:Field" { + errs.push(format!("field {fid}: @type is not cr:Field")); + } + let ty = f["dataType"].as_str().unwrap_or(""); + if !DATA_TYPES.contains(&ty) { + errs.push(format!("field {fid}: dataType {ty:?} is not a known type")); + } + let src = &f["source"]; + match (src.get("fileSet"), src.get("fileObject")) { + (Some(r), _) | (None, Some(r)) => check_ref(r, ids, &format!("{fid}.source"), errs), + (None, None) => errs.push(format!("field {fid}: source names no resource")), + } + let path = src["extract"]["jsonPath"].as_str().unwrap_or(""); + if !path.starts_with('$') { + errs.push(format!( + "field {fid}: extract.jsonPath {path:?} does not start with $" + )); + } +} + +pub const WEEKLY_SCHEME: &str = "trace-weekly-receipt-v1"; +/// G16 `[O]`: 50–200 quorums/day and at most ~0.45 TB/yr raw. The operator's +/// band, not ours; the receipt turns G16 `[U]` into `[V]` or names what is missing. +pub const G16_QUORUMS_PER_DAY: (u64, u64) = (50, 200); +pub const G16_RAW_TB_PER_YEAR: f64 = 0.45; + +/// `YYYY-MM-DD` -> ISO 8601 week `YYYY-Www` (the year of the week's Thursday). +pub fn iso_week(day: &str) -> Result { + let p: Vec = day + .split('-') + .map(str::parse) + .collect::>() + .map_err(|e| format!("{day}: {e}"))?; + let [y, m, d] = p[..] else { + return Err(format!("{day}: want YYYY-MM-DD")); + }; + let n = days_from_civil(y, m, d); + let thursday = n - (n + 3).rem_euclid(7) + 3; + let year = (y - 1..=y + 1) + .rev() + .find(|&yy| days_from_civil(yy, 1, 1) <= thursday) + .unwrap_or(y); + let week = (thursday - days_from_civil(year, 1, 1)) / 7 + 1; + Ok(format!("{year:04}-W{week:02}")) +} + +/// Days since 1970-01-01 (proleptic Gregorian; H. Hinnant's algorithm). +fn days_from_civil(y: i64, m: i64, d: i64) -> i64 { + let y = if m <= 2 { y - 1 } else { y }; + let era = y.div_euclid(400); + let yoe = y - era * 400; + let doy = (153 * (m + if m > 2 { -3 } else { 9 }) + 2) / 5 + d - 1; + era * 146_097 + yoe * 365 + yoe / 4 - yoe / 100 + doy - 719_468 +} + +/// The uncompressed size a zstd frame header declares (RFC 8878 §3.1.1.1), +/// or None when the header does not carry it. Reads the first frame only; +/// the blob store writes one frame per blob. +pub fn zstd_content_size(head: &[u8]) -> Option { + if head.get(..4)? != [0x28, 0xB5, 0x2F, 0xFD] { + return None; + } + let fhd = *head.get(4)?; + let single = fhd & 0x20 != 0; + let at = 5 + usize::from(!single) + [0, 1, 2, 4][usize::from(fhd & 3)]; + let n = match (fhd >> 6, single) { + (0, true) => 1, + (0, false) => return None, + (1, _) => 2, + (2, _) => 4, + _ => 8, + }; + let v = head + .get(at..at + n)? + .iter() + .rev() + .fold(0u64, |a, b| (a << 8) | u64::from(*b)); + Some(if n == 2 { v + 256 } else { v }) +} + +/// One blob's (stored bytes, declared raw bytes); None when it is absent. +fn blob_size(root: &Path, sha: &str) -> Option<(u64, Option)> { + use std::io::Read as _; + let p = root + .join("blobs/sha256") + .join(&sha[..2]) + .join(&sha[2..4]) + .join(format!("{sha}.zst")); + let mut f = fs::File::open(p).ok()?; + let stored = f.metadata().ok()?.len(); + let mut head = [0u8; 18]; + let got = f.read(&mut head).ok()?; + Some((stored, zstd_content_size(&head[..got]))) +} + +#[derive(Default)] +struct WeekYield { + days: Vec, + rows: u64, + index_bytes: u64, + blobs: BTreeSet, +} + +/// The weekly yield/bytes receipt (PRM-C14, G16): per ISO week, quorums/day, +/// rows, index bytes and the CAS blobs the rows address, stored and raw. +/// A week with a missing blob, or a blob whose raw size is undeclared, reads +/// `unmeasured` — never `within`. +#[allow(clippy::disallowed_methods)] // `json!` over numbers and strings +pub fn weekly(s: &Snapshot, root: &Path) -> Result { + let mut weeks: BTreeMap = BTreeMap::new(); + for (day, d) in &s.days { + let w = weeks.entry(iso_week(day)?).or_default(); + w.days.push(d.quorums.len() as u64); + w.rows += d.rows; + w.index_bytes += d.index_bytes; + w.blobs.extend(d.blobs.iter().cloned()); + } + let rows: Vec = weeks.iter().map(|(k, w)| week_row(root, k, w)).collect(); + Ok(json!({ + "schema": WEEKLY_SCHEME, + "snapshot": s.id, + "g16": { + "quorums_per_day": [G16_QUORUMS_PER_DAY.0, G16_QUORUMS_PER_DAY.1], + "raw_tb_per_year_max": G16_RAW_TB_PER_YEAR, + "provenance": "[O] PRM-001 G16", + }, + "weeks": rows, + })) +} + +/// G16 per ISO week from a [`weekly`] receipt: `(week, "within" | "outside")`. +/// G16 moves from `[U]` to `[V]` only when every week is measured. +/// +/// # Errors +/// No weeks, a week whose G16 is `unmeasured` (named), or a malformed receipt. +pub fn g16_decided(weekly: &Value) -> Result, String> { + let weeks = weekly + .get("weeks") + .and_then(Value::as_array) + .ok_or("weekly receipt has no weeks array")?; + if weeks.is_empty() { + return Err("no week has rows: G16 stays [U]".into()); + } + let mut out = Vec::new(); + let mut open = Vec::new(); + for w in weeks { + let (Some(week), Some(g)) = ( + w.get("week").and_then(Value::as_str), + w.get("g16").and_then(Value::as_str), + ) else { + return Err(format!("malformed week row: {w}")); + }; + match g { + "within" | "outside" => out.push((week.to_string(), g.to_string())), + "unmeasured" => open.push(week.to_string()), + other => return Err(format!("{week}: unknown g16 verdict {other:?}")), + } + } + if open.is_empty() { + Ok(out) + } else { + Err(format!( + "unmeasured (missing or undeclared blobs): {}", + open.join(", ") + )) + } +} + +#[allow(clippy::disallowed_methods)] // `json!` over numbers and strings +fn week_row(root: &Path, week: &str, w: &WeekYield) -> Value { + let (mut stored, mut raw, mut missing, mut undeclared) = (0u64, 0u64, 0u64, 0u64); + for sha in &w.blobs { + match blob_size(root, sha) { + None => missing += 1, + Some((st, r)) => { + stored += st; + match r { + Some(r) => raw += r, + None => undeclared += 1, + } + } + } + } + let n = w.days.len() as u64; + let (lo, hi) = ( + w.days.iter().copied().min().unwrap_or(0), + w.days.iter().copied().max().unwrap_or(0), + ); + let raw_total = w.index_bytes + raw; + let tb_year = raw_total as f64 / n.max(1) as f64 * 365.0 / 1e12; + let verdict = if missing + undeclared > 0 { + "unmeasured" + } else if lo >= G16_QUORUMS_PER_DAY.0 + && hi <= G16_QUORUMS_PER_DAY.1 + && tb_year <= G16_RAW_TB_PER_YEAR + { + "within" + } else { + "outside" + }; + json!({ + "week": week, + "days_with_rows": n, + "rows": w.rows, + "quorums": w.days.iter().sum::(), + "quorums_per_day_min": lo, + "quorums_per_day_max": hi, + "index_bytes": w.index_bytes, + "blobs": w.blobs.len(), + "blobs_missing": missing, + "blobs_raw_undeclared": undeclared, + "blob_bytes_stored": stored, + "blob_bytes_raw": raw, + "raw_tb_per_year": tb_year, + "g16": verdict, + }) +} + +/// Deterministic JSON bytes for a card. +pub fn render(card: &Value) -> String { + let mut s = serde_json::to_string_pretty(card).unwrap_or_default(); + s.push('\n'); + s +} + +/// The Datasheet-style card (Gebru et al. sections), aggregate counts only. +pub fn datasheet(s: &Snapshot, m: &CardMeta) -> String { + let (first, last) = s.timeframe(); + let mut o = format!( + "# Datasheet: {} (snapshot {})\n\nGenerated by `{SCHEME}` from the index; do not edit.\n\n", + m.name, s.id + ); + o += "## Motivation\n\nPR-review lane traces for training and evaluating a local \ + review model against the quorum lanes (PRA-001).\n\n"; + o += &format!( + "## Composition\n\n- rows: {}\n- index files: {} ({first}..{last})\n- public-eligible \ + (gold, provider local): {}\n", + s.rows, + s.files.len(), + s.public_eligible + ); + for (col, _) in COUNTED { + for (v, n) in s.column(col) { + o += &format!("- {col} `{v}`: {n}\n"); + } + } + o += "\n## Collection process\n\nCaptured at dispatch, one agent-trace-v1 row per lane \ + per quorum round; full lane I/O lives in the almacen CAS, never in this card.\n\n"; + o += "## Preprocessing / labelling\n\nsilver = quorum verdict; gold = matured PR outcome; \ + pending until matured; quarantined on a secret-scan hit. Split by per-PR group key; \ + deduplicated by cluster.\n\n"; + o += "## Uses\n\nTraining and evaluating the local review model; lane-agreement research. \ + Not for evaluating people.\n\n"; + o += &format!( + "## Distribution\n\nPrivate. Public release carries gold rows with provider local only \ + ({} of {} rows in this snapshot); anthropic and google rows are never released.\n\n", + s.public_eligible, s.rows + ); + o += "## Maintenance\n\nRegenerated per index snapshot under `datacard//`; a card \ + that differs from its regeneration is refused as hand-edited.\n"; + o +} + +/// Write `datacard//{croissant.json, datasheet.md, manifest.sha256, weekly.json}`. +/// Regenerating an unchanged snapshot is a no-op; a card that differs from its +/// regeneration is refused as hand-edited and never overwritten. +pub fn write_snapshot(root: &Path, m: &CardMeta) -> Result { + let s = Snapshot::read(root)?; + let card = croissant(&s, m); + let errs = validate(&card); + if !errs.is_empty() { + return Err(format!( + "generated card fails validation: {}", + errs.join("; ") + )); + } + let dir = root.join(DATACARD_DIR).join(&s.id); + let files = [ + ("croissant.json", render(&card)), + ("datasheet.md", datasheet(&s, m)), + ("manifest.sha256", s.manifest()), + ("weekly.json", render(&weekly(&s, root)?)), + ]; + for (name, body) in &files { + let p = dir.join(name); + match fs::read(&p) { + Ok(got) if got != body.as_bytes() => { + return Err(format!( + "{} is hand-edited (differs from its regeneration); refusing to overwrite", + p.display() + )) + } + _ => {} + } + } + fs::create_dir_all(&dir).map_err(|e| format!("{}: {e}", dir.display()))?; + for (name, body) in &files { + let p = dir.join(name); + if p.exists() { + continue; + } + let mut f = fs::OpenOptions::new() + .write(true) + .create_new(true) + .open(&p) + .map_err(|e| format!("{}: {e}", p.display()))?; + f.write_all(body.as_bytes()) + .map_err(|e| format!("{}: {e}", p.display()))?; + } + Ok(dir) +} + +#[cfg(test)] +#[path = "datacard_tests.rs"] +mod tests; diff --git a/crates/aprender-review-experiment/src/datacard_tests.rs b/crates/aprender-review-experiment/src/datacard_tests.rs new file mode 100644 index 0000000000..128fabe3cc --- /dev/null +++ b/crates/aprender-review-experiment/src/datacard_tests.rs @@ -0,0 +1,464 @@ +// `json!` expands to an `unwrap` of an infallible `to_value`. +#![allow(clippy::disallowed_methods)] + +use super::*; + +use std::fs; +use std::path::PathBuf; + +/// A throwaway traces root under the target dir's tmp; removed on drop. +struct Root(PathBuf); + +impl Root { + fn new(tag: &str) -> Self { + let p = std::env::temp_dir().join(format!("tdc-{tag}-{}", std::process::id())); + let _ = fs::remove_dir_all(&p); + fs::create_dir_all(&p).expect("mkdir root"); + Root(p) + } +} + +impl Drop for Root { + fn drop(&mut self) { + let _ = fs::remove_dir_all(&self.0); + } +} + +/// Synthetic rows only: no real transcript or trace ever enters a fixture. +const SECRET_TEXT: &str = "SENTINEL-FINDING-TEXT-never-in-a-card"; +const SECRET_DIFF: &str = "0badc0de0badc0de0badc0de0badc0de0badc0de0badc0de0badc0de0badc0de"; + +fn row(id: &str, lane: &str, provider: &str, tier: &str, split: &str) -> String { + serde_json::json!({ + "schema": "agent-trace-v1", + "trace_id": id, + "quorum_id": "q1", + "round": 1, + "lane": lane, + "counted": provider != "local", + "provider": provider, + "model_id": format!("{lane}-2026-09-01"), + "access_channel": if provider == "local" { "apr-serve" } else { "claude-code-cli" }, + "repo": "paiml/aprender", + "pr": 4400, + "head_sha": "a".repeat(40), + "diff_sha256": SECRET_DIFF, + "input_sha": "b".repeat(64), + "output_sha": "c".repeat(64), + "prompt_version": "1.0.0", + "lane_state": "Verdict", + "verdict": "approve", + "findings": [{"file": "src/a.rs", "line_start": 1, "line_end": 2, + "severity": "low", "category": "style", "text": SECRET_TEXT}], + "parse_status": "ok", + "label_tier": tier, + "outcome": "merged", + "split_guard": {"group_key": "paiml/aprender#4400", "split": split}, + "secret_scan": {"status": "clean", "hits": 0}, + "trace_status": "ok", + "agreed": true, + "lane_disagreement": 0, + "unique_diff": true, + "dedup_cluster": "c0", + }) + .to_string() +} + +fn day(root: &Path, ymd: (&str, &str, &str), rows: &[String]) { + let d = root.join(INDEX_DIR).join(ymd.0).join(ymd.1); + fs::create_dir_all(&d).expect("mkdir day"); + let mut body = rows.join("\n"); + body.push('\n'); + fs::write(d.join(format!("{}.jsonl", ymd.2)), body).expect("write day"); +} + +fn fixture(tag: &str) -> Root { + let r = Root::new(tag); + day( + &r.0, + ("2026", "09", "24"), + &[ + row("t1", "sonnet", "anthropic", "gold", "train"), + row("t2", "agy", "google", "gold", "train"), + ], + ); + day( + &r.0, + ("2026", "09", "25"), + &[ + row("t3", "qwen-shadow", "local", "gold", "val"), + row("t4", "qwen-shadow", "local", "silver", "train"), + row("t5", "haiku", "anthropic", "pending", "train"), + ], + ); + r +} + +fn card_of(root: &Path) -> (Snapshot, Value) { + let s = Snapshot::read(root).expect("snapshot"); + let c = croissant(&s, &CardMeta::default()); + (s, c) +} + +#[test] +fn falsify_tdc_001_the_generated_card_passes_croissant_validation() { + let r = fixture("v"); + let (_, c) = card_of(&r.0); + assert_eq!(validate(&c), Vec::::new(), "{c:#}"); + + // The validator is not vacuous: each plant is refused, and names why. + let plants: [(&str, fn(&mut Value)); 9] = [ + ("conformsTo", |c| { + c.as_object_mut().expect("obj").remove("conformsTo"); + }), + ("name", |c| c["name"] = json!("")), + ("@type", |c| c["@type"] = json!("sc:CreativeWork")), + ("sha256", |c| { + c["distribution"][0] + .as_object_mut() + .expect("obj") + .remove("sha256"); + }), + ("unknown resource", |c| { + c["recordSet"][0]["field"][0]["source"]["fileSet"]["@id"] = json!("nope") + }), + ("duplicate @id", |c| { + let f = c["recordSet"][0]["field"][0].clone(); + c["recordSet"][0]["field"] + .as_array_mut() + .expect("arr") + .push(f); + }), + ("dataType", |c| { + c["recordSet"][0]["field"][0]["dataType"] = json!("sc:Banana") + }), + ("rai:dataLimitations", |c| { + c.as_object_mut() + .expect("obj") + .remove("rai:dataLimitations"); + }), + ("@context", |c| { + c["@context"].as_object_mut().expect("obj").remove("rai"); + }), + ]; + for (why, plant) in plants { + let mut bad = c.clone(); + plant(&mut bad); + let errs = validate(&bad); + assert!( + errs.iter().any(|e| e.contains(why)), + "plant {why:?} not refused: {errs:?}" + ); + } +} + +#[test] +fn falsify_tdc_002_a_snapshot_is_deterministic_and_bound_to_the_index_bytes() { + let r = fixture("det"); + let (s1, c1) = card_of(&r.0); + let (s2, c2) = card_of(&r.0); + assert_eq!(s1.id, s2.id); + assert_eq!(render(&c1), render(&c2), "same index, same bytes"); + assert_eq!( + datasheet(&s1, &CardMeta::default()), + datasheet(&s2, &CardMeta::default()) + ); + + // The root digest is over every index file's bytes. + let manifest = s1.manifest(); + assert_eq!(manifest.lines().count(), 2, "{manifest}"); + for line in manifest.lines() { + let (sha, rel) = line.split_once(" ").expect("sha path"); + let bytes = fs::read(r.0.join(rel)).expect("read"); + assert_eq!(sha, sha256_hex(&bytes), "{rel}"); + } + assert_eq!( + c1["distribution"][0]["sha256"], + json!(sha256_hex(manifest.as_bytes())) + ); + + // One appended row is a new snapshot. + day( + &r.0, + ("2026", "09", "26"), + &[row("t6", "sonnet", "anthropic", "silver", "test")], + ); + let (s3, c3) = card_of(&r.0); + assert_ne!(s1.id, s3.id); + assert_ne!( + c1["distribution"][0]["sha256"], + c3["distribution"][0]["sha256"] + ); + assert_eq!(s3.rows, 6); +} + +#[test] +fn falsify_tdc_003_only_gold_local_rows_are_public_eligible() { + let r = fixture("pub"); + let (s, c) = card_of(&r.0); + // t1 gold anthropic, t2 gold google: out. t3 gold local: in. t4 silver local, + // t5 pending anthropic: out. + assert_eq!(s.public_eligible, 1); + assert_eq!(s.count("provider", "anthropic"), 2); + assert_eq!(s.count("provider", "google"), 1); + assert_eq!(s.count("provider", "local"), 2); + assert_eq!(s.count("label_tier", "gold"), 3); + assert_eq!(s.count("split", "val"), 1); + let sheet = datasheet(&s, &CardMeta::default()); + assert!( + sheet.contains("public-eligible (gold, provider local): 1"), + "{sheet}" + ); + assert!(c["rai:personalSensitiveInformation"] + .as_str() + .expect("str") + .contains("quarantine")); + + // A secret-hit gold local row is never eligible. + let mut hit: Value = + serde_json::from_str(&row("t7", "qwen-shadow", "local", "gold", "train")).expect("json"); + hit["secret_scan"] = json!({"status": "hit", "hits": 1}); + day(&r.0, ("2026", "09", "27"), &[hit.to_string()]); + let s = Snapshot::read(&r.0).expect("snapshot"); + assert_eq!(s.public_eligible, 1); +} + +#[test] +fn falsify_tdc_004_a_hand_edited_card_is_refused_and_regeneration_is_idempotent() { + let r = fixture("edit"); + let out = write_snapshot(&r.0, &CardMeta::default()).expect("first write"); + assert!(out.join("croissant.json").is_file()); + assert!(out.join("datasheet.md").is_file()); + assert!(out.join("manifest.sha256").is_file()); + assert!(out.join("weekly.json").is_file()); + assert_eq!(out.parent().expect("parent"), r.0.join(DATACARD_DIR)); + + // Regenerating the same snapshot is a no-op, not an error. + assert_eq!( + write_snapshot(&r.0, &CardMeta::default()).expect("again"), + out + ); + + // A hand edit is refused on the next regeneration, never overwritten. + let p = out.join("croissant.json"); + let edited = fs::read_to_string(&p) + .expect("read") + .replace("agent-trace", "agent-trice"); + fs::write(&p, &edited).expect("edit"); + let e = write_snapshot(&r.0, &CardMeta::default()).expect_err("hand edit refused"); + assert!(e.contains("hand-edited"), "{e}"); + assert_eq!( + fs::read_to_string(&p).expect("read"), + edited, + "never overwritten" + ); +} + +#[test] +fn falsify_tdc_005_no_row_content_reaches_the_card() { + let r = fixture("priv"); + let (s, c) = card_of(&r.0); + let text = format!("{}\n{}", render(&c), datasheet(&s, &CardMeta::default())); + assert!( + !text.contains(SECRET_TEXT), + "a finding's text leaked into the card" + ); + assert!( + !text.contains(SECRET_DIFF), + "a diff sha leaked into the card" + ); + assert!( + !text.contains(&"c".repeat(64)), + "an output sha leaked into the card" + ); +} + +#[test] +fn falsify_tdc_006_a_malformed_index_is_refused_not_skipped() { + let r = fixture("bad"); + day(&r.0, ("2026", "09", "28"), &["{not json".to_string()]); + let e = Snapshot::read(&r.0).expect_err("bad json refused"); + assert!( + e.contains("2026/09/28.jsonl:1") && e.contains("bad json"), + "{e}" + ); + + let r = fixture("schema"); + let foreign = row("t9", "sonnet", "anthropic", "gold", "train") + .replace("agent-trace-v1", "agent-trace-v2"); + day(&r.0, ("2026", "09", "28"), &[foreign]); + let e = Snapshot::read(&r.0).expect_err("unknown major refused"); + assert!(e.contains("agent-trace-v2"), "{e}"); + + let empty = Root::new("empty"); + let e = Snapshot::read(&empty.0).expect_err("an empty index is not a dataset"); + assert!(e.contains("no index"), "{e}"); +} + +fn rowq(id: &str, quorum: &str) -> String { + let mut v: Value = + serde_json::from_str(&row(id, "sonnet", "anthropic", "silver", "train")).expect("json"); + v["quorum_id"] = json!(quorum); + v.to_string() +} + +/// A zstd frame header declaring `raw` bytes (single segment, 4-byte FCS). +fn blob(root: &Path, sha: &str, head: &[u8]) { + let d = root.join("blobs/sha256").join(&sha[..2]).join(&sha[2..4]); + fs::create_dir_all(&d).expect("mkdir blob"); + let mut b = head.to_vec(); + b.extend_from_slice(&[0u8; 10]); + fs::write(d.join(format!("{sha}.zst")), b).expect("write blob"); +} + +const ZSTD: [u8; 4] = [0x28, 0xB5, 0x2F, 0xFD]; + +fn declared(raw: u32) -> Vec { + let mut h = ZSTD.to_vec(); + h.push(0xA0); + h.extend_from_slice(&raw.to_le_bytes()); + h +} + +#[test] +fn falsify_tdc_007_the_weekly_receipt_measures_g16_or_says_unmeasured() { + let r = Root::new("weekly"); + let (b, c) = ("b".repeat(64), "c".repeat(64)); + day( + &r.0, + ("2026", "09", "24"), + &[rowq("t1", "q1"), rowq("t2", "q1"), rowq("t3", "q2")], + ); + day(&r.0, ("2026", "09", "25"), &[rowq("t4", "q3")]); + day(&r.0, ("2026", "09", "28"), &[rowq("t5", "q4")]); + blob(&r.0, &b, &declared(1000)); + + let s = Snapshot::read(&r.0).expect("snapshot"); + let w = weekly(&s, &r.0).expect("weekly"); + assert_eq!(w["schema"], json!(WEEKLY_SCHEME)); + let w39 = &w["weeks"][0]; + assert_eq!(w39["week"], json!("2026-W39")); + assert_eq!(w39["days_with_rows"], json!(2)); + assert_eq!(w39["rows"], json!(4)); + assert_eq!(w39["quorums"], json!(3)); + assert_eq!(w39["quorums_per_day_min"], json!(1)); + assert_eq!(w39["quorums_per_day_max"], json!(2)); + let idx: u64 = s.files[..2].iter().map(|f| f.bytes).sum(); + assert_eq!(w39["index_bytes"], json!(idx)); + assert_eq!(w39["blobs"], json!(2)); + assert_eq!(w39["blobs_missing"], json!(1)); + assert_eq!(w39["blob_bytes_stored"], json!(19)); + assert_eq!(w39["blob_bytes_raw"], json!(1000)); + assert_eq!( + w39["g16"], + json!("unmeasured"), + "a missing blob is not a measurement" + ); + assert_eq!(w["weeks"][1]["week"], json!("2026-W40")); + + // A blob whose header does not declare its size is still unmeasured. + blob(&r.0, &c, &[ZSTD[0], ZSTD[1], ZSTD[2], ZSTD[3], 0x00, 0x00]); + let w = weekly(&s, &r.0).expect("weekly"); + assert_eq!(w["weeks"][0]["blobs_raw_undeclared"], json!(1)); + assert_eq!(w["weeks"][0]["g16"], json!("unmeasured")); + + // Every blob measured: 1–2 quorums/day is outside the [O] 50–200 band. + blob(&r.0, &c, &declared(500)); + let w = weekly(&s, &r.0).expect("weekly"); + assert_eq!(w["weeks"][0]["blob_bytes_raw"], json!(1500)); + assert_eq!(w["weeks"][0]["g16"], json!("outside")); + let text = render(&w); + assert!( + !text.contains(&b) && !text.contains(SECRET_TEXT), + "no sha or row text" + ); + + // 50 quorums on one day, every blob measured and tiny: within. + let r = Root::new("weekly-in"); + let rows: Vec = (0..50) + .map(|i| rowq(&format!("t{i}"), &format!("q{i}"))) + .collect(); + day(&r.0, ("2026", "09", "24"), &rows); + blob(&r.0, &b, &declared(1000)); + blob(&r.0, &c, &declared(1000)); + let s = Snapshot::read(&r.0).expect("snapshot"); + assert_eq!( + weekly(&s, &r.0).expect("weekly")["weeks"][0]["g16"], + json!("within") + ); + + // In the quorum band but ~4.29 GB/day raw ≈ 1.57 TB/yr: outside the 0.45 TB cap. + blob(&r.0, &c, &declared(u32::MAX)); + let w = weekly(&s, &r.0).expect("weekly"); + assert!(w["weeks"][0]["raw_tb_per_year"].as_f64().expect("f64") > G16_RAW_TB_PER_YEAR); + assert_eq!(w["weeks"][0]["g16"], json!("outside")); +} + +#[test] +fn falsify_tdc_008_iso_weeks_and_zstd_sizes_match_their_standards() { + for (day, want) in [ + ("2026-09-24", "2026-W39"), + ("2026-09-28", "2026-W40"), + ("2026-01-01", "2026-W01"), + ("2027-01-01", "2026-W53"), + ("2021-01-03", "2020-W53"), + ("2024-12-30", "2025-W01"), + ("2020-02-29", "2020-W09"), + ] { + assert_eq!(iso_week(day).expect("day"), want, "{day}"); + } + assert!(iso_week("2026-9").is_err()); + + let z = |tail: &[u8]| [&ZSTD[..], tail].concat(); + assert_eq!( + zstd_content_size(&z(&[0x20, 5])), + Some(5), + "single segment, 1-byte FCS" + ); + assert_eq!( + zstd_content_size(&z(&[0x40, 0x58, 0, 1])), + Some(512), + "2-byte FCS is +256" + ); + assert_eq!( + zstd_content_size(&z(&[0xA1, 7, 0x10, 0, 0, 0])), + Some(16), + "dict id skipped" + ); + assert_eq!( + zstd_content_size(&z(&[0x00, 0x58])), + None, + "no FCS declared" + ); + assert_eq!(zstd_content_size(&[0, 1, 2, 3, 0x20, 5]), None, "not zstd"); + assert_eq!(zstd_content_size(&z(&[0x80, 1, 2])), None, "truncated"); +} + +/// FALSIFY-TDC-009 (PRM-C14 acceptance): G16 is decided only when every week is +/// measured. `within` and `outside` are both results; one `unmeasured` week, or +/// no week at all, keeps G16 at `[U]` and names the open week. +// `json!` over strings. +#[test] +#[allow(clippy::disallowed_methods)] +fn falsify_tdc_009_g16_is_decided_only_when_every_week_is_measured() { + let rx = |gs: &[(&str, &str)]| serde_json::json!({"weeks": gs.iter().map(|(w, g)| serde_json::json!({"week": w, "g16": g})).collect::>()}); + assert_eq!( + g16_decided(&rx(&[("2026-W38", "within"), ("2026-W39", "outside")])), + Ok(vec![ + ("2026-W38".to_string(), "within".to_string()), + ("2026-W39".to_string(), "outside".to_string()) + ]) + ); + let e = g16_decided(&rx(&[("2026-W38", "within"), ("2026-W39", "unmeasured")])) + .expect_err("an unmeasured week keeps G16 open"); + assert!(e.contains("2026-W39"), "{e}"); + assert!(g16_decided(&rx(&[])).is_err(), "no weeks is not a verdict"); + assert!( + g16_decided(&rx(&[("2026-W38", "fine")])).is_err(), + "unknown verdict" + ); + assert!( + g16_decided(&serde_json::json!({})).is_err(), + "no weeks array" + ); +} diff --git a/crates/aprender-review-experiment/src/dedup.rs b/crates/aprender-review-experiment/src/dedup.rs new file mode 100644 index 0000000000..fd3569a154 --- /dev/null +++ b/crates/aprender-review-experiment/src/dedup.rs @@ -0,0 +1,101 @@ +//! PRM-C8 `trace-dedup-v1`: near-duplicate clusters over trace diffs (gate +//! G-DUP; contract `trace-dedup-v1`, FALSIFY-TDD-001..004). +//! +//! Reuses the [`crate::cluster`] fingerprint (normalised token shingles, +//! bottom-k sketch, containment), so a re-indented or renamed copy of a diff +//! lands in its original's cluster, exactly as G-CON sees it. +//! +//! - **Candidates (LSH)**: an inverted index over sketch hashes; two items are +//! compared only if their sketches share a hash. Items whose sketches share +//! none are never compared — the recall bound, stated, not hidden. +//! - **Match**: either item's sketch is at least [`CONTAINMENT`] contained in +//! the other's shingle set. +//! - **Closure**: union-find, so near-dup chains close transitively. +//! - **Tiny diffs** (below `MIN_SHINGLES`, no sketch) merge only on identical +//! normalised tokens. +//! - **Cluster id**: the id of the earliest member by `(at, id)`, so the id +//! does not depend on input order. The split guard (PRM-C9) assigns a whole +//! cluster to that member's split. + +use std::collections::{BTreeMap, HashMap, HashSet}; + +use crate::cluster::{containment, shingles, tokens, CONTAINMENT, MIN_SHINGLES, SKETCH}; + +pub const SCHEME: &str = "trace-dedup-v1"; + +/// One diff to cluster: its id (a `diff_sha256`), first-seen time (RFC 3339 +/// UTC, so it orders as text) and the diff text. +#[derive(Debug, Clone, Copy)] +pub struct Item<'a> { + pub id: &'a str, + pub at: &'a str, + pub diff: &'a str, +} + +fn root(parent: &mut [usize], mut i: usize) -> usize { + while parent[i] != i { + parent[i] = parent[parent[i]]; + i = parent[i]; + } + i +} + +fn union(parent: &mut [usize], a: usize, b: usize) { + let (ra, rb) = (root(parent, a), root(parent, b)); + parent[ra.max(rb)] = ra.min(rb); +} + +/// The cluster id of each item, in input order. +#[must_use] +pub fn clusters(items: &[Item]) -> Vec { + let sets: Vec> = items.iter().map(|i| shingles(i.diff)).collect(); + let sketches: Vec>> = sets + .iter() + .map(|s| { + (s.len() >= MIN_SHINGLES).then(|| { + let mut v: Vec = s.iter().copied().collect(); + v.sort_unstable(); + v.truncate(SKETCH); + v + }) + }) + .collect(); + let mut parent: Vec = (0..items.len()).collect(); + let mut postings: HashMap> = HashMap::new(); + let mut tiny: HashMap, usize> = HashMap::new(); + for (i, it) in items.iter().enumerate() { + let Some(sk) = &sketches[i] else { + let first = *tiny.entry(tokens(it.diff)).or_insert(i); + union(&mut parent, first, i); + continue; + }; + let mut seen = HashSet::new(); + for h in sk { + let js = postings.entry(*h).or_default(); + for &j in js.iter().filter(|&&j| seen.insert(j)) { + let other = sketches[j].as_deref().unwrap_or_default(); + if containment(sk, &sets[j]) >= CONTAINMENT + || containment(other, &sets[i]) >= CONTAINMENT + { + union(&mut parent, i, j); + } + } + js.push(i); + } + } + let mut earliest: BTreeMap = BTreeMap::new(); + for i in 0..items.len() { + let r = root(&mut parent, i); + let e = earliest.entry(r).or_insert(i); + if (items[i].at, items[i].id) < (items[*e].at, items[*e].id) { + *e = i; + } + } + (0..items.len()) + .map(|i| items[earliest[&root(&mut parent, i)]].id.to_owned()) + .collect() +} + +#[cfg(test)] +#[path = "dedup_tests.rs"] +mod tests; diff --git a/crates/aprender-review-experiment/src/dedup_tests.rs b/crates/aprender-review-experiment/src/dedup_tests.rs new file mode 100644 index 0000000000..f2751ad6ba --- /dev/null +++ b/crates/aprender-review-experiment/src/dedup_tests.rs @@ -0,0 +1,106 @@ +use super::*; + +const A: &str = "--- a/crates/x/src/sum.rs\n+++ b/crates/x/src/sum.rs\n@@ -40,12 +40,14 @@ impl Ledger {\n pub fn total(&self, rows: &[Row]) -> u64 {\n let mut total = 0u64;\n for row in rows {\n- if row.amount > 0 {\n- total += row.amount;\n+ if row.amount > 0 && !row.voided {\n+ total = total.saturating_add(row.amount);\n }\n }\n+ debug_assert!(total >= self.floor);\n total\n }\n"; + +const B: &str = "--- a/crates/y/src/parse.rs\n+++ b/crates/y/src/parse.rs\n@@ -8,9 +8,11 @@\n pub fn parse(s: &str) -> Result {\n let mut it = s.split(':');\n- let key = it.next().ok_or(Error::Empty)?;\n+ let key = it.next().ok_or(Error::Empty)?.trim();\n+ if key.is_empty() {\n+ return Err(Error::Empty);\n+ }\n let value = it.next().unwrap_or_default();\n Ok(Header { key: key.into(), value: value.into() })\n }\n"; + +/// `A` re-pathed, re-indented and renamed: the same change to G-CON. +fn a_perturbed() -> String { + A.replace("crates/x/src/sum.rs", "crates/q/src/acc.rs") + .replace(" ", " ") + .replace("total", "acc") + .replace("row", "entry") +} + +fn item<'a>(id: &'a str, at: &'a str, diff: &'a str) -> Item<'a> { + Item { id, at, diff } +} + +#[test] +fn falsify_tdd_001_a_perturbed_copy_joins_its_original_and_an_unrelated_diff_does_not() { + let p = a_perturbed(); + let items = [ + item("a", "2026-09-01T00:00:00Z", A), + item("b", "2026-09-02T00:00:00Z", B), + item("p", "2026-09-03T00:00:00Z", &p), + ]; + assert_eq!(clusters(&items), ["a", "b", "a"]); + // A diff embedded in a later, larger one: either arrival order. + let big = format!("{A}{}", segs(1, 5)); + let early = [ + item("a", "2026-09-01T00:00:00Z", A), + item("g", "2026-09-02T00:00:00Z", &big), + ]; + assert_eq!(clusters(&early), ["a", "a"]); + let late = [early[1], early[0]]; + assert_eq!(clusters(&late), ["a", "a"]); +} + +#[test] +fn falsify_tdd_002_the_cluster_id_is_the_earliest_member_whatever_the_order() { + let p = a_perturbed(); + // The copy was seen first: it names the cluster, and input order is moot. + let items = [ + item("a", "2026-09-05T00:00:00Z", A), + item("b", "2026-09-02T00:00:00Z", B), + item("p", "2026-09-03T00:00:00Z", &p), + ]; + assert_eq!(clusters(&items), ["p", "b", "p"]); + let rev: Vec = items.iter().rev().copied().collect(); + assert_eq!(clusters(&rev), ["p", "b", "p"]); + // Equal times: the smaller id wins. + let tie = [ + item("z", "2026-09-01T00:00:00Z", A), + item("y", "2026-09-01T00:00:00Z", &p), + ]; + assert_eq!(clusters(&tie), ["y", "y"]); +} + +/// Segments `from..to` of a synthetic diff; each segment's literals are its own. +fn segs(from: u32, to: u32) -> String { + (from..to) + .flat_map(|k| { + (0..4).map(move |j| { + format!("+ let v = w.get({k}).map(|x| x * {k}{j}).unwrap_or({j});\n") + }) + }) + .collect() +} + +#[test] +fn falsify_tdd_003_near_dup_chains_close_transitively() { + // X~Y and Y~Z share 8 of 10 segments; X and Z share only 6. + let (x, y, z) = (segs(1, 11), segs(3, 13), segs(5, 15)); + let xz = [ + item("x", "2026-09-01T00:00:00Z", &x), + item("z", "2026-09-03T00:00:00Z", &z), + ]; + assert_eq!( + clusters(&xz), + ["x", "z"], + "premise: X and Z alone are not near-dups" + ); + let chain = [ + item("z", "2026-09-03T00:00:00Z", &z), + item("x", "2026-09-01T00:00:00Z", &x), + item("b", "2026-09-02T00:00:00Z", B), + item("y", "2026-09-04T00:00:00Z", &y), + ]; + assert_eq!(clusters(&chain), ["x", "x", "b", "x"]); +} + +#[test] +fn falsify_tdd_004_tiny_diffs_merge_only_when_identical() { + let t1 = "@@ -1,4 +1,4 @@\n fn f(x: u32) -> bool {\n- x > 1\n+ x >= 1\n }\n"; + let t1_respaced = "@@ -1,4 +1,4 @@\n fn g(y: u32) -> bool {\n- y > 1\n+ y >= 1\n }\n"; + let t2 = "@@ -1,4 +1,4 @@\n fn f(x: u32) -> bool {\n- x > 1\n+ x > 2\n }\n"; + let items = [ + item("t1", "2026-09-01T00:00:00Z", t1), + item("t2", "2026-09-02T00:00:00Z", t2), + item("t3", "2026-09-03T00:00:00Z", t1_respaced), + item("e", "2026-09-04T00:00:00Z", ""), + item("f", "2026-09-05T00:00:00Z", ""), + ]; + // Two empty diffs are the same (empty) text; a tiny edit is not. + assert_eq!(clusters(&items), ["t1", "t2", "t1", "e", "e"]); +} diff --git a/crates/aprender-review-experiment/src/fewshot.rs b/crates/aprender-review-experiment/src/fewshot.rs new file mode 100644 index 0000000000..8f0cf356ec --- /dev/null +++ b/crates/aprender-review-experiment/src/fewshot.rs @@ -0,0 +1,156 @@ +//! REX-11 B1 challengers (§5.3): prompt versions (B1a) and retrieval +//! few-shot (B1b), built so that the sealed test split cannot leak (R-2). +//! +//! - The retrieval pool is gold specimens outside the test split. A test +//! item offered to [`select`] is an error, not a skipped row: the caller +//! read a sealed diff, and that is the violation. +//! - The query never retrieves itself (same id or same diff sha), so a dev +//! evaluation is not an open-book test. +//! - Similarity is the Jaccard index of changed-line token bigrams: +//! deterministic, no model, ties broken by id. +//! - Every prompt a challenger sends, B1a file or rendered B1b prompt, passes +//! [`guard`]: any sealed diff sha or hunk inside it is a hard FAIL +//! (`review-corpus-contamination-v1`). + +use std::collections::BTreeSet; + +use crate::contamination::Index; +use crate::corpus::{Item, Split}; + +/// Token bigrams of the changed lines (`+`/`-`, not the file headers). +fn shingles(diff: &str) -> BTreeSet<(String, String)> { + let toks: Vec = diff + .lines() + .filter(|l| { + (l.starts_with('+') || l.starts_with('-')) + && !l.starts_with("+++") + && !l.starts_with("---") + }) + .flat_map(|l| { + l[1..] + .split(|c: char| !c.is_alphanumeric() && c != '_') + .filter(|t| !t.is_empty()) + .map(str::to_string) + .collect::>() + }) + .collect(); + toks.windows(2) + .map(|w| (w[0].clone(), w[1].clone())) + .collect() +} + +/// Jaccard index of two diffs' changed-line bigrams; 0 when either is empty. +#[must_use] +pub fn similarity(a: &str, b: &str) -> f64 { + let (x, y) = (shingles(a), shingles(b)); + let union = x.union(&y).count(); + if union == 0 { + return 0.0; + } + x.intersection(&y).count() as f64 / union as f64 +} + +/// The `k` pool items most similar to the query, most similar first. +pub fn select<'a>( + query: &Item, + query_diff: &str, + pool: &'a [(Item, String)], + k: usize, +) -> Result, String> { + if let Some((i, _)) = pool.iter().find(|(i, _)| i.split == Split::Test) { + return Err(format!( + "{}: a sealed test item in the retrieval pool (R-2)", + i.id + )); + } + let mut ranked: Vec<(f64, &(Item, String))> = pool + .iter() + .filter(|(i, _)| i.id != query.id && i.diff_sha256 != query.diff_sha256) + .map(|p| (similarity(query_diff, &p.1), p)) + .collect(); + ranked.sort_by(|a, b| b.0.total_cmp(&a.0).then_with(|| a.1 .0.id.cmp(&b.1 .0.id))); + Ok(ranked.into_iter().take(k).map(|(_, p)| p).collect()) +} + +/// The gold answer shown for an example: the verdict line, then the defect +/// locations for a defect. +fn gold(item: &Item) -> String { + if item.class.is_defect() { + let locs: Vec = item + .defect + .iter() + .map(|l| format!("- defect at {}:{}", l.file, l.line)) + .collect(); + format!("VERDICT: FAIL\n{}", locs.join("\n")) + } else { + "VERDICT: PASS".into() + } +} + +/// `base`, then the examples with their gold answers. The item under review +/// is appended by `harness::compose` as before. +#[must_use] +pub fn render(base: &str, examples: &[&(Item, String)]) -> String { + let mut s = base.trim_end().to_string(); + if examples.is_empty() { + return s + "\n"; + } + s.push_str("\n\nReviewed examples:\n"); + for (n, (item, diff)) in examples.iter().enumerate() { + let nl = if diff.ends_with('\n') { "" } else { "\n" }; + s.push_str(&format!( + "\nExample {}:\n```diff\n{diff}{nl}```\n{}\n", + n + 1, + gold(item) + )); + } + s.push_str("\nNow review this diff.\n"); + s +} + +/// The bodies of the prompt's fenced blocks. Text after a fence (a gold +/// `- defect at …` line, prose starting with `-`) would otherwise join the +/// last hunk and change its fingerprint, so each block is scanned alone. +fn fenced(prompt: &str) -> Vec { + let mut out = Vec::new(); + let mut cur: Option = None; + for l in prompt.lines() { + if l.trim_start().starts_with("```") { + match cur.take() { + Some(b) => out.push(b), + None => cur = Some(String::new()), + } + } else if let Some(b) = cur.as_mut() { + b.push_str(l); + b.push('\n'); + } + } + out.extend(cur); + out +} + +/// Refuse a prompt that carries any sealed item, whether in running text or +/// inside a fenced block. An empty index proves nothing, so it is refused too. +pub fn guard(prompt: &str, sealed: &Index, what: &str) -> Result<(), String> { + if sealed.is_empty() { + return Err("no sealed manifest indexed: the guard would prove nothing".into()); + } + let mut hits = sealed.scan(prompt, what); + for block in fenced(prompt) { + hits.extend(sealed.scan(&block, what)); + } + hits.sort(); + hits.dedup(); + if hits.is_empty() { + Ok(()) + } else { + Err(format!( + "{what}: {} sealed test leak(s): {hits:?}", + hits.len() + )) + } +} + +#[cfg(test)] +#[path = "fewshot_tests.rs"] +mod tests; diff --git a/crates/aprender-review-experiment/src/fewshot_tests.rs b/crates/aprender-review-experiment/src/fewshot_tests.rs new file mode 100644 index 0000000000..41f9e3ee97 --- /dev/null +++ b/crates/aprender-review-experiment/src/fewshot_tests.rs @@ -0,0 +1,121 @@ +use super::*; +use crate::corpus::{hunk_fingerprints, sha256_hex, Class, Loc, Sealed}; + +fn diff(body: &str) -> String { + format!("--- a/src/a.rs\n+++ b/src/a.rs\n@@ -10,5 +10,5 @@\n fn f(x: u32) -> bool {{\n{body} }}\n \n") +} + +fn item(id: &str, class: Class, split: Split, d: &str) -> (Item, String) { + let mut i = Item::new(id.into(), class, "src".into(), d); + i.split = split; + if class.is_defect() { + i.defect = vec![Loc { + file: "src/a.rs".into(), + line: 11, + }]; + } + (i, d.to_string()) +} + +fn sealed(test: &[&(Item, String)]) -> Index { + Index::new( + &test + .iter() + .map(|(i, d)| Sealed { + id: i.id.clone(), + diff_sha256: sha256_hex(d.as_bytes()), + hunks: hunk_fingerprints(d), + }) + .collect::>(), + ) +} + +fn pool() -> Vec<(Item, String)> { + vec![ + item( + "P01", + Class::P, + Split::Dev, + &diff("- x > 1\n+ x >= 1\n"), + ), + item( + "G01", + Class::G, + Split::Dev, + &diff("- let y = 2;\n+ let y = 3;\n"), + ), + item( + "R01", + Class::R, + Split::Dev, + &diff("- x > 1 && x < 9\n+ x >= 1 && x < 9\n"), + ), + ] +} + +#[test] +fn falsify_rxc_004_a_test_item_in_the_pool_is_an_error() { + let mut p = pool(); + p.push(item( + "T01", + Class::R, + Split::Test, + &diff("- a\n+ b\n"), + )); + let (q, qd) = item( + "Q", + Class::P, + Split::Test, + &diff("- x > 2\n+ x >= 2\n"), + ); + let e = select(&q, &qd, &p, 2).expect_err("sealed item in pool"); + assert!(e.contains("T01") && e.contains("R-2"), "{e}"); +} + +#[test] +fn falsify_rxc_005_retrieval_ranks_by_similarity_and_never_returns_the_query() { + let p = pool(); + let (q, qd) = item( + "Q", + Class::P, + Split::Test, + &diff("- x > 1 && x < 9 || y\n+ x >= 1 && x < 9 || y\n"), + ); + let got: Vec<&str> = select(&q, &qd, &p, 2) + .expect("selects") + .iter() + .map(|(i, _)| i.id.as_str()) + .collect(); + assert_eq!(got, ["R01", "P01"]); + // A dev query retrieves neither itself nor a byte-identical copy. + let self_copy = select(&p[2].0, &p[2].1, &p, 3).expect("selects"); + assert!(self_copy.iter().all(|(i, _)| i.id != "R01")); + assert_eq!(similarity("", &p[0].1), 0.0); +} + +#[test] +fn falsify_rxc_006_a_rendered_prompt_carrying_a_test_hunk_is_refused() { + let p = pool(); + let test = item( + "T07", + Class::R, + Split::Test, + &diff("- z * 2\n+ z * 3\n"), + ); + let ix = sealed(&[&test]); + let ok = render("BASE", &[&p[0], &p[1]]); + assert!(ok.contains("VERDICT: FAIL\n- defect at src/a.rs:11")); + assert!(ok.contains("VERDICT: PASS")); + guard(&ok, &ix, "prompt").expect("clean prompt"); + // The same prompt with the sealed item smuggled in as an example. + let leaked = render("BASE", &[&p[0], &test]); + // The gold line after the fence joins the hunk in running text: the + // whole-prompt scan alone is blind here, which is why guard scans blocks. + assert!(ix.scan(&leaked, "prompt").is_empty()); + let e = guard(&leaked, &ix, "prompt").expect_err("leak"); + assert!(e.contains("T07"), "{e}"); + assert!( + guard(&ok, &Index::default(), "prompt").is_err(), + "an empty index proves nothing" + ); +} diff --git a/crates/aprender-review-experiment/src/harness.rs b/crates/aprender-review-experiment/src/harness.rs index 576df9b716..28ceb263d4 100644 --- a/crates/aprender-review-experiment/src/harness.rs +++ b/crates/aprender-review-experiment/src/harness.rs @@ -12,8 +12,12 @@ use crate::receipt::{ }; use crate::stats::SplitMix64; use serde_json::{json, Value}; +use std::io::Read; use std::time::{Duration, Instant}; +/// Largest reply body read, in bytes (ureq's `into_string` stops at 10 MB). +const MAX_BODY: u64 = 256 << 20; + /// Fixed decoding (§2.1 `[A]`): greedy, seed = the epic number, 512 tokens /// (the prompt asks for a verdict line and ≤ 5 short bullets). pub const DECODING: Decoding = Decoding { @@ -109,7 +113,9 @@ pub fn post(url: &str, body: &Value) -> Result { Err(ureq::Error::Status(code, r)) => (code, r), Err(e) => return Err(e.to_string()), }; - let body = resp.into_string().map_err(|e| e.to_string())?; + // `into_string` refuses bodies over 10 MB; the /tokenize reply for a large + // candidate diff exceeds that. Read bounded at MAX_BODY instead. + let body = read_bounded(resp.into_reader(), MAX_BODY)?; Ok(Reply { status, body, @@ -117,6 +123,30 @@ pub fn post(url: &str, body: &Value) -> Result { }) } +/// Read `r` to a string of at most `cap` bytes. +/// +/// # Errors +/// The read failed, or the body ran past `cap`. +pub fn read_bounded(r: impl Read, cap: u64) -> Result { + let mut limited = r.take(cap); + let mut bytes = Vec::new(); + limited.read_to_end(&mut bytes).map_err(|e| e.to_string())?; + // Probe the reader itself, not a cap + 1 limit: Ok means it hit EOF, for + // every cap, u64::MAX included. + let mut probe = [0u8; 1]; + let more = loop { + match limited.get_mut().read(&mut probe) { + Ok(n) => break n > 0, + Err(e) if e.kind() == std::io::ErrorKind::Interrupted => {} + Err(e) => return Err(e.to_string()), + } + }; + if more { + return Err(format!("reply body ran past the {cap}-byte cap")); + } + String::from_utf8(bytes).map_err(|e| e.to_string()) +} + /// The reply's text, token counts, server timings — or why none. #[derive(Debug, Clone, PartialEq)] pub struct Parsed { diff --git a/crates/aprender-review-experiment/src/harness_tests.rs b/crates/aprender-review-experiment/src/harness_tests.rs index 6653883abe..0852e11184 100644 --- a/crates/aprender-review-experiment/src/harness_tests.rs +++ b/crates/aprender-review-experiment/src/harness_tests.rs @@ -190,6 +190,40 @@ fn rfc3339_by_hand() { assert_eq!(rfc3339(4_107_542_399), "2100-02-28T23:59:59Z"); } +/// A bounded read refuses a body past its cap instead of truncating it (#4459 +/// quorum r2, 3/3 lanes): a cut-off reply returned as `Ok` would be scored as +/// if it were the whole one. +#[test] +fn read_bounded_refuses_past_its_cap_and_never_truncates() { + let table: &[(&[u8], u64, Result<&str, ()>)] = &[ + (b"", 4, Ok("")), + (b"abc", 4, Ok("abc")), + (b"abcd", 4, Ok("abcd")), + (b"abcde", 4, Err(())), + (b"abcdefgh", 4, Err(())), + ]; + for &(input, cap, ref want) in table { + let got = read_bounded(input, cap); + match want { + Ok(s) => assert_eq!(got.as_deref(), Ok(*s), "input={input:?} cap={cap}"), + Err(()) => assert!( + got.as_ref().is_err_and(|e| e.contains("cap")), + "input={input:?} cap={cap} got={got:?}" + ), + } + } +} + +/// Invalid UTF-8 inside the cap is refused, never returned lossily; a body that +/// ends exactly at the cap is whole, whatever the cap (#4459 quorum r3). +#[test] +fn read_bounded_refuses_invalid_utf8_and_takes_a_body_ending_at_any_cap() { + assert!(read_bounded(&[0x61, 0xff][..], 4).is_err()); + assert_eq!(read_bounded(&b"abcd"[..], u64::MAX).as_deref(), Ok("abcd")); + assert_eq!(read_bounded(&b""[..], 0).as_deref(), Ok("")); + assert!(read_bounded(&b"a"[..], 0).is_err_and(|e| e.contains("cap"))); +} + #[test] fn request_body_carries_the_fixed_decoding_and_composed_message() { let b = request_body("m", "P\n", "d\n"); diff --git a/crates/aprender-review-experiment/src/kappa_probe.rs b/crates/aprender-review-experiment/src/kappa_probe.rs new file mode 100644 index 0000000000..1dd1c46fde --- /dev/null +++ b/crates/aprender-review-experiment/src/kappa_probe.rs @@ -0,0 +1,414 @@ +//! PRM-C13 (was PRA-001 T13): the lane κ_err probe and its independence gate +//! (PRM-001 v3 §3 H7 and S-14; contract +//! `lane-independence-v1`). +//! +//! κ_err is Cohen's κ on binary per-item error indicators against GOLD, for +//! every (shadow lane, counted lane) pair, over matured gold rows of one split. +//! It reuses the prereg-locked [`crate::stats::error_kappa`]; this module only +//! decides which items count and what an error is. +//! +//! - An item is one `(quorum_id, round)` whose rows are `label_tier = gold`, +//! matured (`outcome_matured_at` non-null), on the requested split, with +//! `trace_status = ok` and a settled outcome: `merged` is clean, +//! `reverted_le14d` / `regression_escape` is a defect. `pending` and +//! `closed_unmerged` carry no gold label and are excluded. +//! - A lane flags a defect with `request_changes` and clears it with `approve`. +//! Any other verdict, a `failed` parse or a non-`Verdict` state is UNPARSED: +//! the item is excluded for that pair, never counted as an error. +//! - A pair with fewer than `min_n` shared items, or with κ undefined, is +//! `insufficient`: it never prints a number (S-14). +//! +//! The gate is H7 as PRM-001 v3 §3 states it, δ-free: on the items the shadow +//! lane and EVERY counted lane scored, κ_err(shadow, V) ≤ the largest +//! voter–voter κ_err for each voter V, decided by the prereg-locked +//! [`crate::stats::h7_holds`]. It refuses (S-14) below `min_n` common items, +//! on a voter with no rows, and when `h7_holds` is undecided. The voter set and +//! `min_n` come only from the run manifest, registered strictly before +//! training started. + +use serde::Serialize; +use serde_json::Value; +use std::collections::{BTreeMap, BTreeSet}; + +/// The fields of one agent-trace-v1 row the probe reads. +#[derive(Debug, Clone)] +pub struct Row { + /// `quorum_id` + pub quorum_id: String, + /// `round` + pub round: u64, + /// `lane` + pub lane: String, + /// `counted` + pub counted: bool, + /// `true` = flagged a defect, `false` = cleared it, `None` = unparsed. + pub flag: Option, + /// `true` = gold defect, `false` = clean, `None` = no gold label. + pub defect: Option, + /// Gold, matured, `trace_status = ok`. + pub gold_matured: bool, + /// `split_guard.split` + pub split: String, +} + +fn text<'a>(v: &'a Value, k: &str) -> &'a str { + v.get(k).and_then(Value::as_str).unwrap_or("") +} + +fn row_of(v: &Value) -> Result { + let quorum_id = v + .get("quorum_id") + .and_then(Value::as_str) + .ok_or("no quorum_id")? + .to_string(); + let round = v.get("round").and_then(Value::as_u64).ok_or("no round")?; + let lane = v + .get("lane") + .and_then(Value::as_str) + .ok_or("no lane")? + .to_string(); + let parsed = + text(v, "lane_state") == "Verdict" && matches!(text(v, "parse_status"), "ok" | "repaired"); + let flag = match (parsed, text(v, "verdict")) { + (true, "request_changes") => Some(true), + (true, "approve") => Some(false), + _ => None, + }; + let defect = match text(v, "outcome") { + "merged" => Some(false), + "reverted_le14d" | "regression_escape" => Some(true), + _ => None, + }; + let matured = v.get("outcome_matured_at").is_some_and(|m| !m.is_null()); + Ok(Row { + quorum_id, + round, + lane, + counted: v.get("counted").and_then(Value::as_bool).unwrap_or(false), + flag, + defect, + gold_matured: text(v, "label_tier") == "gold" && matured && text(v, "trace_status") == "ok", + split: v + .get("split_guard") + .map(|s| text(s, "split")) + .unwrap_or("") + .to_string(), + }) +} + +/// Parse agent-trace-v1 JSONL. A malformed line is an error naming its line +/// number, never a silently dropped row. +pub fn parse_rows(jsonl: &str) -> Result, String> { + jsonl + .lines() + .enumerate() + .filter(|(_, l)| !l.trim().is_empty()) + .map(|(i, l)| { + serde_json::from_str::(l) + .map_err(|e| e.to_string()) + .and_then(|v| row_of(&v)) + .map_err(|e| format!("line {}: {e}", i + 1)) + }) + .collect() +} + +/// Whether a pair produced a κ. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum PairStatus { + /// κ_err is defined on at least `min_n` shared items. + Ok, + /// Too few shared items, or κ undefined (S-14). + Insufficient, +} + +/// κ_err of one (shadow lane, counted lane) pair. +#[derive(Debug, Clone, Serialize)] +pub struct PairKappa { + /// The counted lane. + pub counted_lane: String, + /// Shared scored items. + pub n: usize, + /// `None` unless `status` is `Ok`. + pub kappa_err: Option, + /// See [`PairStatus`]. + pub status: PairStatus, +} + +/// Every pair's κ_err for one shadow lane on one split. +#[derive(Debug, Clone, Serialize)] +pub struct Scoreboard { + /// Always `lane-independence-v1`. + pub schema: &'static str, + /// The split scored. + pub split: String, + /// The shadow lane (the candidate side of every pair). + pub shadow_lane: String, + /// The shared-item floor applied. + pub min_n: usize, + /// One entry per counted lane present in the rows, sorted by lane. + pub pairs: Vec, +} + +type Items<'a> = BTreeMap<(&'a str, u64), BTreeMap<&'a str, Vec<&'a Row>>>; + +/// Matured gold `split` rows grouped item → lane → rows. +fn items_of<'a>(rows: &'a [Row], split: &str) -> Items<'a> { + let mut items: Items<'a> = BTreeMap::new(); + for r in rows + .iter() + .filter(|r| r.gold_matured && r.split == split && r.defect.is_some()) + { + items + .entry((&r.quorum_id, r.round)) + .or_default() + .entry(&r.lane) + .or_default() + .push(r); + } + items +} + +/// A lane's `(flag, defect)` on one item; `None` if absent, unparsed, or +/// ambiguous (two rows for one item). +fn one(m: &BTreeMap<&str, Vec<&Row>>, l: &str) -> Option<(bool, bool)> { + match m.get(l).map(Vec::as_slice) { + Some([r]) => Some((r.flag?, r.defect?)), + _ => None, + } +} + +/// κ_err of `shadow` against every counted lane, on matured gold `split` rows. +#[must_use] +pub fn probe(rows: &[Row], split: &str, shadow: &str, min_n: usize) -> Scoreboard { + let items = items_of(rows, split); + let counted: BTreeSet<&str> = rows + .iter() + .filter(|r| r.gold_matured && r.split == split && r.defect.is_some()) + .filter(|r| r.counted && r.lane != shadow) + .map(|r| r.lane.as_str()) + .collect(); + let pairs = counted + .into_iter() + .map(|lane| { + let (mut a, mut b) = (Vec::new(), Vec::new()); + for m in items.values() { + if let (Some((fs, ds)), Some((fl, dl))) = (one(m, shadow), one(m, lane)) { + // Both rows describe one PR; disagreeing gold is not gold. + if ds == dl { + a.push(fs != ds); + b.push(fl != dl); + } + } + } + let kappa_err = (a.len() >= min_n) + .then(|| crate::stats::error_kappa(&a, &b)) + .flatten(); + PairKappa { + counted_lane: lane.to_string(), + n: a.len(), + kappa_err, + status: if kappa_err.is_some() { + PairStatus::Ok + } else { + PairStatus::Insufficient + }, + } + }) + .collect(); + Scoreboard { + schema: "lane-independence-v1", + split: split.to_string(), + shadow_lane: shadow.to_string(), + min_n, + pairs, + } +} + +/// The `lane_independence` block of a run manifest. H7 is δ-free (PRM-001 v3 +/// §3), so what is pre-registered is the voter set and the item floor. +#[derive(Debug, Clone, Serialize)] +pub struct Manifest { + /// Shared-item floor for the gate to decide. + pub min_n: usize, + /// The counted lanes (voters) the gate must see on every item. + pub counted_lanes: Vec, + /// When the block was registered (`YYYY-MM-DDTHH:MM:SSZ`). + pub registered_at: String, + /// When candidate training started (`YYYY-MM-DDTHH:MM:SSZ`). + pub training_started_at: String, +} + +/// Strict UTC `YYYY-MM-DDTHH:MM:SSZ`, so string order is time order. +fn utc_stamp(s: &str) -> bool { + let b = s.as_bytes(); + b.len() == 20 + && b.iter().enumerate().all(|(i, &c)| match i { + 4 | 7 => c == b'-', + 10 => c == b'T', + 13 | 16 => c == b':', + 19 => c == b'Z', + _ => c.is_ascii_digit(), + }) +} + +/// Parse and validate a run manifest's `lane_independence` block. A missing +/// or unusable field, a block registered after training started, or a `delta` +/// (v2's rule, gone in v3: nothing would read it) is an error. +pub fn parse_manifest(json: &str) -> Result { + let v: Value = serde_json::from_str(json).map_err(|e| format!("manifest: {e}"))?; + let li = v + .get("lane_independence") + .ok_or("manifest has no lane_independence block")?; + if li.get("delta").is_some() { + return Err( + "lane_independence.delta: H7 is δ-free in PRM-001 v3 (κ(qwen, V) ≤ max voter–voter κ); remove it" + .into(), + ); + } + let min_n = li + .get("min_n") + .and_then(Value::as_u64) + .filter(|&n| n > 0) + .ok_or("lane_independence.min_n must be a positive integer")?; + let counted_lanes: Vec = li + .get("counted_lanes") + .and_then(Value::as_array) + .map(|a| { + a.iter() + .filter_map(|l| l.as_str().map(str::to_string)) + .collect() + }) + .filter(|a: &Vec| !a.is_empty()) + .ok_or("lane_independence.counted_lanes must name at least one lane")?; + let stamp = |k: &str| -> Result { + li.get(k) + .and_then(Value::as_str) + .filter(|s| utc_stamp(s)) + .map(str::to_string) + .ok_or(format!( + "lane_independence.{k} must be YYYY-MM-DDTHH:MM:SSZ" + )) + }; + let registered_at = stamp("registered_at")?; + let training_started_at = stamp("training_started_at")?; + if registered_at >= training_started_at { + return Err(format!( + "lane_independence registered at {registered_at}, not before training started at {training_started_at}" + )); + } + Ok(Manifest { + min_n: usize::try_from(min_n).map_err(|e| e.to_string())?, + counted_lanes, + registered_at, + training_started_at, + }) +} + +/// One lane pair's κ_err on the gate's common items. +#[derive(Debug, Clone, Serialize)] +pub struct GatePair { + pub pair: [String; 2], + pub kappa_err: Option, +} + +/// The gate's decision and the κ it decided on. +#[derive(Debug, Clone, Serialize)] +pub struct GateVerdict { + /// `true` only when [`crate::stats::h7_holds`] returned `Some(true)` on at + /// least `min_n` common items. + pub pass: bool, + /// Every reason for refusal; empty when `pass`. + pub refusals: Vec, + /// Items every lane (shadow and each voter) scored, gold agreeing. + pub n: usize, + /// κ_err(shadow, V) per voter, manifest order. + pub qwen_vs_voter: Vec, + /// κ_err(V_i, V_j) per voter pair, i < j. + pub voter_vs_voter: Vec, + /// The largest voter–voter κ_err, when every pair is defined. + pub ceiling: Option, +} + +/// H7 (PRM-001 v3 §3, δ-free) on candidate rows: with I the matured gold +/// `split` items that `shadow` and EVERY counted lane scored (one parsed row +/// each, gold agreeing), `pass` ⇔ `stats::h7_holds` on I is `Some(true)`, i.e. +/// κ_err(shadow, V) ≤ max voter–voter κ_err for every voter V. Refuses (S-14) +/// on fewer than `min_n` common items, a voter with no row, and an undecided +/// `h7_holds` (fewer than two voters, or an undefined κ). +#[must_use] +pub fn gate(rows: &[Row], split: &str, shadow: &str, m: &Manifest) -> GateVerdict { + let items = items_of(rows, split); + let mut refusals = Vec::new(); + let voters: Vec<&str> = m.counted_lanes.iter().map(String::as_str).collect(); + for v in &voters { + if !items.values().any(|l| l.contains_key(v)) { + refusals.push(format!("S-14 {v}: no scored rows")); + } + } + // err[lane] over I, in item order. + let lanes: Vec<&str> = std::iter::once(shadow) + .chain(voters.iter().copied()) + .collect(); + let mut err: Vec> = vec![Vec::new(); lanes.len()]; + for m in items.values() { + let Some(got) = lanes.iter().map(|l| one(m, l)).collect::>>() else { + continue; + }; + // All rows describe one PR; disagreeing gold is not gold. + if got.iter().all(|&(_, d)| d == got[0].1) { + for (e, (f, d)) in err.iter_mut().zip(got) { + e.push(f != d); + } + } + } + let n = err[0].len(); + if n < m.min_n { + refusals.push(format!( + "S-14 insufficient: {n} items scored by the shadow lane and every voter, min_n={}", + m.min_n + )); + } + let k = |i: usize, j: usize| GatePair { + pair: [lanes[i].to_string(), lanes[j].to_string()], + kappa_err: crate::stats::error_kappa(&err[i], &err[j]), + }; + let qwen_vs_voter: Vec = (1..lanes.len()).map(|j| k(0, j)).collect(); + let voter_vs_voter: Vec = (1..lanes.len()) + .flat_map(|i| (i + 1..lanes.len()).map(move |j| (i, j))) + .map(|(i, j)| k(i, j)) + .collect(); + let kappas = |p: &[GatePair]| p.iter().map(|p| p.kappa_err).collect::>(); + let ceiling = kappas(&voter_vs_voter) + .into_iter() + .collect::>>() + .and_then(|v| v.into_iter().reduce(f64::max)); + match crate::stats::h7_holds(&kappas(&qwen_vs_voter), &kappas(&voter_vs_voter)) { + Some(true) => {} + Some(false) => { + let c = ceiling.unwrap_or(f64::NAN); + for p in &qwen_vs_voter { + if let Some(q) = p.kappa_err.filter(|&q| q > c) { + refusals.push(format!( + "S-14 {}: κ_err {q:.4} > voter–voter max {c:.4}", + p.pair[1] + )); + } + } + } + None => refusals + .push("S-14 H7 undecided: fewer than two voters, or a κ_err is undefined".into()), + } + GateVerdict { + pass: refusals.is_empty(), + refusals, + n, + qwen_vs_voter, + voter_vs_voter, + ceiling, + } +} + +#[cfg(test)] +#[path = "kappa_probe_tests.rs"] +mod tests; diff --git a/crates/aprender-review-experiment/src/kappa_probe_tests.rs b/crates/aprender-review-experiment/src/kappa_probe_tests.rs new file mode 100644 index 0000000000..469a0d50ba --- /dev/null +++ b/crates/aprender-review-experiment/src/kappa_probe_tests.rs @@ -0,0 +1,305 @@ +use super::*; + +/// One agent-trace-v1 row, as much of it as the probe reads. +fn row(lane: &str, key: u32, verdict: &str, outcome: &str) -> String { + row_with(lane, key, verdict, outcome, "gold", "val", true) +} + +fn row_with( + lane: &str, + key: u32, + verdict: &str, + outcome: &str, + tier: &str, + split: &str, + matured: bool, +) -> String { + let counted = lane != "qwen-shadow"; + let matured = if matured { + r#""2026-09-01T00:00:00Z""# + } else { + "null" + }; + format!( + r#"{{"schema":"agent-trace-v1","quorum_id":"q{key}","round":1,"lane":"{lane}","counted":{counted}, + "lane_state":"Verdict","verdict":"{verdict}","parse_status":"ok","label_tier":"{tier}", + "outcome":"{outcome}","outcome_matured_at":{matured},"split_guard":{{"split":"{split}"}},"trace_status":"ok"}}"# + ) + .replace('\n', "") +} + +/// `n` items; `defects` of them regressed. Each lane's verdict per item comes from `flag(i)`. +fn lane_rows( + lane: &str, + n: u32, + defect: impl Fn(u32) -> bool, + flag: impl Fn(u32) -> bool, +) -> Vec { + (0..n) + .map(|i| { + let v = if flag(i) { + "request_changes" + } else { + "approve" + }; + let o = if defect(i) { + "reverted_le14d" + } else { + "merged" + }; + row(lane, i, v, o) + }) + .collect() +} + +fn board(lines: &[String], min_n: usize) -> Scoreboard { + let rows = parse_rows(&lines.join("\n")).expect("fixture parses"); + probe(&rows, "val", "qwen-shadow", min_n) +} + +fn pair<'a>(b: &'a Scoreboard, lane: &str) -> &'a PairKappa { + b.pairs + .iter() + .find(|p| p.counted_lane == lane) + .expect("pair present") +} + +/// Qwen and sonnet share half their errors: a known, non-degenerate κ. +fn base_fixture() -> Vec { + let defect = |i: u32| i % 2 == 0; + let mut v = lane_rows("sonnet", 40, defect, |i| (i % 2 == 0) != (i % 5 == 0)); + v.extend(lane_rows("qwen-shadow", 40, defect, |i| { + (i % 2 == 0) != (i % 5 == 0 || i % 7 == 0) + })); + v +} + +#[test] +fn falsify_lind_001_only_matured_gold_val_rows_are_scored() { + let clean = board(&base_fixture(), 10); + let k = pair(&clean, "sonnet").kappa_err.expect("defined"); + // Planted noise: every lane gets rows that would be pure agreement-errors if counted. + let mut noisy = base_fixture(); + for (i, (tier, split, matured, outcome)) in [ + ("silver", "val", true, "reverted_le14d"), + ("pending", "val", true, "reverted_le14d"), + ("quarantined", "val", true, "reverted_le14d"), + ("gold", "train", true, "reverted_le14d"), + ("gold", "test", true, "reverted_le14d"), + ("gold", "val", false, "reverted_le14d"), + ("gold", "val", true, "pending"), + ("gold", "val", true, "closed_unmerged"), + ] + .into_iter() + .enumerate() + { + let key = 1000 + i as u32; + for lane in ["sonnet", "qwen-shadow"] { + noisy.push(row_with( + lane, key, "approve", outcome, tier, split, matured, + )); + } + } + // Unparsed / non-verdict rows are excluded, never counted as errors. + noisy.push(row("sonnet", 2000, "abstain", "reverted_le14d")); + noisy.push(row("qwen-shadow", 2000, "approve", "reverted_le14d")); + let n = board(&noisy, 10); + assert_eq!( + pair(&n, "sonnet").n, + pair(&clean, "sonnet").n, + "noise rows were scored" + ); + assert_eq!(pair(&n, "sonnet").kappa_err, Some(k)); + // And the κ is the frozen stats::error_kappa on the error vectors, not a second formula. + let (a, b): (Vec, Vec) = (0..40u32) + .map(|i| (i % 5 == 0 || i % 7 == 0, i % 5 == 0)) + .unzip(); + assert_eq!(Some(k), crate::stats::error_kappa(&a, &b)); +} + +#[test] +fn falsify_lind_002_thin_or_undefined_pairs_are_insufficient() { + let thin = board(&base_fixture(), 41); + let p = pair(&thin, "sonnet"); + assert_eq!(p.status, PairStatus::Insufficient); + assert_eq!(p.kappa_err, None, "a thin pair must not print a number"); + // Both lanes always right → constant error vector → κ undefined. + let defect = |i: u32| i % 2 == 0; + let mut v = lane_rows("sonnet", 40, defect, defect); + v.extend(lane_rows("qwen-shadow", 40, defect, defect)); + let u = board(&v, 10); + assert_eq!(pair(&u, "sonnet").status, PairStatus::Insufficient); + assert_eq!(pair(&u, "sonnet").kappa_err, None); + // The gate refuses an undefined κ (S-14), it never passes by default. + v.extend(lane_rows("haiku", 40, defect, defect)); + let g = run_gate(&[v], &manifest(&["sonnet", "haiku"], 10)); + assert!( + !g.pass && g.refusals.iter().any(|r| r.contains("S-14")), + "{:?}", + g.refusals + ); +} + +fn manifest(voters: &[&str], min_n: usize) -> Manifest { + Manifest { + min_n, + counted_lanes: voters.iter().map(|v| (*v).to_string()).collect(), + registered_at: "2026-09-20T00:00:00Z".into(), + training_started_at: "2026-09-21T00:00:00Z".into(), + } +} + +/// `n` items, half regressed; `lane` errs exactly on the items where `err(i)`. +fn errs(lane: &str, n: u32, err: impl Fn(u32) -> bool) -> Vec { + let defect = |i: u32| i % 2 == 0; + lane_rows(lane, n, defect, move |i| defect(i) != err(i)) +} + +fn run_gate(lines: &[Vec], m: &Manifest) -> GateVerdict { + let rows = parse_rows(&lines.concat().join("\n")).expect("fixture parses"); + gate(&rows, "val", "qwen-shadow", m) +} + +fn five(i: u32) -> bool { + i % 5 == 0 +} + +/// Sonnet and haiku share most errors; qwen errs elsewhere. +fn voters() -> Vec> { + vec![ + errs("sonnet", 40, five), + errs("haiku", 40, |i| five(i) || i % 11 == 0), + ] +} + +#[test] +fn falsify_lind_003_h7_is_qwen_kappa_at_most_max_voter_kappa() { + let m = manifest(&["sonnet", "haiku"], 10); + let mut v = voters(); + v.push(errs("qwen-shadow", 40, |i| i % 3 == 0)); + let g = run_gate(&v, &m); + assert!(g.pass, "{:?}", g.refusals); + assert_eq!(g.n, 40); + let c = g.ceiling.expect("defined"); + assert!(g + .qwen_vs_voter + .iter() + .all(|p| p.kappa_err.expect("defined") <= c)); + // A qwen that copies sonnet's blind spots: κ = 1 > the voter–voter max. + let mut v = voters(); + v.push(errs("qwen-shadow", 40, five)); + let g = run_gate(&v, &m); + assert!(c < 1.0 && !g.pass, "a voter-copying qwen passed"); + assert!( + g.refusals.iter().any(|r| r.contains("sonnet")), + "{:?}", + g.refusals + ); + // Inclusive: identical voters put the ceiling at 1, so the same copy passes. + let v = vec![ + errs("sonnet", 40, five), + errs("haiku", 40, five), + errs("qwen-shadow", 40, five), + ]; + let g = run_gate(&v, &m); + assert_eq!(g.ceiling, Some(1.0)); + assert!(g.pass, "κ = ceiling must pass (≤): {:?}", g.refusals); + // The ceiling is the MAX over every voter pair, not the first or the least. + let mut v = voters(); + v.push(errs("agy", 40, |i| i % 3 == 0)); + v.push(errs("qwen-shadow", 40, |i| i % 4 == 0)); + let g = run_gate(&v, &manifest(&["sonnet", "haiku", "agy"], 10)); + let vv: Vec = g + .voter_vs_voter + .iter() + .map(|p| p.kappa_err.expect("defined")) + .collect(); + assert_eq!(vv.len(), 3); + assert_eq!(g.ceiling, vv.iter().copied().reduce(f64::max)); + assert!( + vv.iter().any(|&k| Some(k) != g.ceiling), + "the fixture must separate max from the rest" + ); +} + +#[test] +fn falsify_lind_002_the_gate_refuses_what_it_cannot_decide() { + let mut v = voters(); + v.push(errs("qwen-shadow", 40, |i| i % 3 == 0)); + // One voter: no voter–voter κ, so H7 is undecided — a refusal, not a pass. + let g = run_gate(&v, &manifest(&["sonnet"], 10)); + assert!( + !g.pass && g.refusals.iter().any(|r| r.contains("undecided")), + "{:?}", + g.refusals + ); + // A voter the manifest names but the rows lack is a refusal, not a skip. + let g = run_gate(&v, &manifest(&["sonnet", "haiku", "agy"], 10)); + assert!( + !g.pass && g.refusals.iter().any(|r| r.contains("agy")), + "{:?}", + g.refusals + ); + // Below min_n common items. + let g = run_gate(&v, &manifest(&["sonnet", "haiku"], 41)); + assert!( + !g.pass && g.refusals.iter().any(|r| r.contains("insufficient")), + "{:?}", + g.refusals + ); + assert!(run_gate(&v, &manifest(&["sonnet", "haiku"], 40)).pass); +} + +/// Every κ, the ceiling included, is taken on I, the items qwen AND every +/// voter scored — not on each pair's own overlap. +#[test] +fn falsify_lind_002_kappas_are_on_the_common_items() { + let haiku = |i: u32| if i < 20 { i % 3 == 0 } else { five(i) }; + let v = vec![ + errs("sonnet", 40, five), + errs("haiku", 40, haiku), + errs("qwen-shadow", 20, |i| i % 4 == 0), + ]; + // An item whose gold the lanes disagree on is not gold: it is not in I. + let mut v = v; + v.push(vec![ + row("sonnet", 99, "approve", "merged"), + row("haiku", 99, "approve", "reverted_le14d"), + row("qwen-shadow", 99, "approve", "merged"), + ]); + let g = run_gate(&v, &manifest(&["sonnet", "haiku"], 10)); + assert_eq!(g.n, 20); + let e = |f: &dyn Fn(u32) -> bool, n: u32| (0..n).map(f).collect::>(); + let on_i = crate::stats::error_kappa(&e(&five, 20), &e(&haiku, 20)); + let on_all = crate::stats::error_kappa(&e(&five, 40), &e(&haiku, 40)); + assert_ne!( + on_i, on_all, + "the fixture must separate the two, or this proves nothing" + ); + assert_eq!(g.ceiling, on_i); +} + +#[test] +fn falsify_lind_004_voters_and_min_n_are_pre_registered_and_delta_is_gone() { + let ok = r#"{"lane_independence":{"min_n":30,"counted_lanes":["sonnet","haiku","agy"], + "registered_at":"2026-09-20T00:00:00Z","training_started_at":"2026-09-21T00:00:00Z"}}"#; + let m = parse_manifest(ok).expect("valid manifest"); + assert_eq!((m.min_n, m.counted_lanes.len()), (30, 3)); + for bad in [ + "{}", + r#"{"lane_independence":{}}"#, + // v2's δ: H7 is δ-free in v3, and a δ nothing reads is a false assurance + r#"{"lane_independence":{"delta":0.05,"min_n":30,"counted_lanes":["sonnet"],"registered_at":"2026-09-20T00:00:00Z","training_started_at":"2026-09-21T00:00:00Z"}}"#, + r#"{"lane_independence":{"min_n":0,"counted_lanes":["sonnet"],"registered_at":"2026-09-20T00:00:00Z","training_started_at":"2026-09-21T00:00:00Z"}}"#, + r#"{"lane_independence":{"min_n":30,"counted_lanes":[],"registered_at":"2026-09-20T00:00:00Z","training_started_at":"2026-09-21T00:00:00Z"}}"#, + // registered AFTER training started: the voters could be picked after seeing κ + r#"{"lane_independence":{"min_n":30,"counted_lanes":["sonnet"],"registered_at":"2026-09-22T00:00:00Z","training_started_at":"2026-09-21T00:00:00Z"}}"#, + // same instant is not "before" + r#"{"lane_independence":{"min_n":30,"counted_lanes":["sonnet"],"registered_at":"2026-09-21T00:00:00Z","training_started_at":"2026-09-21T00:00:00Z"}}"#, + // a timestamp that does not compare lexicographically (offset, not Z) + r#"{"lane_independence":{"min_n":30,"counted_lanes":["sonnet"],"registered_at":"2026-09-20T00:00:00+02:00","training_started_at":"2026-09-21T00:00:00Z"}}"#, + "not json", + ] { + assert!(parse_manifest(bad).is_err(), "accepted: {bad}"); + } +} diff --git a/crates/aprender-review-experiment/src/ladder.rs b/crates/aprender-review-experiment/src/ladder.rs new file mode 100644 index 0000000000..d757920c9a --- /dev/null +++ b/crates/aprender-review-experiment/src/ladder.rs @@ -0,0 +1,296 @@ +//! PRM-09 promotion ladder (PRM-001 v3 §5.4; contract `rex-promotion-ladder-v2`, +//! which supersedes the REX-09 shadow → tripwire → vote ladder of v1). +//! +//! The ladder reads only a `prm-001-report-v1` report (the §9 template: +//! `spec: PRM-001, version: 3`) and fails closed: +//! - **shadow** is the default, and anything the ladder cannot read keeps it; +//! - **tripwire** needs H4 and the replay p95 within the queue budget; +//! - **tie-breaker** (decides only a 1–1 split) also needs H5, H7, H1, H2 and +//! 7-day availability ≥ 95%; +//! - **vote** also needs ≥ 20 decided tie-breaks whose 14-day outcome agreement +//! is at least the voters'. +//! +//! H4/H5 are decided by the Holm-adjusted p; the CI is report-only. H1/H2 (count +//! tests) and H7 (δ-free, outside the Holm family) are decided by their verdict. +//! `hardware_ruling.rung_eligible` caps the rung. Lambda is rung-2 shadow only +//! (R-9): a lambda or unnamed primary cell caps the lane at tripwire (S-4). +//! +//! [`evidence`] is the rung one report grants; [`decide`] steps the lane from +//! its current rung by at most one rung either way, except that S-4 caps at once. + +use serde::{Deserialize, Serialize}; + +pub const SPEC: &str = "PRM-001"; +pub const SPEC_VERSION: u64 = 3; +/// §5.4: 7-day availability for tie-breaker. +pub const MIN_AVAILABILITY: f64 = 0.95; +/// §5.4: decided tie-breaks for vote. +pub const MIN_TIE_BREAKS: u64 = 20; + +/// A rung of the ladder, lowest first. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub enum Mode { + Shadow, + Tripwire, + TieBreaker, + Vote, +} + +impl Mode { + const ALL: [Mode; 4] = [Mode::Shadow, Mode::Tripwire, Mode::TieBreaker, Mode::Vote]; + + fn up(self) -> Mode { + Self::ALL[(self as usize + 1).min(3)] + } + + fn down(self) -> Mode { + Self::ALL[(self as usize).saturating_sub(1)] + } +} + +#[derive(Debug, Deserialize)] +struct Hypothesis { + id: String, + /// Holm-adjusted one-sided p. `ci` may be present and is ignored. + #[serde(default)] + p_holm: Option, + #[serde(default)] + verdict: Option, +} + +#[derive(Debug, Default, Deserialize)] +struct Results { + #[serde(default)] + hypotheses: Vec, +} + +#[derive(Debug, Default, Deserialize)] +struct Lane { + #[serde(default)] + availability_7d: Option, +} + +#[derive(Debug, Default, Deserialize)] +struct Speed { + #[serde(default)] + within_budget: Option, +} + +#[derive(Debug, Deserialize)] +struct Ruling { + #[serde(default)] + primary: Option, + #[serde(default)] + rung_eligible: Option, +} + +#[derive(Debug, Default, Deserialize)] +struct Promotion { + #[serde(default)] + tie_breaks_decided: Option, + #[serde(default)] + tie_break_outcome_agreement: Option, + #[serde(default)] + voter_outcome_agreement: Option, +} + +#[derive(Debug, Deserialize)] +struct Report { + #[serde(default)] + spec: Option, + #[serde(default)] + version: Option, + prereg_sha: String, + #[serde(default)] + exploratory: bool, + #[serde(default)] + lane: Lane, + #[serde(default)] + speed: Speed, + #[serde(default)] + results: Results, + #[serde(default)] + hardware_ruling: Option, + #[serde(default)] + promotion: Promotion, +} + +/// The ladder's decision and every reason it stopped below `vote`. +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +pub struct Decision { + pub mode: Mode, + pub reasons: Vec, +} + +fn verdict_holds<'a>(r: &'a Report, id: &str) -> Result<&'a Hypothesis, String> { + let h = r + .results + .hypotheses + .iter() + .find(|h| h.id == id) + .ok_or_else(|| format!("{id}: not in the report"))?; + if h.verdict.as_deref() == Some("holds") { + Ok(h) + } else { + Err(format!("{id}: verdict {:?}", h.verdict)) + } +} + +/// H4/H5: an explicit `holds` verdict whose Holm-adjusted p is finite and ≤ α. +fn holm_holds(r: &Report, id: &str) -> Result<(), String> { + match verdict_holds(r, id)?.p_holm { + Some(p) if p.is_finite() && (0.0..=crate::stats::ALPHA).contains(&p) => Ok(()), + p => Err(format!( + "{id}: Holm p {p:?} is not ≤ {}", + crate::stats::ALPHA + )), + } +} + +fn shadow(reason: String) -> Decision { + Decision { + mode: Mode::Shadow, + reasons: vec![reason], + } +} + +/// The rung the evidence alone grants, and every unmet gate. +fn gates(r: &Report, reasons: &mut Vec) -> Mode { + let mut trip = Vec::new(); + trip.extend(holm_holds(r, "H4").err()); + if r.speed.within_budget != Some(true) { + trip.push(format!( + "speed: replay p95 within the queue budget is {:?}", + r.speed.within_budget + )); + } + if !trip.is_empty() { + reasons.extend(trip); + return Mode::Shadow; + } + let mut tb: Vec = [ + holm_holds(r, "H5"), + verdict_holds(r, "H7").map(|_| ()), + verdict_holds(r, "H1").map(|_| ()), + verdict_holds(r, "H2").map(|_| ()), + ] + .into_iter() + .filter_map(Result::err) + .collect(); + match r.lane.availability_7d { + Some(a) if a >= MIN_AVAILABILITY => {} + a => tb.push(format!("availability_7d {a:?} is not ≥ {MIN_AVAILABILITY}")), + } + if !tb.is_empty() { + reasons.extend(tb); + return Mode::Tripwire; + } + let p = &r.promotion; + let decided = p.tie_breaks_decided.unwrap_or(0); + if decided < MIN_TIE_BREAKS { + reasons.push(format!( + "tie_breaks_decided {decided} is not ≥ {MIN_TIE_BREAKS}" + )); + return Mode::TieBreaker; + } + match (p.tie_break_outcome_agreement, p.voter_outcome_agreement) { + (Some(t), Some(v)) if t >= v => Mode::Vote, + (t, v) => { + reasons.push(format!( + "tie-break outcome agreement {t:?} is not ≥ the voters' {v:?}" + )); + Mode::TieBreaker + } + } +} + +/// `true` when the primary cell is lambda or unnamed (R-9: lambda may not sit +/// at tie-breaker or above). +fn s4(r: &Report) -> bool { + r.hardware_ruling + .as_ref() + .and_then(|h| h.primary.as_deref()) + .is_none_or(|p| p.starts_with("lambda")) +} + +fn read(report_json: &str, prereg_sha: &str) -> Result { + let r: Report = serde_json::from_str(report_json) + .map_err(|e| shadow(format!("report does not parse: {e}")))?; + if r.spec.as_deref() != Some(SPEC) || r.version != Some(SPEC_VERSION) { + return Err(shadow(format!( + "spec {:?} version {:?} is not {SPEC} v{SPEC_VERSION} (prm-001-report-v1)", + r.spec, r.version + ))); + } + if r.prereg_sha != prereg_sha { + return Err(shadow("prereg_sha differs from the lock".into())); + } + if r.exploratory { + return Err(shadow("exploratory report (R-1)".into())); + } + Ok(r) +} + +/// The rung one report grants, capped by its hardware ruling. Never errors: +/// an unreadable, foreign, exploratory or stale report is shadow. +#[must_use] +pub fn evidence(report_json: &str, prereg_sha: &str) -> Decision { + match read(report_json, prereg_sha) { + Ok(r) => capped(&r), + Err(d) => d, + } +} + +fn capped(r: &Report) -> Decision { + let mut reasons = Vec::new(); + let mut mode = gates(r, &mut reasons); + let Some(h) = &r.hardware_ruling else { + reasons.push("no hardware ruling".into()); + return Decision { + mode: Mode::Shadow, + reasons, + }; + }; + let cap = h.rung_eligible.unwrap_or(Mode::Shadow); + if cap < mode { + reasons.push(format!("hardware ruling caps the lane at {cap:?}")); + mode = cap; + } + if s4(r) && mode > Mode::Tripwire { + reasons.push(format!( + "S-4: primary cell {} is lambda or unnamed; lambda is shadow-only (R-9)", + h.primary.as_deref().unwrap_or("(unnamed)") + )); + mode = Mode::Tripwire; + } + Decision { mode, reasons } +} + +/// Step the lane from `current` on one report: at most one rung up, and a +/// demotion drops exactly one rung. A lambda or unnamed primary cell above +/// tripwire is capped at once (S-4), the one exception to the one-rung rule. +#[must_use] +pub fn decide(report_json: &str, prereg_sha: &str, current: Mode) -> Decision { + let (mut d, lambda) = match read(report_json, prereg_sha) { + Ok(r) => (capped(&r), s4(&r)), + Err(d) => (d, false), + }; + if lambda && current > Mode::Tripwire { + d.mode = Mode::Tripwire; + return d; + } + let stepped = d.mode.clamp(current.down(), current.up()); + if stepped != d.mode { + d.reasons.push(format!( + "one rung per report: {current:?} → {stepped:?}, not {:?}", + d.mode + )); + d.mode = stepped; + } + d +} + +#[cfg(test)] +#[path = "ladder_tests.rs"] +mod tests; diff --git a/crates/aprender-review-experiment/src/ladder_tests.rs b/crates/aprender-review-experiment/src/ladder_tests.rs new file mode 100644 index 0000000000..250f71d454 --- /dev/null +++ b/crates/aprender-review-experiment/src/ladder_tests.rs @@ -0,0 +1,316 @@ +// `json!` expands to an `unwrap` of an infallible `to_value`. +#![allow(clippy::disallowed_methods)] + +use super::*; +use serde_json::{json, Value}; + +const PREREG: &str = "ef51087dc79bab0ad160e8a14f5b13e2ea43986b30c05dafc63c84c2dc21cdc0"; + +fn h(id: &str, p: f64) -> Value { + json!({"id": id, "ci": [0.01, 0.2], "p_holm": p, "verdict": "holds"}) +} + +/// A `prm-001-report-v1` report on which every gate up to vote passes. +fn full() -> Value { + json!({ + "spec": "PRM-001", + "version": 3, + "prereg_sha": PREREG, + "lane": {"rung": "tie-breaker", "availability_7d": 0.97}, + "speed": {"within_budget": true}, + "results": {"hypotheses": [ + {"id": "H1", "verdict": "holds"}, + {"id": "H2", "verdict": "holds"}, + h("H4", 0.01), + h("H5", 0.04), + {"id": "H7", "verdict": "holds"}, + ]}, + "hardware_ruling": {"primary": "gx10-cpu", "rung_eligible": "vote"}, + "promotion": { + "tie_breaks_decided": 20, + "tie_break_outcome_agreement": 0.9, + "voter_outcome_agreement": 0.9 + } + }) +} + +fn set(mut v: Value, ptr: &str, x: Value) -> Value { + *v.pointer_mut(ptr).unwrap_or_else(|| panic!("{ptr}")) = x; + v +} + +fn drop_h(mut v: Value, id: &str) -> Value { + let hs = v["results"]["hypotheses"] + .as_array_mut() + .expect("hypotheses"); + hs.retain(|h| h["id"] != id); + v +} + +fn replace_h(v: Value, x: Value) -> Value { + let id = x["id"].as_str().expect("id").to_string(); + let mut v = drop_h(v, &id); + v["results"]["hypotheses"] + .as_array_mut() + .expect("hypotheses") + .push(x); + v +} + +/// The rung the evidence grants, before the one-rung step. +fn rung(v: &Value) -> Mode { + evidence(&v.to_string(), PREREG).mode +} + +#[test] +fn every_gate_passed_is_a_full_vote() { + let d = evidence(&full().to_string(), PREREG); + assert_eq!(d.mode, Mode::Vote, "{:?}", d.reasons); + assert!(d.reasons.is_empty()); + assert_eq!( + serde_json::to_value(Mode::TieBreaker).expect("mode"), + json!("tie-breaker") + ); +} + +/// FALSIFY-RXG-001: a failing H4 (the planted report) keeps the lane in shadow. +#[test] +fn falsify_rxg_001_a_failing_h4_keeps_the_lane_in_shadow() { + let fail = json!({"id": "H4", "ci": [-0.05, 0.1], "p_holm": 0.3, "verdict": "fails"}); + assert_eq!(rung(&replace_h(full(), fail)), Mode::Shadow); + for bad in [ + json!({"id": "H4", "p_holm": 0.051, "verdict": "holds"}), + json!({"id": "H4", "p_holm": -0.1, "verdict": "holds"}), + json!({"id": "H4", "verdict": "holds"}), + ] { + assert_eq!(rung(&replace_h(full(), bad.clone())), Mode::Shadow, "{bad}"); + } + assert_eq!(rung(&drop_h(full(), "H4")), Mode::Shadow); + let nan = full() + .to_string() + .replace("\"p_holm\":0.01", "\"p_holm\":NaN"); + assert_eq!(evidence(&nan, PREREG).mode, Mode::Shadow); +} + +/// FALSIFY-RXG-002: H4 alone is tripwire, never above; the speed gate and the +/// hardware ruling cap the rung. +#[test] +fn falsify_rxg_002_h4_alone_is_tripwire_never_vote() { + let h5_fail = json!({"id": "H5", "p_holm": 0.6, "verdict": "fails"}); + let d = evidence(&replace_h(full(), h5_fail).to_string(), PREREG); + assert_eq!(d.mode, Mode::Tripwire, "{:?}", d.reasons); + assert!( + d.reasons.iter().any(|r| r.starts_with("H5")), + "{:?}", + d.reasons + ); + // the planted report: H4 only + let h4_only = drop_h(drop_h(drop_h(drop_h(full(), "H5"), "H7"), "H1"), "H2"); + assert_eq!(rung(&h4_only), Mode::Tripwire); + // speed: replay p95 over the queue budget, or unmeasured, is shadow + assert_eq!( + rung(&set(full(), "/speed/within_budget", json!(false))), + Mode::Shadow + ); + let mut no_speed = full(); + no_speed.as_object_mut().expect("obj").remove("speed"); + assert_eq!(rung(&no_speed), Mode::Shadow); + // the hardware ruling caps; none is shadow + for (cap, want) in [ + ("shadow", Mode::Shadow), + ("tripwire", Mode::Tripwire), + ("tie-breaker", Mode::TieBreaker), + ] { + let v = set(full(), "/hardware_ruling/rung_eligible", json!(cap)); + assert_eq!(rung(&v), want, "{cap}"); + } + // a ruling that names no eligible rung grants none + let mut no_rung = full(); + no_rung["hardware_ruling"] + .as_object_mut() + .expect("ruling") + .remove("rung_eligible"); + assert_eq!(rung(&no_rung), Mode::Shadow); + let mut no_ruling = full(); + no_ruling + .as_object_mut() + .expect("obj") + .remove("hardware_ruling"); + assert_eq!(rung(&no_ruling), Mode::Shadow); +} + +/// FALSIFY-RXG-003: only a locked, confirmatory prm-001-report-v1 is read. +#[test] +fn falsify_rxg_003_only_a_locked_confirmatory_report_is_read() { + let old = full() + .to_string() + .replace("\"spec\":\"PRM-001\"", "\"schema\":\"rex-001-report-v2\""); + let cases = [ + ("rex-001-report-v2", old), + ("v2 spec", set(full(), "/version", json!(2)).to_string()), + ( + "stale prereg", + set(full(), "/prereg_sha", json!("0".repeat(64))).to_string(), + ), + ("exploratory", { + let mut v = full(); + v["exploratory"] = json!(true); + v.to_string() + }), + ("not json", "{".to_string()), + ]; + for (name, text) in cases { + assert_eq!(evidence(&text, PREREG).mode, Mode::Shadow, "{name}"); + } +} + +/// FALSIFY-RXG-004: the CI is report-only; the Holm p decides H4/H5. +#[test] +fn falsify_rxg_004_the_ci_is_report_only() { + let crossing = json!({"id": "H4", "ci": [-0.01, 0.2], "p_holm": 0.02, "verdict": "holds"}); + assert_eq!(rung(&replace_h(full(), crossing)), Mode::Vote); + let clean = json!({"id": "H4", "ci": [0.05, 0.2], "p_holm": 0.2, "verdict": "holds"}); + assert_eq!(rung(&replace_h(full(), clean)), Mode::Shadow); +} + +/// FALSIFY-RXG-005: tie-breaker needs H4 ∧ H5 ∧ H7 ∧ H1 ∧ H2 and 7-day +/// availability ≥ 95%; without any one of them the lane stays tripwire. +#[test] +fn falsify_rxg_005_tie_breaker_needs_h4_h5_h7_h1_h2_and_availability() { + let tb = set(full(), "/promotion/tie_breaks_decided", json!(0)); + assert_eq!(rung(&tb), Mode::TieBreaker); + for id in ["H5", "H7", "H1", "H2"] { + let d = evidence(&drop_h(tb.clone(), id).to_string(), PREREG); + assert_eq!(d.mode, Mode::Tripwire, "without {id}"); + assert!( + d.reasons.iter().any(|r| r.starts_with(id)), + "{id}: {:?}", + d.reasons + ); + let fails = json!({"id": id, "p_holm": 0.01, "verdict": "fails"}); + assert_eq!( + rung(&replace_h(tb.clone(), fails)), + Mode::Tripwire, + "{id} fails" + ); + } + // H5 is in the Holm family: a holds verdict over α does not hold + let h5_p = json!({"id": "H5", "p_holm": 0.6, "verdict": "holds"}); + assert_eq!(rung(&replace_h(tb.clone(), h5_p)), Mode::Tripwire); + // H7 is outside the Holm family: its verdict decides, a p is not needed + assert!(full()["results"]["hypotheses"][4].get("p_holm").is_none()); + // availability: inclusive at 0.95, below or absent is tripwire + assert_eq!( + rung(&set(tb.clone(), "/lane/availability_7d", json!(0.95))), + Mode::TieBreaker + ); + assert_eq!( + rung(&set(tb.clone(), "/lane/availability_7d", json!(0.9499))), + Mode::Tripwire + ); + let mut no_avail = tb.clone(); + no_avail["lane"] + .as_object_mut() + .expect("lane") + .remove("availability_7d"); + assert_eq!(rung(&no_avail), Mode::Tripwire); +} + +/// FALSIFY-RXG-006: a lambda primary cell is refused at tie-breaker and above +/// (R-9, S-4), whatever the evidence. +#[test] +fn falsify_rxg_006_lambda_primary_is_refused_above_tripwire() { + let v = set(full(), "/hardware_ruling/primary", json!("lambda-cuda")); + let d = evidence(&v.to_string(), PREREG); + assert_eq!(d.mode, Mode::Tripwire, "{:?}", d.reasons); + assert!( + d.reasons.iter().any(|r| r.contains("S-4")), + "{:?}", + d.reasons + ); + // the reason names the cell as written, never a Rust Debug rendering + assert!( + d.reasons + .iter() + .any(|r| r.contains("primary cell lambda-cuda is") && !r.contains("Some(")), + "{:?}", + d.reasons + ); + // a missing primary is not a non-lambda cell + let mut none = full(); + none["hardware_ruling"] + .as_object_mut() + .expect("ruling") + .remove("primary"); + assert_eq!(rung(&none), Mode::Tripwire); + let n = evidence(&none.to_string(), PREREG); + assert!( + n.reasons + .iter() + .any(|r| r.contains("primary cell (unnamed) is")), + "{:?}", + n.reasons + ); +} + +/// FALSIFY-RXG-007: vote needs ≥ 20 decided tie-breaks whose 14-day outcome +/// agreement is at least the voters' on the same rounds. +#[test] +fn falsify_rxg_007_vote_needs_twenty_agreeing_tie_breaks() { + assert_eq!( + rung(&set(full(), "/promotion/tie_breaks_decided", json!(19))), + Mode::TieBreaker + ); + assert_eq!( + rung(&set( + full(), + "/promotion/tie_break_outcome_agreement", + json!(0.89) + )), + Mode::TieBreaker + ); + let mut none = full(); + none.as_object_mut().expect("obj").remove("promotion"); + assert_eq!(rung(&none), Mode::TieBreaker); + let mut no_voters = full(); + no_voters["promotion"] + .as_object_mut() + .expect("promotion") + .remove("voter_outcome_agreement"); + assert_eq!(rung(&no_voters), Mode::TieBreaker); +} + +/// FALSIFY-RXG-008: the ladder never skips a rung, and a demotion drops +/// exactly one rung; S-4 is the exception and caps at once. +#[test] +fn falsify_rxg_008_never_skips_a_rung_and_demotes_one() { + let all = full().to_string(); + let step = |cur| decide(&all, PREREG, cur).mode; + assert_eq!(step(Mode::Shadow), Mode::Tripwire); + assert_eq!(step(Mode::Tripwire), Mode::TieBreaker); + assert_eq!(step(Mode::TieBreaker), Mode::Vote); + let d = decide("{", PREREG, Mode::Vote); + assert_eq!(d.mode, Mode::TieBreaker, "{:?}", d.reasons); + assert_eq!(decide("{", PREREG, Mode::Tripwire).mode, Mode::Shadow); + assert_eq!(decide("{", PREREG, Mode::Shadow).mode, Mode::Shadow); + let lambda = set(full(), "/hardware_ruling/primary", json!("lambda-cuda")).to_string(); + // S-4 caps a vote lane at tripwire at once, but never promotes past one rung + assert_eq!(decide(&lambda, PREREG, Mode::Vote).mode, Mode::Tripwire); + assert_eq!(decide(&lambda, PREREG, Mode::Shadow).mode, Mode::Tripwire); + assert_eq!( + decide(&lambda, PREREG, Mode::TieBreaker).mode, + Mode::Tripwire + ); + // S-4 never promotes: a lambda lane with no evidence stays in shadow + let lambda_h4_fail = drop_h( + set(full(), "/hardware_ruling/primary", json!("lambda-cuda")), + "H4", + ); + assert_eq!( + decide(&lambda_h4_fail.to_string(), PREREG, Mode::Shadow).mode, + Mode::Shadow + ); + // a tripwire-only report demotes a vote lane one rung, not two + let h4_only = drop_h(drop_h(full(), "H5"), "H7").to_string(); + assert_eq!(decide(&h4_only, PREREG, Mode::Vote).mode, Mode::TieBreaker); +} diff --git a/crates/aprender-review-experiment/src/ledger.rs b/crates/aprender-review-experiment/src/ledger.rs new file mode 100644 index 0000000000..9f63fffc13 --- /dev/null +++ b/crates/aprender-review-experiment/src/ledger.rs @@ -0,0 +1,343 @@ +//! REX-07 day-0 shadow lane: `review-ledger-v2` rows built from quorum +//! receipts (`docs/audits/quorum-*.json`), and the shadow-coverage check. +//! v2 (PRA-001 T2) keeps each counted lane's findings text and its +//! agent-trace-v1 shas; v1 rows stay readable through [`read_row`] and +//! migrate with [`v1::Row::migrate`]. Outcomes are appended as +//! [`OutcomeJoin`]s; a written row is never rewritten. +//! +//! The 4B lane runs inside the quorum rail as a non-voting advisory lane +//! (paiml-implement `advisory-lane.sh`, which records `.advisory_lane` in the +//! receipt). This module does not run the lane; it reads what the rail wrote +//! and holds it to §7 REX-07: +//! - every quorum after activation carries a shadow row (a receipt with no +//! `advisory_lane` is **missing**, never silently skipped); +//! - the apr row is one typed value, `Verdict | NotRun | Refused` (the +//! paiml-implement#436 wire form, one spelling with receipt-lint); an +//! unknown key or value fails to parse, so free text is never a reason; +//! - the shadow is never counted, and the quorum width is the counted lanes +//! alone, so a gx10-down round keeps its width. + +use serde::{Deserialize, Serialize}; + +pub const SCHEME: &str = "review-ledger-v2"; + +/// Rows written before v2 stay readable: see [`v1`] and [`read_row`]. +pub const SCHEME_V1: &str = "review-ledger-v1"; + +#[path = "ledger_v1.rs"] +pub mod v1; + +#[derive(Debug, Deserialize)] +struct Lane { + #[serde(default)] + family: Option, + #[serde(default)] + model: Option, + #[serde(default)] + verdict: Option, + #[serde(default)] + findings: Option>, + /// agent-trace-v1 blob shas (exact bytes sent / full raw reply). + #[serde(default)] + input_sha256: Option, + #[serde(default)] + output_sha256: Option, +} + +#[derive(Debug, Deserialize)] +struct Advisory { + #[serde(default)] + state: Option, + /// The typed apr row. Absent on an `off` lane; absent otherwise is a violation. + #[serde(default)] + row: Option, + #[serde(default)] + apr: Option, + #[serde(default)] + weights_sha256: Option, + /// The rail's uncounted flag (`counted` there is the counted round's verdict). + #[serde(default)] + counts: Option, +} + +#[derive(Debug, Deserialize)] +struct QuorumReceipt { + ticket: String, + head: String, + #[serde(default)] + pr: Option, + #[serde(default)] + diff_sha256: Option, + #[serde(default)] + agreed: Option, + #[serde(default)] + width: Option, + #[serde(default)] + lanes: Vec, + #[serde(default)] + advisory_lane: Option, +} + +/// One counted lane, as the ledger keeps it. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct LaneRow { + pub family: Option, + pub model: Option, + pub verdict: Option, + /// Carried by both versions; equals the text's length when text exists. + pub findings_count: usize, + /// The findings text, verbatim: a count cannot train anything. `None` is + /// text never captured (a v1 row, or a receipt without the key): a trace + /// gap, never an empty list. + pub findings: Option>, + /// agent-trace-v1 blob shas; a lane without both is a trace gap. + pub input_sha256: Option, + pub output_sha256: Option, +} + +/// The apr lane's verdict. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)] +pub enum Verdict { + #[serde(rename = "PASS")] + Pass, + #[serde(rename = "FAIL")] + Fail, +} + +/// Why the apr lane did not run. Closed: a new reason is a schema change. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +pub enum NotRun { + NoExecutor, + Busy, + Timeout, + ContextOverflow, + TrainActive, +} + +/// What removed a cell from the ladder. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub enum RemovedBy { + Ladder, + GpuProof, + Parse, +} + +/// The apr lane's result on one quorum — the paiml-implement#436 wire form: +/// `{"Verdict":{"verdict":"PASS","cell":"gx10-cuda","backend":"cuda"}}`, +/// `{"NotRun":"Busy"}`, `{"Refused":{"cell":"gx10-cuda","removed_by":"gpu-proof"}}`. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub enum Shadow { + Verdict { + verdict: Verdict, + cell: String, + backend: String, + }, + NotRun(NotRun), + Refused { + cell: String, + removed_by: RemovedBy, + }, +} + +/// One `review-ledger-v2` row (§5.2; PRA-001 T2). +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Row { + pub schema: String, + pub repo: String, + pub ticket: String, + pub head: String, + pub pr: Option, + pub diff_sha256: Option, + pub agreed: Option, + pub width: usize, + pub lanes: Vec, + pub shadow: Shadow, + pub apr_tag: Option, + /// From the receipt; a row without it is an identity gap. + pub weights_sha256: Option, + /// Joined later (merged / reverted ≤ 14 d / escape); `pending` until then. + pub outcome: String, +} + +/// What a set of receipts resolves to. +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize)] +pub struct Coverage { + pub quorums: usize, + pub carried: usize, + /// Receipts with no shadow row: REX-07 requires 0. + pub missing: Vec, + /// Receipts where the shadow was counted or changed the width. + pub violations: Vec, + /// Rows without an apr tag or weights sha (the §5.2 identity fields). + pub identity_gaps: usize, + /// Counted lanes without findings text or both agent-trace-v1 shas: the + /// Jidoka goal is 0. + pub trace_gaps: usize, +} + +impl Coverage { + /// §7 REX-07 acceptance: every quorum carries a shadow row, none counted, + /// and every row names its apr tag and weights sha (PRM v2 item 5). + #[must_use] + pub fn holds(&self) -> bool { + self.quorums > 0 + && self.missing.is_empty() + && self.violations.is_empty() + && self.identity_gaps == 0 + && self.trace_gaps == 0 + } +} + +/// A finding as text: a string stays verbatim, anything else is its JSON. +fn finding_text(v: &serde_json::Value) -> String { + v.as_str().map_or_else(|| v.to_string(), str::to_owned) +} + +/// An outcome join, appended to its own JSONL. Rows are never rewritten. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct OutcomeJoin { + pub repo: String, + pub head: String, + pub outcome: String, + /// RFC 3339 UTC; the latest join for a (repo, head) wins. + pub at: String, +} + +/// A row's outcome read through the append-only joins, latest first; +/// the row's own value (`pending`) when nothing joined. +#[must_use] +pub fn outcome<'a>(row: &'a Row, joins: &'a [OutcomeJoin]) -> &'a str { + joins + .iter() + .filter(|j| j.repo == row.repo && j.head == row.head) + .max_by(|a, b| a.at.cmp(&b.at)) + .map_or(row.outcome.as_str(), |j| j.outcome.as_str()) +} + +/// A counted lane whose trace a fine-tune could not use: no findings text, +/// or not both agent-trace-v1 shas. +fn trace_gap(l: &LaneRow) -> bool { + l.findings.is_none() || l.input_sha256.is_none() || l.output_sha256.is_none() +} + +/// A ledger line of either version, dispatched on its `schema`. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum AnyRow { + V1(v1::Row), + V2(Row), +} + +/// Read one ledger line. An unknown `schema` is an error, never a guess. +pub fn read_row(line: &str) -> Result { + #[derive(Deserialize)] + struct Head { + schema: String, + } + let h: Head = serde_json::from_str(line).map_err(|e| e.to_string())?; + match h.schema.as_str() { + SCHEME_V1 => serde_json::from_str(line) + .map(AnyRow::V1) + .map_err(|e| format!("{SCHEME_V1}: {e}")), + SCHEME => serde_json::from_str(line) + .map(AnyRow::V2) + .map_err(|e| format!("{SCHEME}: {e}")), + s => Err(format!("unknown ledger schema {s:?}")), + } +} + +/// Build ledger rows from `(name, json)` receipts and measure shadow coverage. +/// A receipt that does not parse is a violation, never dropped. +#[must_use] +pub fn build(receipts: &[(String, String)], repo: &str) -> (Vec, Coverage) { + let mut rows = Vec::new(); + let mut c = Coverage { + quorums: receipts.len(), + ..Coverage::default() + }; + for (name, text) in receipts { + let q: QuorumReceipt = match serde_json::from_str(text) { + Ok(q) => q, + Err(e) => { + c.violations.push(format!("{name}: {e}")); + continue; + } + }; + // `off` is the rail saying the lane was not run: no shadow row. + let Some(a) = q + .advisory_lane + .as_ref() + .filter(|a| a.state.as_deref() != Some("off")) + else { + c.missing.push(name.clone()); + continue; + }; + let shadow = match a.row.clone().map(serde_json::from_value::) { + Some(Ok(s)) => s, + Some(Err(e)) => { + c.violations.push(format!("{name}: apr row: {e}")); + continue; + } + None => { + c.violations.push(format!("{name}: no typed apr row")); + continue; + } + }; + c.carried += 1; + if a.counts != Some(false) { + c.violations + .push(format!("{name}: shadow lane not recorded as uncounted")); + } + let width = q.lanes.len(); + if q.width.is_some_and(|w| w != width as u64) { + c.violations.push(format!( + "{name}: width {:?} is not the {width} counted lanes", + q.width + )); + } + let row = Row { + schema: SCHEME.into(), + repo: repo.into(), + ticket: q.ticket, + head: q.head, + pr: q.pr, + diff_sha256: q.diff_sha256, + agreed: q.agreed, + width, + lanes: q + .lanes + .iter() + .map(|l| LaneRow { + family: l.family.clone(), + model: l.model.clone(), + verdict: l.verdict.clone(), + findings_count: l.findings.as_ref().map_or(0, Vec::len), + findings: l + .findings + .as_ref() + .map(|f| f.iter().map(finding_text).collect()), + input_sha256: l.input_sha256.clone(), + output_sha256: l.output_sha256.clone(), + }) + .collect(), + shadow, + apr_tag: a.apr.clone(), + weights_sha256: a.weights_sha256.clone(), + outcome: "pending".into(), + }; + if row.apr_tag.is_none() || row.weights_sha256.is_none() { + c.identity_gaps += 1; + } + c.trace_gaps += row.lanes.iter().filter(|l| trace_gap(l)).count(); + rows.push(row); + } + (rows, c) +} + +#[cfg(test)] +#[path = "ledger_tests.rs"] +mod tests; diff --git a/crates/aprender-review-experiment/src/ledger_tests.rs b/crates/aprender-review-experiment/src/ledger_tests.rs new file mode 100644 index 0000000000..5239ed13b2 --- /dev/null +++ b/crates/aprender-review-experiment/src/ledger_tests.rs @@ -0,0 +1,212 @@ +use super::*; + +fn receipt(advisory: Option<&str>, width: u64) -> String { + let lanes = r#"[{"family":"claude","model":"claude-sonnet-5","verdict":"PASS","findings":[],"input_sha256":"i1","output_sha256":"o1"}, + {"family":"gemini","model":"gemini-3.1-pro-high","verdict":"PASS","findings":["ledger.rs:205 drops the weights sha",{"f":1}],"input_sha256":"i2","output_sha256":"o2"}, + {"family":"claude","model":"claude-haiku-4-5","verdict":"PASS","findings":[],"input_sha256":"i3","output_sha256":"o3"}]"#; + let adv = advisory.map_or(String::new(), |a| format!(r#","advisory_lane":{a}"#)); + format!( + r#"{{"ticket":"PMAT-1","head":"abc","diff_sha256":"d","agreed":true,"width":{width},"lanes":{lanes}{adv}}}"# + ) +} + +const ANSWERED: &str = r#"{"state":"answered","row":{"Verdict":{"verdict":"PASS","cell":"gx10-cuda","backend":"cuda"}},"apr":"0.69.3","weights_sha256":"w4b","counted":"PASS","counts":false}"#; +const DOWN: &str = r#"{"state":"unavailable","row":{"NotRun":"NoExecutor"},"apr":"0.69.3","weights_sha256":"w4b","counted":"PASS","counts":false}"#; + +fn one(text: String) -> (Vec, Coverage) { + build(&[("quorum-PMAT-1.json".into(), text)], "paiml/aprender") +} + +#[test] +fn an_answered_shadow_becomes_a_ledger_row() { + let (rows, c) = one(receipt(Some(ANSWERED), 3)); + assert!(c.holds(), "{c:?}"); + let r = &rows[0]; + assert_eq!(r.schema, SCHEME); + assert_eq!(r.width, 3); + assert_eq!( + r.lanes[1].findings, + Some(vec![ + "ledger.rs:205 drops the weights sha".to_string(), + r#"{"f":1}"#.to_string() + ]), + "findings are kept as text, verbatim" + ); + assert_eq!(r.lanes[1].output_sha256.as_deref(), Some("o2")); + assert_eq!(c.trace_gaps, 0); + assert_eq!( + r.shadow, + Shadow::Verdict { + verdict: Verdict::Pass, + cell: "gx10-cuda".into(), + backend: "cuda".into() + } + ); + assert_eq!(r.weights_sha256.as_deref(), Some("w4b")); + assert_eq!(r.outcome, "pending"); + assert_eq!(c.identity_gaps, 0); +} + +#[test] +fn falsify_rxl_001_a_quorum_without_a_shadow_row_is_missing() { + let recs = vec![ + ("a.json".to_string(), receipt(Some(ANSWERED), 3)), + ("b.json".to_string(), receipt(None, 3)), + ]; + let (rows, c) = build(&recs, "paiml/aprender"); + assert_eq!(rows.len(), 1); + assert_eq!((c.quorums, c.carried), (2, 1)); + assert_eq!(c.missing, ["b.json"]); + assert!(!c.holds(), "one missing shadow row fails REX-07"); + assert!(!build(&[], "r").1.holds(), "no quorum is not 100 %"); + let off = r#"{"state":"off","counts":false,"why":"advisory_lane disabled"}"#; + assert!( + !one(receipt(Some(off), 3)).1.holds(), + "a lane that was off is not a shadow row" + ); + let (_, bad) = build(&[("x".into(), "{".into())], "r"); + assert!(!bad.holds(), "an unparsable receipt is not dropped"); +} + +#[test] +fn falsify_rxl_002_gx10_down_is_unknown_and_keeps_the_width() { + let (rows, c) = one(receipt(Some(DOWN), 3)); + assert!(c.holds(), "{c:?}"); + assert_eq!(rows[0].width, 3); + assert_eq!(rows[0].shadow, Shadow::NotRun(NotRun::NoExecutor)); + // A verdict without its cell is not a verdict. + let half = ANSWERED.replace(r#","cell":"gx10-cuda""#, ""); + assert!(!one(receipt(Some(&half), 3)).1.holds(), "no cell"); +} + +#[test] +fn falsify_rxl_003_a_counted_or_width_changing_shadow_is_a_violation() { + let counted = ANSWERED.replace(r#""counts":false"#, r#""counts":true"#); + assert!(!one(receipt(Some(&counted), 3)).1.holds(), "counted"); + let silent = ANSWERED.replace(r#","counts":false"#, ""); + assert!( + !one(receipt(Some(&silent), 3)).1.holds(), + "uncounted unrecorded" + ); + assert!(!one(receipt(Some(ANSWERED), 4)).1.holds(), "width 4"); + assert!(!one(receipt(Some(DOWN), 2)).1.holds(), "width 2"); +} + +#[test] +fn every_wire_form_round_trips() { + for row in [ + r#"{"Verdict":{"verdict":"FAIL","cell":"intel-wgpu","backend":"wgpu"}}"#, + r#"{"NotRun":"NoExecutor"}"#, + r#"{"NotRun":"Busy"}"#, + r#"{"NotRun":"Timeout"}"#, + r#"{"NotRun":"ContextOverflow"}"#, + r#"{"NotRun":"TrainActive"}"#, + r#"{"Refused":{"cell":"gx10-cuda","removed_by":"ladder"}}"#, + r#"{"Refused":{"cell":"gx10-cuda","removed_by":"gpu-proof"}}"#, + r#"{"Refused":{"cell":"gx10-cuda","removed_by":"parse"}}"#, + ] { + let s: Shadow = serde_json::from_str(row).expect(row); + assert_eq!(serde_json::to_string(&s).expect("ser"), row); + } +} + +#[test] +fn falsify_rxl_004_a_receipt_without_a_weights_sha_does_not_hold() { + let bare = ANSWERED.replace(r#","weights_sha256":"w4b""#, ""); + let (rows, c) = one(receipt(Some(&bare), 3)); + assert_eq!(rows.len(), 1, "the row is kept"); + assert_eq!(c.identity_gaps, 1); + assert!(!c.holds(), "an identity gap fails holds()"); + let untagged = ANSWERED.replace(r#""apr":"0.69.3""#, r#""apr":null"#); + assert!(!one(receipt(Some(&untagged), 3)).1.holds(), "no apr tag"); +} + +#[test] +fn falsify_rxl_005_a_free_text_reason_fails_to_parse() { + for row in [ + r#"{"NotRun":"gx10 connect refused"}"#, + r#"{"LaneUnavailable":{"why":"gx10 connect refused"}}"#, + r#"{"Refused":{"cell":"gx10-cuda","removed_by":"aprender#9999: refuses qwen35"}}"#, + r#"{"Verdict":{"verdict":"LGTM","cell":"gx10-cuda","backend":"cuda"}}"#, + r#"{"Verdict":{"verdict":"PASS","cell":"gx10-cuda","backend":"cuda","why":"x"}}"#, + ] { + assert!(serde_json::from_str::(row).is_err(), "{row}"); + let adv = ANSWERED.replace( + r#"{"Verdict":{"verdict":"PASS","cell":"gx10-cuda","backend":"cuda"}}"#, + row, + ); + let (rows, c) = one(receipt(Some(&adv), 3)); + assert!(rows.is_empty() && !c.holds(), "{row} is a violation"); + } + let untyped = r#"{"state":"answered","apr":"0.69.3","weights_sha256":"w","counts":false}"#; + assert!(!one(receipt(Some(untyped), 3)).1.holds(), "no typed row"); +} + +#[test] +fn falsify_rxl_006_a_counted_lane_without_trace_shas_does_not_hold() { + for gap in [ + r#","output_sha256":"o2""#, + r#","input_sha256":"i2""#, + r#""findings":["ledger.rs:205 drops the weights sha",{"f":1}],"#, + ] { + let (rows, c) = one(receipt(Some(ANSWERED), 3).replace(gap, "")); + assert_eq!(rows.len(), 1, "the row is kept"); + assert_eq!(c.trace_gaps, 1, "{gap}"); + assert!(!c.holds(), "a lane without its trace is RED: {gap}"); + } +} + +#[test] +fn falsify_rxl_007_an_outcome_join_appends_and_never_rewrites() { + let (rows, _) = one(receipt(Some(ANSWERED), 3)); + let before = rows.clone(); + let j = |head: &str, outcome: &str, at: &str| OutcomeJoin { + repo: "paiml/aprender".into(), + head: head.into(), + outcome: outcome.into(), + at: at.into(), + }; + assert_eq!(outcome(&rows[0], &[]), "pending"); + let joins = [ + j("abc", "reverted", "2026-09-30T00:00:00Z"), + j("abc", "merged", "2026-09-25T00:00:00Z"), + j("other", "escape", "2026-10-01T00:00:00Z"), + ]; + assert_eq!(outcome(&rows[0], &joins), "reverted", "latest join wins"); + assert_eq!(rows, before, "the row itself is never rewritten"); +} + +// v1 rows exactly as the two v1 writers serialized them. +const V1_LEGACY: &str = r#"{"schema":"review-ledger-v1","repo":"paiml/aprender","ticket":"PMAT-1","head":"abc","pr":null,"diff_sha256":"d","agreed":true,"width":3,"lanes":[{"family":"claude","model":"claude-sonnet-5","verdict":"PASS","findings":2}],"shadow":{"state":"Unknown","reason":{"LaneUnavailable":{"why":"gx10 down"}}},"apr_tag":null,"weights_sha256":null,"outcome":"pending"}"#; +const V1_TYPED: &str = r#"{"schema":"review-ledger-v1","repo":"paiml/aprender","ticket":"PMAT-1","head":"abc","pr":4354,"diff_sha256":"d","agreed":true,"width":3,"lanes":[{"family":"gemini","model":"gemini-3.1-pro-high","verdict":"FAIL","findings":2}],"shadow":{"Verdict":{"verdict":"PASS","cell":"gx10-cuda","backend":"cuda"}},"apr_tag":"0.69.3","weights_sha256":"w4b","outcome":"merged"}"#; + +#[test] +fn falsify_rxl_008_v1_rows_parse_and_migrate_without_invented_text() { + for line in [V1_LEGACY, V1_TYPED] { + assert!( + matches!(read_row(line), Ok(AnyRow::V1(_))), + "{:?}", + read_row(line) + ); + } + let Ok(AnyRow::V1(typed)) = read_row(V1_TYPED) else { + panic!("v1 typed row") + }; + let m = typed.migrate().expect("a typed v1 row migrates"); + assert_eq!(m.schema, SCHEME); + assert_eq!(m.lanes[0].findings_count, 2, "the count carries over"); + assert_eq!(m.lanes[0].findings, None, "text never captured stays None"); + assert_eq!((m.pr, m.outcome.as_str()), (Some(4354), "merged")); + // A migrated row round-trips as v2 and still reads as a trace gap. + let line = serde_json::to_string(&m).expect("ser"); + assert_eq!(read_row(&line), Ok(AnyRow::V2(m.clone()))); + assert!(trace_gap(&m.lanes[0])); + // A legacy shadow has no cell/backend: refused, not guessed. + let Ok(AnyRow::V1(legacy)) = read_row(V1_LEGACY) else { + panic!("v1 legacy row") + }; + assert!(legacy.migrate().is_err()); + // A v1 body under the v2 name, and an unknown schema, are errors. + assert!(read_row(&V1_TYPED.replace(SCHEME_V1, SCHEME)).is_err()); + assert!(read_row(&V1_TYPED.replace(SCHEME_V1, "review-ledger-v9")).is_err()); +} diff --git a/crates/aprender-review-experiment/src/ledger_v1.rs b/crates/aprender-review-experiment/src/ledger_v1.rs new file mode 100644 index 0000000000..a4fa330cd0 --- /dev/null +++ b/crates/aprender-review-experiment/src/ledger_v1.rs @@ -0,0 +1,105 @@ +//! The frozen `review-ledger-v1` row, kept so rows written before v2 stay +//! readable. Two shapes were written under that one schema string: the +//! REX-07 original (402aff870, a `state`-tagged shadow) and PRM v2 (a) +//! (89d994bc1, the typed paiml-implement#436 shadow). Both parse here. +//! Nothing writes v1 any more. + +use serde::Deserialize; + +use super::Shadow; + +/// A v1 counted lane: findings were stored as a count only. +#[derive(Debug, Clone, PartialEq, Eq, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct LaneRow { + pub family: Option, + pub model: Option, + pub verdict: Option, + pub findings: usize, +} + +/// 402aff870's reason for a missing shadow verdict. +#[derive(Debug, Clone, PartialEq, Eq, Deserialize)] +pub enum Unknown { + LaneUnavailable { why: Option }, +} + +/// 402aff870's shadow, before the #436 wire form existed. +#[derive(Debug, Clone, PartialEq, Eq, Deserialize)] +#[serde(tag = "state")] +pub enum LegacyShadow { + Answered { verdict: String, served_by: String }, + Unknown { reason: Unknown }, +} + +/// Either v1 shadow; the typed form is tried first. +#[derive(Debug, Clone, PartialEq, Eq, Deserialize)] +#[serde(untagged)] +pub enum V1Shadow { + Typed(Shadow), + Legacy(LegacyShadow), +} + +/// One `review-ledger-v1` row. +#[derive(Debug, Clone, PartialEq, Eq, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Row { + pub schema: String, + pub repo: String, + pub ticket: String, + pub head: String, + pub pr: Option, + pub diff_sha256: Option, + pub agreed: Option, + pub width: usize, + pub lanes: Vec, + pub shadow: V1Shadow, + pub apr_tag: Option, + pub weights_sha256: Option, + pub outcome: String, +} + +impl Row { + /// Migrate to v2. The finding count carries over. Findings text and + /// trace shas were never captured, so they stay `None` (a trace gap), + /// never an empty list. A legacy shadow has no cell or backend, so it + /// has no typed #436 form: it is refused and stays readable as v1. + pub fn migrate(self) -> Result { + let shadow = match self.shadow { + V1Shadow::Typed(s) => s, + V1Shadow::Legacy(l) => { + return Err(format!( + "{}@{}: legacy shadow {l:?} has no typed #436 form", + self.ticket, self.head + )) + } + }; + Ok(super::Row { + schema: super::SCHEME.into(), + repo: self.repo, + ticket: self.ticket, + head: self.head, + pr: self.pr, + diff_sha256: self.diff_sha256, + agreed: self.agreed, + width: self.width, + lanes: self + .lanes + .into_iter() + .map(|l| super::LaneRow { + family: l.family, + model: l.model, + verdict: l.verdict, + findings_count: l.findings, + findings: None, + input_sha256: None, + output_sha256: None, + }) + .collect(), + shadow, + apr_tag: self.apr_tag, + weights_sha256: self.weights_sha256, + outcome: self.outcome, + }) + } +} diff --git a/crates/aprender-review-experiment/src/lib.rs b/crates/aprender-review-experiment/src/lib.rs index 3441691096..7993fd967a 100644 --- a/crates/aprender-review-experiment/src/lib.rs +++ b/crates/aprender-review-experiment/src/lib.rs @@ -1,6 +1,6 @@ -//! REX-001 review-lane experiment (spec `docs/specifications/review-experiment-protocol.md`). +//! PRM-001 PROMETHEUS review-lane experiment (spec `docs/specifications/PRM-001-prometheus.md`; was REX-001). //! -//! - [`prereg`]: the REX-00 pre-registration lock (`rex-prereg-v1`). +//! - [`prereg`]: the REX-00 pre-registration lock (`rex-prereg-v2`). //! - [`stats`]: the frozen analysis code — Wilson, McNemar exact, seeded //! bootstrap, Holm, H6 interference. //! - [`corpus`]: the REX-02 review corpus (`review-corpus-v1`). @@ -10,13 +10,41 @@ //! - [`score`]: the REX-03 scorer (§2.3 metrics, §3 hypothesis inputs). //! - [`harness`]: the REX-03 client that drives a resident `apr serve`. //! - [`admission`]: the REX-04 per-cell admission file (`rex-cell-admission-v1`). +//! - [`pilot`]: the REX-05 pilot projection and the §2.2 sample-size rule. +//! - [`ledger`]: the REX-07 shadow-lane `review-ledger-v1` rows and coverage. +//! - [`ladder`]: the REX-09 shadow → tripwire → vote promotion gates (H4/H5). +//! - [`ratchet`]: the REX-10 `review-lane-perf-ratchet-v1` p95 ratchet (§5.1). +//! - [`champion`]: the REX-11 §5.4 champion/challenger promotion gate. +//! - [`fewshot`]: the REX-11 B1 challengers: prompt versions and leak-proof retrieval few-shot. +//! - [`b2`]: the REX-12 B2 loop: verb-gated row status and the teacher-logit dataset receipt. +//! - [`datacard`]: PRM-C14 (was PRA T14): the Croissant + RAI card, Datasheet and G16 weekly +//! yield/bytes receipt over the agent-trace index (`trace-datacard-v1`). pub mod admission; +pub mod b2; pub mod build_corpus; +pub mod champion; +pub mod cluster; pub mod contamination; pub mod corpus; +pub mod datacard; +pub mod dedup; +pub mod fewshot; pub mod harness; +pub mod kappa_probe; +pub mod ladder; +pub mod ledger; +pub mod outcome_join; +pub mod pilot; +pub mod pool; pub mod prereg; +pub mod ratchet; pub mod receipt; +pub mod replay; pub mod score; +pub mod secret; +pub mod sparse_logits; +pub mod split_guard; pub mod stats; +pub mod terms; +pub mod workload; diff --git a/crates/aprender-review-experiment/src/outcome_join.rs b/crates/aprender-review-experiment/src/outcome_join.rs new file mode 100644 index 0000000000..0e82292d6c --- /dev/null +++ b/crates/aprender-review-experiment/src/outcome_join.rs @@ -0,0 +1,202 @@ +//! PRM-C10 `trace-outcome-join-v1`: PR outcomes and HRQ rulings as gold +//! (gate G-MAT; contract `trace-outcome-join-v1`, FALSIFY-TOJ-001..005). +//! +//! - **Outcome**: from a PR's merge / close / revert / escape events. A revert +//! at most [`WINDOW_DAYS`] after the merge is `reverted_le14d`; a later +//! revert or any escape is `regression_escape`. +//! - **Maturity**: an outcome is gold only once its window has closed, +//! [`WINDOW_DAYS`] UTC dates after the merge (or close). Until then it reads +//! `pending`, and pending is never a negative. +//! - **HRQ rulings** are gold when made; the latest ruling on a round wins. +//! - [`gold`] emits rows whose `label_source` is a [`crate::pool::GOLD_SOURCES`] +//! value; [`matured_between`] is the weekly matured count. + +use std::collections::BTreeMap; + +use serde::{Deserialize, Serialize}; + +use crate::ledger::Verdict; +use crate::split_guard::{date, day}; + +pub const SCHEME: &str = "trace-outcome-join-v1"; + +/// The outcome window, from the merge. +pub const WINDOW_DAYS: i64 = 14; + +/// The v3 lifecycle `outcome`. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum Outcome { + Pending, + Merged, + RevertedLe14d, + RegressionEscape, + ClosedUnmerged, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum EventKind { + Merged, + Closed, + Reverted, + /// A regression traced to the PR. + Escape, +} + +/// One PR event, as the join collects it. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Event { + pub repo: String, + pub pr: u64, + pub kind: EventKind, + /// RFC 3339 UTC. + pub at: String, +} + +/// A PR's outcome as of `now`. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Resolved { + /// `repo#PR`. + pub group_key: String, + pub outcome: Outcome, + /// `YYYY-MM-DD` the window closed; `None` while pending. + pub outcome_matured_at: Option, +} + +/// A human ruling on one quorum round. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Ruling { + pub repo: String, + pub pr: u64, + pub head: String, + pub verdict: Verdict, + pub ruled_by: String, + /// RFC 3339 UTC. + pub at: String, +} + +/// One gold label. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "label_source", rename_all = "snake_case", deny_unknown_fields)] +pub enum Label { + Outcome { + group_key: String, + outcome: Outcome, + outcome_matured_at: String, + }, + HrqRuling { + group_key: String, + head: String, + verdict: Verdict, + ruled_by: String, + at: String, + }, +} + +fn day_of(what: &str, at: &str) -> Result { + day(at).ok_or_else(|| format!("{what}: at {at:?} is not a date")) +} + +/// Each PR's outcome as of `now`, one row per `repo#PR`, sorted by key. +/// A revert or escape with no merge, or an unreadable date, is an error. +pub fn resolve(events: &[Event], now: &str) -> Result, String> { + let today = day_of("now", now)?; + let mut by_pr: BTreeMap> = BTreeMap::new(); + for e in events { + let key = format!("{}#{}", e.repo, e.pr); + let d = day_of(&key, &e.at)?; + by_pr.entry(key).or_default().push((e.kind, d)); + } + by_pr + .into_iter() + .map(|(key, evs)| { + let first = |k: EventKind| evs.iter().filter(|e| e.0 == k).map(|e| e.1).min(); + let (outcome, from) = if let Some(m) = first(EventKind::Merged) { + let late = |k| evs.iter().any(|&(ek, d)| ek == k && d >= m); + let outcome = match first(EventKind::Reverted).filter(|&r| r >= m) { + Some(r) if r - m <= WINDOW_DAYS => Outcome::RevertedLe14d, + Some(_) => Outcome::RegressionEscape, + None if late(EventKind::Escape) => Outcome::RegressionEscape, + None => Outcome::Merged, + }; + (outcome, m) + } else if let Some(c) = first(EventKind::Closed) { + if evs + .iter() + .any(|e| matches!(e.0, EventKind::Reverted | EventKind::Escape)) + { + return Err(format!("{key}: a revert or escape of an unmerged PR")); + } + (Outcome::ClosedUnmerged, c) + } else { + return Err(format!("{key}: a revert or escape with no merge")); + }; + let matured = today - from >= WINDOW_DAYS; + Ok(Resolved { + group_key: key, + outcome: if matured { outcome } else { Outcome::Pending }, + outcome_matured_at: matured.then(|| date(from + WINDOW_DAYS)), + }) + }) + .collect() +} + +/// Gold labels: matured outcomes, then the latest HRQ ruling per round. An +/// outcome that is neither pending nor matured, or a ruling with no ruler or +/// date, is an error. +pub fn gold(resolved: &[Resolved], rulings: &[Ruling]) -> Result, String> { + let mut out = Vec::new(); + for r in resolved { + match (&r.outcome, &r.outcome_matured_at) { + (Outcome::Pending, _) => {} + (&outcome, Some(m)) => out.push(Label::Outcome { + group_key: r.group_key.clone(), + outcome, + outcome_matured_at: m.clone(), + }), + (o, None) => return Err(format!("{}: {o:?} before its window closed", r.group_key)), + } + } + let mut latest: BTreeMap<(String, &str), &Ruling> = BTreeMap::new(); + for r in rulings { + let key = format!("{}#{}", r.repo, r.pr); + if r.ruled_by.trim().is_empty() { + return Err(format!("{key}@{}: ruling has no ruler", r.head)); + } + day_of(&key, &r.at)?; + let e = latest.entry((key, &r.head)).or_insert(r); + if r.at > e.at { + *e = r; + } + } + out.extend( + latest + .into_iter() + .map(|((group_key, head), r)| Label::HrqRuling { + group_key, + head: head.to_owned(), + verdict: r.verdict, + ruled_by: r.ruled_by.clone(), + at: r.at.clone(), + }), + ); + Ok(out) +} + +/// Outcomes whose window closed in `[from, to)` (`YYYY-MM-DD`). +#[must_use] +pub fn matured_between(resolved: &[Resolved], from: &str, to: &str) -> usize { + resolved + .iter() + .filter_map(|r| r.outcome_matured_at.as_deref()) + .filter(|m| *m >= from && *m < to) + .count() +} + +#[cfg(test)] +#[path = "outcome_join_tests.rs"] +mod tests; diff --git a/crates/aprender-review-experiment/src/outcome_join_tests.rs b/crates/aprender-review-experiment/src/outcome_join_tests.rs new file mode 100644 index 0000000000..fd448a74bc --- /dev/null +++ b/crates/aprender-review-experiment/src/outcome_join_tests.rs @@ -0,0 +1,236 @@ +use super::*; + +const NOW: &str = "2026-09-25T12:00:00Z"; + +fn ev(pr: u64, kind: EventKind, at: &str) -> Event { + Event { + repo: "paiml/aprender".into(), + pr, + kind, + at: at.into(), + } +} + +fn ruling(head: &str, verdict: Verdict, ruled_by: &str, at: &str) -> Ruling { + Ruling { + repo: "paiml/aprender".into(), + pr: 7, + head: head.into(), + verdict, + ruled_by: ruled_by.into(), + at: at.into(), + } +} + +fn one(events: &[Event], now: &str) -> Resolved { + let r = resolve(events, now).expect("resolves"); + assert_eq!(r.len(), 1, "{r:?}"); + r.into_iter().next().expect("one") +} + +/// FALSIFY-TOJ-001: a 3-day-old outcome is not gold. Until the 14 d window +/// has closed the PR reads `pending`, and only day 14 matures it. +#[test] +fn falsify_toj_001_immature_outcome_is_refused_as_gold() { + let three = one(&[ev(1, EventKind::Merged, "2026-09-22T09:00:00Z")], NOW); + assert_eq!(three.outcome, Outcome::Pending); + assert_eq!(three.outcome_matured_at, None); + assert!(gold(&[three], &[]).expect("gold").is_empty()); + // A 3-day-old revert is known but still inside the window. + let rev = one( + &[ + ev(1, EventKind::Merged, "2026-09-21T00:00:00Z"), + ev(1, EventKind::Reverted, "2026-09-22T00:00:00Z"), + ], + NOW, + ); + assert_eq!(rev.outcome, Outcome::Pending); + // Days count by UTC date: 09-12 to 09-25 is 13, whatever the hour. + let d13 = one( + &[ev(1, EventKind::Merged, "2026-09-12T00:00:00Z")], + "2026-09-25T23:59:59Z", + ); + assert_eq!(d13.outcome, Outcome::Pending, "13 days is immature"); + let d14 = one(&[ev(1, EventKind::Merged, "2026-09-11T08:00:00Z")], NOW); + assert_eq!(d14.outcome, Outcome::Merged); + assert_eq!(d14.outcome_matured_at.as_deref(), Some("2026-09-25")); + let g = gold(&[d14], &[]).expect("gold"); + assert_eq!( + g, + vec![Label::Outcome { + group_key: "paiml/aprender#1".into(), + outcome: Outcome::Merged, + outcome_matured_at: "2026-09-25".into(), + }] + ); +} + +/// FALSIFY-TOJ-002: the terminal outcome. A revert on day ≤ 14 after the +/// merge is `reverted_le14d`; a later revert or any escape is +/// `regression_escape`; a PR closed without a merge is `closed_unmerged`. +#[test] +fn falsify_toj_002_outcome_follows_the_revert_window() { + let old = "2026-08-01T00:00:00Z"; + let at = |kind, at| one(&[ev(2, EventKind::Merged, old), ev(2, kind, at)], NOW).outcome; + assert_eq!( + at(EventKind::Reverted, "2026-08-15T23:00:00Z"), + Outcome::RevertedLe14d + ); + assert_eq!( + at(EventKind::Reverted, "2026-08-16T00:00:00Z"), + Outcome::RegressionEscape + ); + assert_eq!( + at(EventKind::Escape, "2026-08-03T00:00:00Z"), + Outcome::RegressionEscape + ); + assert_eq!( + one(&[ev(2, EventKind::Closed, old)], NOW).outcome, + Outcome::ClosedUnmerged + ); + assert_eq!( + one(&[ev(2, EventKind::Merged, old)], NOW).outcome, + Outcome::Merged + ); + // A merged PR's close event does not make it unmerged. + assert_eq!( + one( + &[ev(2, EventKind::Closed, old), ev(2, EventKind::Merged, old)], + NOW + ) + .outcome, + Outcome::Merged + ); + // The window runs from the merge, not from an earlier event. + let r = one( + &[ + ev(2, EventKind::Reverted, "2026-08-20T00:00:00Z"), + ev(2, EventKind::Merged, "2026-08-10T00:00:00Z"), + ], + NOW, + ); + assert_eq!(r.outcome, Outcome::RevertedLe14d); + assert_eq!(r.outcome_matured_at.as_deref(), Some("2026-08-24")); + // A revert or escape dated before the merge is not of this merge. + for kind in [EventKind::Reverted, EventKind::Escape] { + let early = one( + &[ + ev(2, kind, "2026-08-05T00:00:00Z"), + ev(2, EventKind::Merged, "2026-08-10T00:00:00Z"), + ], + NOW, + ); + assert_eq!(early.outcome, Outcome::Merged, "{kind:?}"); + } + assert_eq!( + serde_json::to_string(&Outcome::RevertedLe14d).expect("ser"), + "\"reverted_le14d\"" + ); +} + +/// FALSIFY-TOJ-003: pending is never a negative, and a join that cannot be +/// read is an error, never a guess. +#[test] +fn falsify_toj_003_pending_is_never_a_label_and_bad_joins_refuse() { + let open = one(&[ev(4, EventKind::Merged, "2026-09-24T00:00:00Z")], NOW); + assert_eq!(open.outcome, Outcome::Pending); + assert!(gold(&[open], &[]).expect("gold").is_empty()); + for bad in [ + vec![ev(3, EventKind::Reverted, "2026-08-01T00:00:00Z")], + vec![ev(3, EventKind::Escape, "2026-08-01T00:00:00Z")], + vec![ + ev(3, EventKind::Closed, "2026-08-01T00:00:00Z"), + ev(3, EventKind::Reverted, "2026-08-02T00:00:00Z"), + ], + vec![ev(3, EventKind::Merged, "yesterday")], + ] { + assert!(resolve(&bad, NOW).is_err(), "{bad:?}"); + } + assert!(resolve(&[], "now").is_err()); + // A hand-built immature row is refused, not silently dropped. + let forged = Resolved { + group_key: "paiml/aprender#9".into(), + outcome: Outcome::Merged, + outcome_matured_at: None, + }; + assert!(gold(&[forged], &[]).is_err()); +} + +/// FALSIFY-TOJ-004: an HRQ ruling is gold as soon as it is made; the latest +/// ruling on a round wins; a ruling with no ruler is refused. +#[test] +fn falsify_toj_004_hrq_rulings_are_gold() { + let g = gold( + &[], + &[ + ruling("abc", Verdict::Pass, "noah", "2026-09-25T10:00:00Z"), + ruling("abc", Verdict::Fail, "noah", "2026-09-25T11:00:00Z"), + ruling("def", Verdict::Pass, "noah", "2026-09-25T09:00:00Z"), + ], + ) + .expect("gold"); + assert_eq!( + g, + vec![ + Label::HrqRuling { + group_key: "paiml/aprender#7".into(), + head: "abc".into(), + verdict: Verdict::Fail, + ruled_by: "noah".into(), + at: "2026-09-25T11:00:00Z".into(), + }, + Label::HrqRuling { + group_key: "paiml/aprender#7".into(), + head: "def".into(), + verdict: Verdict::Pass, + ruled_by: "noah".into(), + at: "2026-09-25T09:00:00Z".into(), + }, + ] + ); + let v = serde_json::to_value(&g[0]).expect("ser"); + assert_eq!(v["label_source"], "hrq_ruling"); + assert!(crate::pool::GOLD_SOURCES.contains(&v["label_source"].as_str().expect("str"))); + for bad in [ + ruling("abc", Verdict::Pass, " ", "2026-09-25T10:00:00Z"), + ruling("abc", Verdict::Pass, "noah", "today"), + ] { + assert!(gold(&[], &[bad]).is_err()); + } +} + +/// FALSIFY-TOJ-005: the weekly count is the outcomes whose window closed in +/// `[from, to)`, and nothing still pending. +#[test] +fn falsify_toj_005_weekly_matured_count() { + let rs = resolve( + &[ + ev(10, EventKind::Merged, "2026-09-04T00:00:00Z"), // matures 09-18: in + ev(11, EventKind::Closed, "2026-09-10T00:00:00Z"), // matures 09-24: in + ev(12, EventKind::Merged, "2026-09-11T00:00:00Z"), // matures 09-25: out (to) + ev(13, EventKind::Merged, "2026-09-03T00:00:00Z"), // matures 09-17: out (from-1) + ev(14, EventKind::Merged, "2026-09-20T00:00:00Z"), // pending + ], + NOW, + ) + .expect("resolves"); + assert_eq!(rs.len(), 5); + assert_eq!(matured_between(&rs, "2026-09-18", "2026-09-25"), 2); + assert_eq!(matured_between(&rs, "2026-09-17", "2026-09-26"), 4); + let v = serde_json::to_value(gold(&rs, &[]).expect("gold").first().expect("one")).expect("ser"); + assert_eq!(v["label_source"], "outcome"); +} + +#[test] +fn date_inverts_day() { + for s in [ + "1970-01-01", + "2000-02-29", + "2024-12-31", + "2026-09-25", + "2100-03-01", + ] { + let d = crate::split_guard::day(s).expect("day"); + assert_eq!(crate::split_guard::date(d), s); + } +} diff --git a/crates/aprender-review-experiment/src/pilot.rs b/crates/aprender-review-experiment/src/pilot.rs new file mode 100644 index 0000000000..058c94ea7b --- /dev/null +++ b/crates/aprender-review-experiment/src/pilot.rs @@ -0,0 +1,107 @@ +//! REX-05 pilot projection: the variance and runtime of the dev-item pilot on +//! one admitted cell, and the §2.2 sample-size rule applied to it. +//! +//! The rule (analysis plan §Sample-size rule): project the test-split Wilson +//! half-width on recall at the pilot's point estimate; above +//! [`MAX_HALF_WIDTH`] the corpus grows, and the hypotheses do not change. +//! A pilot with no defect item in its denominator has no point estimate, so +//! the rule falls back to the §2.2 design basis p = 0.5 and says so. + +use crate::score::Ratio; +use crate::stats::{percentile, sample_sd, wilson_half_width}; +use serde::Serialize; + +/// §2.2: grow the corpus when the projected half-width exceeds this. +pub const MAX_HALF_WIDTH: f64 = 0.12; + +/// Where the projected recall came from. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)] +pub enum Basis { + /// The pilot's own recall point estimate. + Pilot, + /// No defect item in the pilot denominator: the §2.2 design p = 0.5. + DesignWorstCase, +} + +/// Spread of one timing sample, in milliseconds. +#[derive(Debug, Clone, Copy, PartialEq, Serialize)] +pub struct Spread { + pub n: usize, + pub mean: f64, + /// `None` below two values. + pub sd: Option, + pub p50: f64, + pub p95: f64, +} + +impl Spread { + /// `None` for an empty sample (nothing ran, so there is no spread). + #[must_use] + pub fn of(x: &[f64]) -> Option { + if x.is_empty() || x.iter().any(|v| !v.is_finite()) { + return None; + } + Some(Self { + n: x.len(), + mean: x.iter().sum::() / x.len() as f64, + sd: sample_sd(x), + p50: percentile(x, 0.50), + p95: percentile(x, 0.95), + }) + } +} + +/// What the pilot projects for the full run on this cell. +#[derive(Debug, Clone, PartialEq, Serialize)] +pub struct Projection { + pub recall: Ratio, + pub basis: Basis, + pub p: f64, + pub n_defect_test: u64, + /// Wilson 95 % half-width at `p` on `n_defect_test` (`None` when n = 0). + pub half_width: Option, + /// §2.2: true when `half_width` exceeds [`MAX_HALF_WIDTH`] or cannot be + /// computed. An unknown width is never a reason to keep the corpus. + pub grow_corpus: bool, + pub warm: Option, + pub cold: Option, + /// Test items plus the 10 % determinism re-run. + pub projected_requests: u64, + /// `projected_requests` × warm mean; `None` when nothing warm ran. + pub projected_runtime_s: Option, +} + +/// Apply the §2.2 rule and the runtime projection to one pilot. +#[must_use] +pub fn project( + recall: Ratio, + warm_wall_ms: &[f64], + cold_wall_ms: &[f64], + n_defect_test: u64, + n_test: u64, +) -> Projection { + let (basis, p) = match recall.value() { + Some(p) => (Basis::Pilot, p), + None => (Basis::DesignWorstCase, 0.5), + }; + let k = (p * n_defect_test as f64).round() as u64; + let half_width = wilson_half_width(k, n_defect_test); + let warm = Spread::of(warm_wall_ms); + let projected_requests = n_test + n_test.div_ceil(10); + Projection { + recall, + basis, + p, + n_defect_test, + half_width, + grow_corpus: half_width.is_none_or(|h| h > MAX_HALF_WIDTH), + warm, + cold: Spread::of(cold_wall_ms), + projected_requests, + projected_runtime_s: warm.map(|w| w.mean * projected_requests as f64 / 1e3), + } +} + +#[cfg(test)] +#[path = "pilot_tests.rs"] +mod tests; diff --git a/crates/aprender-review-experiment/src/pilot_tests.rs b/crates/aprender-review-experiment/src/pilot_tests.rs new file mode 100644 index 0000000000..5ebf9a50e4 --- /dev/null +++ b/crates/aprender-review-experiment/src/pilot_tests.rs @@ -0,0 +1,49 @@ +use super::*; + +#[test] +fn the_design_point_sits_just_under_the_rule() { + // §2.2 [C] quotes about 0.117 for 70 defect items at p = 0.5; that is the + // Wald width. The frozen Wilson width the rule reads is about 0.114. + let pr = project(Ratio::new(0, 0), &[], &[], 70, 140); + assert_eq!(pr.basis, Basis::DesignWorstCase); + let h = pr.half_width.expect("n > 0"); + assert!((h - 0.114).abs() < 0.001, "{h}"); + assert!(!pr.grow_corpus); +} + +#[test] +fn falsify_rxp_001_a_wide_projection_grows_the_corpus() { + // 40 defect items at the pilot's p = 0.5: half-width about 0.15 > 0.12. + let pr = project(Ratio::new(3, 6), &[1.0], &[], 40, 80); + assert_eq!(pr.basis, Basis::Pilot); + assert!(pr.half_width.expect("n > 0") > MAX_HALF_WIDTH); + assert!(pr.grow_corpus, "a > 0.12 projection must grow the corpus"); + // An unknown width (no test defects) is never a reason to keep the corpus. + assert!(project(Ratio::new(3, 6), &[1.0], &[], 0, 80).grow_corpus); +} + +#[test] +fn falsify_rxp_002_the_projection_uses_the_pilot_estimate() { + // A high pilot recall narrows the interval below the rule at n = 50. + let pr = project(Ratio::new(9, 10), &[1.0], &[], 50, 100); + assert_eq!(pr.p, 0.9); + assert!(!pr.grow_corpus, "{:?}", pr.half_width); + // The same n at the design p = 0.5 would grow it: the estimate matters. + assert!(project(Ratio::new(5, 10), &[1.0], &[], 50, 100).grow_corpus); +} + +#[test] +fn falsify_rxp_003_runtime_counts_the_rerun_and_nothing_warm_is_unknown() { + let pr = project(Ratio::new(1, 2), &[1000.0, 3000.0], &[9000.0], 70, 145); + assert_eq!(pr.projected_requests, 145 + 15); + assert_eq!(pr.projected_runtime_s, Some(2.0 * 160.0)); + let w = pr.warm.expect("warm ran"); + assert_eq!((w.n, w.p50, w.p95), (2, 1000.0, 3000.0)); + assert_eq!(pr.cold.map(|c| c.n), Some(1)); + let none = project(Ratio::new(1, 2), &[], &[], 70, 145); + assert_eq!( + none.projected_runtime_s, None, + "no warm sample is not zero seconds" + ); + assert!(Spread::of(&[1.0, f64::NAN]).is_none()); +} diff --git a/crates/aprender-review-experiment/src/pool.rs b/crates/aprender-review-experiment/src/pool.rs new file mode 100644 index 0000000000..aaaab14578 --- /dev/null +++ b/crates/aprender-review-experiment/src/pool.rs @@ -0,0 +1,213 @@ +//! Training-pool admission gate (ruling agent-trace item 6; PRA-001 T6, +//! contract `trace-admission-secret-v1`). +//! +//! A captured row enters the fine-tune/distill pool only when it is secret-scan +//! clean under BOTH scanners of [`crate::secret`] AND carries no sealed-test +//! hash (`review-corpus-contamination-v1`). +//! A secret hit QUARANTINES the row as-is: it is never redacted in place, since a +//! redacted row is a row the scanner already failed on once. A sealed-test hit +//! REFUSES the row. +//! +//! G-PROV (PRM-001 v3 R-18, contract `trace-admission-prov-v1`): a row is +//! train-eligible only when it is a local row (qwen, the 27B teacher: provider +//! `local`, fully terms-tagged per [`crate::terms::tags`]) or a gold label +//! (`label_source` one of [`GOLD_SOURCES`]). Hosted outputs are never +//! eligible, whatever else the row claims, and an untagged row cannot be shown +//! not to be hosted, so it is ineligible too. + +use std::collections::BTreeMap; + +use serde::Serialize; +use serde_json::Value; + +use crate::contamination::Index; +use crate::secret::{scan, Hit}; +use crate::terms::{tags, Gap, Provider}; + +/// The `label_source` values that make a row a gold label (v3 G-PROV): a +/// matured outcome, an HRQ ruling, a sealed-corpus label. +pub const GOLD_SOURCES: [&str; 3] = ["outcome", "hrq_ruling", "sealed_corpus"]; + +/// Why a row fails G-PROV. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum Prov { + /// A hosted provider's output: never a training target (R-18). + Hosted(Provider), + /// Terms tags missing or inconsistent: provenance unproven. + Untagged(Vec), + /// A `label_source` that is not a gold source. + NotGold(String), +} + +/// Where one input row went. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum Verdict { + /// Clean: the row is in the pool. + Admitted, + /// A secret was found; the row is held byte-for-byte outside the pool. + Quarantined { secrets: Vec }, + /// The row carries a sealed test item; it never enters the pool. + Refused { items: Vec }, + /// Not a local row nor a gold label (G-PROV); it never enters the pool. + Ineligible(Prov), + /// The sealed index is empty, so contamination is unproven either way. + Unchecked, +} + +/// The outcome for a JSONL batch: one verdict per non-empty line, and the pool. +#[derive(Debug, Default)] +pub struct Admission { + pub verdicts: Vec<(String, Verdict)>, +} + +impl Admission { + /// The rows admitted to the training pool, in input order. + #[must_use] + pub fn pool(&self) -> Vec<&str> { + self.with(|v| matches!(v, Verdict::Admitted)) + } + + /// The quarantined rows, byte-identical to their input. + #[must_use] + pub fn quarantine(&self) -> Vec<&str> { + self.with(|v| matches!(v, Verdict::Quarantined { .. })) + } + + /// The backfill receipt (PRM-C6): one count per verdict, and how many rows + /// each secret rule quarantined, keyed `scanner:rule`. + #[must_use] + pub fn receipt(&self) -> Backfill { + let mut b = Backfill { + rows: self.verdicts.len(), + ..Backfill::default() + }; + for (_, v) in &self.verdicts { + match v { + Verdict::Admitted => b.admitted += 1, + Verdict::Quarantined { secrets } => { + b.quarantined += 1; + for h in secrets { + let key = format!("{:?}:{}", h.scanner, h.rule).to_lowercase(); + *b.rules.entry(key).or_default() += 1; + } + } + Verdict::Refused { .. } => b.refused += 1, + Verdict::Ineligible(_) => b.ineligible += 1, + Verdict::Unchecked => b.unchecked += 1, + } + } + b + } + + fn with(&self, keep: impl Fn(&Verdict) -> bool) -> Vec<&str> { + self.verdicts + .iter() + .filter(|(_, v)| keep(v)) + .map(|(r, _)| r.as_str()) + .collect() + } +} + +/// Counts for one backfill over a JSONL batch ([`Admission::receipt`]). +#[derive(Debug, Default, Clone, PartialEq, Eq, Serialize)] +pub struct Backfill { + pub rows: usize, + pub admitted: usize, + pub quarantined: usize, + pub refused: usize, + pub ineligible: usize, + pub unchecked: usize, + /// Quarantined rows per `scanner:rule` (a row with two hits counts twice). + pub rules: BTreeMap, +} + +/// Admit a JSONL batch of captured rows against the sealed-test index. +/// +/// Sealed-test hits are checked first: such a row is refused outright, even if +/// it also carries a secret, so no test item is ever stored beside training data. +#[must_use] +pub fn admit(rows: &str, sealed: &Index) -> Admission { + let verdicts = rows + .lines() + .filter(|l| !l.trim().is_empty()) + .map(|l| (l.to_string(), verdict(l, sealed))) + .collect(); + Admission { verdicts } +} + +fn verdict(row: &str, sealed: &Index) -> Verdict { + if sealed.is_empty() { + return Verdict::Unchecked; + } + let mut items: Vec = sealed + .scan(row, "row") + .into_iter() + .map(|h| h.item) + .collect(); + items.dedup(); + if !items.is_empty() { + return Verdict::Refused { items }; + } + // Before the secret scan, so a hosted row never reaches the quarantine, + // whose rows may be released after review. + if let Err(p) = provenance(row) { + return Verdict::Ineligible(p); + } + let found = scan(row); + if found.is_empty() { + Verdict::Admitted + } else { + Verdict::Quarantined { secrets: found } + } +} + +/// G-PROV for one row. +/// +/// # Errors +/// The row is hosted, untagged, or labelled by a non-gold source. +pub fn provenance(row: &str) -> Result<(), Prov> { + let Ok(v) = serde_json::from_str::(row) else { + return Err(Prov::Untagged(vec![Gap::NotJson])); + }; + // A hosted provider refuses first: a gold label never carries a hosted output. + if let Some(p) = v + .get("provider") + .and_then(Value::as_str) + .and_then(Provider::parse) + { + if p != Provider::Local { + return Err(Prov::Hosted(p)); + } + } + if let Some(s) = v.get("label_source") { + return match s.as_str() { + Some(g) if GOLD_SOURCES.contains(&g) => Ok(()), + _ => Err(Prov::NotGold(s.to_string())), + }; + } + // Hosted providers returned above, so tagged here means local. + tags(&v).map(|_| ()).map_err(Prov::Untagged) +} + +/// Test fixture: `v` with the terms tags of a local (apr-serve) row added. +#[cfg(test)] +pub(crate) fn local_tagged(mut v: Value) -> String { + let terms = r#"{"url":"https://www.anthropic.com/legal/commercial-terms","effective_date":"2025-06-17","fetched_at":"2026-09-25T17:00:00Z"}"#; + if let Value::Object(m) = &mut v { + m.insert("provider".into(), "local".into()); + m.insert( + "model_id".into(), + "3f1a9c0b5e2d4f6a8b7c9d0e1f2a3b4c5d6e7f8091a2b3c4d5e6f708192a3b4c".into(), + ); + m.insert("access_channel".into(), "apr-serve".into()); + m.insert( + "terms_ref".into(), + serde_json::from_str(terms).expect("fixture terms_ref is JSON"), + ); + } + v.to_string() +} + +#[cfg(test)] +#[path = "pool_tests.rs"] +mod tests; diff --git a/crates/aprender-review-experiment/src/pool_tests.rs b/crates/aprender-review-experiment/src/pool_tests.rs new file mode 100644 index 0000000000..193041b026 --- /dev/null +++ b/crates/aprender-review-experiment/src/pool_tests.rs @@ -0,0 +1,229 @@ +// `json!` expands to an `unwrap` of an infallible `to_value`. +#![allow(clippy::disallowed_methods)] + +use super::*; +use crate::corpus::{hunk_fingerprints, sha256_hex, Sealed}; +use crate::secret::{Hit, Scanner}; +use crate::terms::Provider; + +const TEST_DIFF: &str = "--- a/crates/x/src/a.rs\n+++ b/crates/x/src/a.rs\n@@ -10,5 +10,5 @@\n fn f(x: u32) -> bool {\n- x > 1\n+ x >= 1\n }\n \n"; + +fn index() -> Index { + Index::new(&[Sealed { + id: "R007".into(), + diff_sha256: sha256_hex(TEST_DIFF.as_bytes()), + hunks: hunk_fingerprints(TEST_DIFF), + }]) +} + +/// AWS's documented example access key id, assembled at run time so the source +/// file itself carries no key literal for repo secret scanners to flag. +fn planted_aws_key() -> String { + ["AKIA", "IOSFODNN7", "EXAMPLE"].concat() +} + +/// A local (qwen) lane row: G-PROV eligible, so only the check under test decides. +fn row(output: &str) -> String { + local_tagged(serde_json::json!({"lane": "qwen", "output": output})) +} + +/// FALSIFY-TAS-004: a planted AWS key literal in a lane output is quarantined, +/// verbatim, and is not in the pool; a clean neighbour is admitted. +#[test] +fn falsify_tas_004_planted_aws_key_is_quarantined() { + let dirty = row(&format!("export AWS_ACCESS_KEY_ID={}", planted_aws_key())); + let clean = row("LGTM: the bound check is inclusive now."); + let a = admit(&format!("{clean}\n{dirty}\n"), &index()); + assert_eq!(a.verdicts[0], (clean.clone(), Verdict::Admitted)); + assert_eq!( + a.verdicts[1], + ( + dirty.clone(), + Verdict::Quarantined { + secrets: vec![Hit { + scanner: Scanner::Builtin, + rule: "aws-access-key-id" + }] + } + ), + "the row is held byte-for-byte, never redacted in place" + ); + assert_eq!(a.pool(), vec![clean.as_str()]); + assert_eq!(a.quarantine(), vec![dirty.as_str()]); +} + +/// FALSIFY-TAS-005: a row carrying a sealed test-split diff is refused and not +/// in the pool, as is a row carrying only the item's diff sha. +#[test] +fn falsify_tas_005_planted_test_split_diff_is_refused() { + let leak = serde_json::json!({"lane": "haiku-4-5", "input": TEST_DIFF}).to_string(); + let sha = row(&format!("reviewed {}", sha256_hex(TEST_DIFF.as_bytes()))); + let clean = row("no findings"); + let a = admit(&format!("{leak}\n{sha}\n{clean}\n"), &index()); + for (i, r) in [&leak, &sha].into_iter().enumerate() { + assert_eq!( + a.verdicts[i], + ( + r.clone(), + Verdict::Refused { + items: vec!["R007".into()] + } + ) + ); + } + assert_eq!(a.pool(), vec![clean.as_str()]); +} + +/// An empty sealed index proves nothing about contamination: admission refuses +/// to run rather than admit everything vacuously. +#[test] +fn empty_sealed_index_refuses_everything() { + let a = admit(&format!("{}\n", row("no findings")), &Index::new(&[])); + assert!(a.pool().is_empty()); +} + +/// A correctly tagged hosted lane row: clean of secrets and sealed items, so +/// only G-PROV can keep it out. +fn hosted(provider: &str, model: &str, channel: &str) -> String { + let mut v: serde_json::Value = serde_json::from_str(&row("LGTM")).expect("row is JSON"); + v["provider"] = provider.into(); + v["model_id"] = model.into(); + v["access_channel"] = channel.into(); + v.to_string() +} + +/// FALSIFY-TAP-001 (G-PROV, R-18): a planted hosted row in a pool is +/// ineligible and not in the pool, for each hosted provider; its local +/// neighbour is admitted. +#[test] +fn falsify_tap_001_planted_hosted_row_is_ineligible() { + let claude = hosted("anthropic", "claude-sonnet-5", "claude-code-cli"); + let gemini = hosted("google", "gemini-3.1-pro-high", "antigravity"); + let local = row("LGTM"); + assert!(crate::terms::tags(&serde_json::from_str(&claude).expect("json")).is_ok()); + let a = admit(&format!("{claude}\n{gemini}\n{local}\n"), &index()); + assert_eq!( + a.verdicts[0].1, + Verdict::Ineligible(Prov::Hosted(Provider::Anthropic)) + ); + assert_eq!( + a.verdicts[1].1, + Verdict::Ineligible(Prov::Hosted(Provider::Google)) + ); + assert_eq!( + a.pool(), + vec![local.as_str()], + "hosted rows in the pool = 0" + ); +} + +/// FALSIFY-TAP-002: a hosted row cannot launder itself: claiming a gold +/// `label_source`, or carrying a secret, it is still ineligible (never +/// quarantined, whose rows may be released). +#[test] +fn falsify_tap_002_hosted_row_is_ineligible_whatever_it_claims() { + let mut v: serde_json::Value = + serde_json::from_str(&hosted("anthropic", "claude-sonnet-5", "anthropic-api")) + .expect("json"); + v["label_source"] = "outcome".into(); + let gold_claim = v.to_string(); + v["output"] = format!("AWS_ACCESS_KEY_ID={}", planted_aws_key()).into(); + let with_secret = v.to_string(); + let a = admit(&format!("{gold_claim}\n{with_secret}\n"), &index()); + for (_, verdict) in &a.verdicts { + assert_eq!( + *verdict, + Verdict::Ineligible(Prov::Hosted(Provider::Anthropic)) + ); + } + assert!(a.pool().is_empty() && a.quarantine().is_empty()); +} + +/// FALSIFY-TAP-003: an untagged or mis-tagged row is ineligible: provenance +/// that cannot be shown local is not local. +#[test] +fn falsify_tap_003_untagged_row_is_ineligible() { + let bare = serde_json::json!({"lane": "sonnet-5", "output": "LGTM"}).to_string(); + let mut v: serde_json::Value = serde_json::from_str(&row("LGTM")).expect("json"); + v["provider"] = "Anthropic".into(); + let misspelt = v.to_string(); + let a = admit(&format!("{bare}\n{misspelt}\nnot json\n"), &index()); + for (r, verdict) in &a.verdicts { + assert!( + matches!(verdict, Verdict::Ineligible(Prov::Untagged(g)) if !g.is_empty()), + "{r}: {verdict:?}" + ); + } + assert!(a.pool().is_empty()); +} + +/// FALSIFY-TAP-004: a gold label (each source) is eligible without provider +/// tags; any other `label_source` is not. +#[test] +fn falsify_tap_004_gold_labels_are_eligible_and_only_gold() { + let gold: Vec = GOLD_SOURCES + .iter() + .map(|s| serde_json::json!({"pr": 4354, "label_source": s, "defect": true}).to_string()) + .collect(); + let silver = serde_json::json!({"pr": 4354, "label_source": "quorum_majority"}).to_string(); + let a = admit(&format!("{}\n{silver}\n", gold.join("\n")), &index()); + assert_eq!( + a.pool(), + gold.iter().map(String::as_str).collect::>() + ); + assert_eq!( + a.verdicts[3].1, + Verdict::Ineligible(Prov::NotGold("\"quorum_majority\"".into())) + ); +} + +/// FALSIFY-TAS-009 (PRM-C6 acceptance): a backfill over the AWS key literal and +/// the 20 labelled canaries, each in its own lane row, quarantines all 21 +/// byte-identical, admits the clean neighbour, and the receipt counts every +/// canary's labelled rule. +#[test] +fn falsify_tas_009_backfill_quarantines_the_aws_key_and_twenty_canaries() { + let canaries = crate::secret::tests::canaries(); + assert_eq!(canaries.len(), 20, "precondition: the canary set is 20"); + let mut planted = vec![row(&format!("key {}", planted_aws_key()))]; + planted.extend(canaries.iter().map(|(_, text)| row(text))); + let clean = row("LGTM: the bound is inclusive, as the test expects."); + let batch = format!("{}\n{clean}\n", planted.join("\n")); + + let a = admit(&batch, &index()); + let r = a.receipt(); + assert_eq!( + ( + r.rows, + r.quarantined, + r.admitted, + r.refused, + r.ineligible, + r.unchecked + ), + (22, 21, 1, 0, 0, 0), + "{r:?}" + ); + assert_eq!( + a.quarantine(), + planted.iter().map(String::as_str).collect::>() + ); + assert_eq!(a.pool(), vec![clean.as_str()]); + for (hit, _) in &canaries { + let key = format!("{:?}:{}", hit.scanner, hit.rule).to_lowercase(); + assert!( + r.rules.get(&key).is_some_and(|n| *n >= 1), + "{key} not counted: {:?}", + r.rules + ); + } + assert!( + r.rules.keys().any(|k| k.starts_with("builtin:aws")), + "the AWS key needs the builtin scanner: {:?}", + r.rules + ); + + // No sealed index: nothing is admitted and every row is counted unchecked. + let r = admit(&batch, &Index::new(&[])).receipt(); + assert_eq!((r.admitted, r.unchecked), (0, 22), "{r:?}"); +} diff --git a/crates/aprender-review-experiment/src/prereg.rs b/crates/aprender-review-experiment/src/prereg.rs index d21ca0dd91..a758956f8d 100644 --- a/crates/aprender-review-experiment/src/prereg.rs +++ b/crates/aprender-review-experiment/src/prereg.rs @@ -1,4 +1,9 @@ -//! REX-00 pre-registration lock (contract `rex-prereg-v1`, rule R-1). +//! PRM-00b pre-registration lock (contract `rex-prereg-v1`, scheme `rex-prereg-v2`, rule R-1). +//! +//! `rex-prereg-v2` locks PRM-001 v3 (`docs/specifications/PRM-001-prometheus.md`). The v1 +//! lock (`docs/audits/rex-001/prereg.lock`, prereg_sha `ef51087d…`) is SUPERSEDED, never +//! deleted or edited (operator, 2026-09-25); its spec and plan stay byte-frozen in-tree and +//! `falsify_rex_prereg_004` keeps them verifying. //! //! The prereg sha is a sha256 over the sha256 of four frozen components, in a //! fixed order: @@ -6,29 +11,37 @@ //! 1. `spec_s2_s5` — spec bytes from the line `## §2 ` up to (not including) //! the line `## §6 ` (design, hypotheses, decision rule, improvement loop); //! 2. `stats_rs` — the analysis code; -//! 3. `analysis_plan` — `docs/audits/rex-001/analysis-plan.md`; +//! 3. `analysis_plan` — `docs/audits/prm-001/analysis-plan.md` (plan v2); //! 4. `prompt_v1` — the fixed review prompt. //! //! The sealed test-item manifest is NOT a prereg component: it is filled once by //! REX-02 (placeholder → sealed) and carried separately as the corpus version. -//! Everything outside §2–§5 (status line, §7 tickets, §9 schema) may be edited +//! Everything outside §2–§5 (header, §7 tickets, §9 schema) may be edited //! without a new spec version. use sha2::{Digest, Sha256}; /// The spec as committed in-tree. -pub const SPEC: &str = include_str!("../../../docs/specifications/review-experiment-protocol.md"); +pub const SPEC: &str = include_str!("../../../docs/specifications/PRM-001-prometheus.md"); /// The analysis code. pub const STATS_RS: &str = include_str!("stats.rs"); /// The analysis plan. -pub const ANALYSIS_PLAN: &str = include_str!("../../../docs/audits/rex-001/analysis-plan.md"); +pub const ANALYSIS_PLAN: &str = include_str!("../../../docs/audits/prm-001/analysis-plan.md"); /// Prompt v1 (the G1 prompt header; the diff is appended after it). pub const PROMPT_V1: &str = include_str!("../../../docs/audits/review-corpus/prompts/v1.txt"); /// The committed lock. -pub const LOCK: &str = include_str!("../../../docs/audits/rex-001/prereg.lock"); +pub const LOCK: &str = include_str!("../../../docs/audits/prm-001/prereg.lock"); + +/// The superseded v1 lock and the v1 spec and plan it froze (never edited). +pub const LOCK_V1: &str = include_str!("../../../docs/audits/rex-001/prereg.lock"); +/// The v1 spec (`rex-prereg-v1`), byte-frozen in §2–§5. +pub const SPEC_V1: &str = + include_str!("../../../docs/specifications/review-experiment-protocol.md"); +/// The v1 analysis plan, byte-frozen. +pub const ANALYSIS_PLAN_V1: &str = include_str!("../../../docs/audits/rex-001/analysis-plan.md"); /// Contract id stamped into the digest so a lock can never match another scheme. -pub const SCHEME: &str = "rex-prereg-v1"; +pub const SCHEME: &str = "rex-prereg-v2"; /// Lowercase hex sha256. #[must_use] @@ -99,7 +112,7 @@ impl Components { #[must_use] pub fn render_lock(&self) -> String { format!( - "# REX-001 pre-registration lock (rex-prereg-v1). Written once by REX-00.\n\ + "# PRM-001 pre-registration lock (rex-prereg-v2) over spec PRM-001 v3. Written by PRM-00b; v1 had 0 data rows.\n\ # A mismatch against the tree is a new spec version (R-1), never an edit here.\n\ spec_s2_s5={}\nstats_rs={}\nanalysis_plan={}\nprompt_v1={}\nprereg_sha={}\n", self.spec_s2_s5, @@ -167,7 +180,7 @@ mod tests { /// FALSIFY-REX-PREREG-002: a planted edit to §3 changes the prereg sha. #[test] fn falsify_rex_prereg_002_planted_section3_edit_is_caught() { - let planted = SPEC.replacen("McNemar exact, paired, one-sided", "t-test", 1); + let planted = SPEC.replacen("McNemar exact, one-sided", "t-test", 1); assert_ne!(planted, SPEC, "the §3 anchor text must exist"); let c = Components::of(&planted, STATS_RS, ANALYSIS_PLAN, PROMPT_V1).expect("span"); assert!(!verify(LOCK, &c).is_empty()); @@ -177,8 +190,8 @@ mod tests { #[test] fn falsify_rex_prereg_003_status_line_edit_is_not_a_new_version() { let edited = SPEC.replacen( - "**Status:** spec, not implemented.", - "**Status:** implemented.", + "**Runner:** the aprender traffic cop", + "**Runner:** the fleet", 1, ); assert_ne!(edited, SPEC); @@ -186,6 +199,35 @@ mod tests { assert!(verify(LOCK, &c).is_empty()); } + /// FALSIFY-REX-PREREG-004: the superseded v1 lock is untouched and the v1 + /// spec §2–§5, plan and prompt it froze still verify against it. (v1 + /// `stats_rs` is verified against git `5f0ae10ac`; the code moved on.) + #[test] + fn falsify_rex_prereg_004_v1_lock_is_frozen_and_verifies() { + assert_eq!( + lock_value(LOCK_V1, "prereg_sha"), + Some("ef51087dc79bab0ad160e8a14f5b13e2ea43986b30c05dafc63c84c2dc21cdc0") + ); + assert!(LOCK_V1.starts_with("# REX-001 pre-registration lock (rex-prereg-v1).")); + let span = spec_sections_2_to_5(SPEC_V1).expect("v1 §2..§6"); + assert_eq!( + lock_value(LOCK_V1, "spec_s2_s5"), + Some(sha256_hex(span.as_bytes()).as_str()) + ); + assert_eq!( + lock_value(LOCK_V1, "analysis_plan"), + Some(sha256_hex(ANALYSIS_PLAN_V1.as_bytes()).as_str()) + ); + assert_eq!( + lock_value(LOCK_V1, "prompt_v1"), + Some(sha256_hex(PROMPT_V1.as_bytes()).as_str()) + ); + assert_ne!( + lock_value(LOCK_V1, "prereg_sha"), + locked_prereg_sha(), + "v2 is a new lock" + ); + } #[test] fn span_requires_both_headings() { assert!(spec_sections_2_to_5("## §2 a\nb\n## §6 c\n").is_some()); @@ -208,7 +250,8 @@ mod tests { fn render_lock_writes_every_component_and_round_trips() { let c = Components::of("## §2 a\nb\n## §6 c\n", "stats", "plan", "prompt").expect("span"); let lock = c.render_lock(); - assert!(lock.starts_with("# REX-001 pre-registration lock (rex-prereg-v1).")); + assert!(lock + .starts_with("# PRM-001 pre-registration lock (rex-prereg-v2) over spec PRM-001 v3.")); assert!(lock.contains(&format!("spec_s2_s5={}\n", c.spec_s2_s5))); assert!(lock.contains(&format!("stats_rs={}\n", c.stats_rs))); assert!(lock.contains(&format!("analysis_plan={}\n", c.analysis_plan))); diff --git a/crates/aprender-review-experiment/src/ratchet.rs b/crates/aprender-review-experiment/src/ratchet.rs new file mode 100644 index 0000000000..3d2847c301 --- /dev/null +++ b/crates/aprender-review-experiment/src/ratchet.rs @@ -0,0 +1,382 @@ +//! REX-10 perf ratchet (§5.1): review p95 on the primary cell may not rise. +//! +//! Every tag reruns the perf arm on the primary cell over one fixed item set +//! (full dev split + 30 fixed test items, timing only). `record` turns that +//! tag's warm, non-rerun receipts into one `review-lane-perf-ratchet-v1` entry +//! holding each item's wall time. `check` reads the entries of one cell in +//! record order: +//! - fewer than [`ARM_AFTER`] tags: **arming**, nothing to compare yet; +//! - each later tag is paired by item with the best (lowest-p95) earlier tag, +//! and a paired bootstrap gives a CI for `p95(new) / p95(best)`, each p95 +//! the Harrell–Davis estimate. A CI whose lower bound is above 1 is a +//! regression and the andon is RED. +//! +//! The paired CI, not "the two p95 CIs overlap", is the rule: the same items +//! are timed every tag, so pairing removes item-to-item spread. The unpaired +//! overlap rule could not see a +10 % p95 regression at the pilot's n +//! (FALSIFY-RXR-002 measures this). Nor could a paired nearest-rank p95: it is +//! one order statistic, so one item's run-to-run noise decides it; the +//! Harrell–Davis estimate of the same quantile weights the whole tail. +//! The only level used is the prereg's [`ALPHA`]; no noise threshold is +//! invented (R-7). + +use std::collections::BTreeMap; + +use serde::{Deserialize, Serialize}; + +use crate::stats::{percentile, percentile_sorted, SplitMix64, ALPHA, BOOTSTRAP_RESAMPLES}; + +pub const SCHEME: &str = "review-lane-perf-ratchet-v1"; +/// The ratchet holds after this many recorded tags (§5.1). +pub const ARM_AFTER: usize = 3; +const P95: f64 = 0.95; +/// The harness seed (`harness::DECODING.seed`), so every CI reproduces. +const SEED: u64 = 4354; + +/// One recorded tag on one cell. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Entry { + pub schema: String, + pub tag: String, + pub cell: String, + pub apr_sha256: String, + /// Warm wall time per item, ms (the median when an item ran twice). + pub items: BTreeMap, + pub p95_ms: f64, + /// Unpaired percentile-bootstrap CI of this tag's p95, for display. + pub p95_ci_ms: [f64; 2], + /// The same p95 from llama.cpp on the same cell and items, if it ran. + pub llama_cpp_p95_ms: Option, + pub receipts_sha256: String, +} + +/// A warm, non-rerun timing already filtered to one cell and one tag. +#[derive(Debug, Clone)] +pub struct Sample<'a> { + pub item_id: &'a str, + pub apr_sha256: &'a str, + pub wall_ms: f64, +} + +fn median(v: &mut [f64]) -> f64 { + v.sort_by(f64::total_cmp); + let n = v.len(); + if n % 2 == 1 { + v[n / 2] + } else { + (v[n / 2 - 1] + v[n / 2]) / 2.0 + } +} + +/// Per-item wall time: the median of each item's finite, positive timings. +fn per_item(samples: &[Sample<'_>]) -> Result, String> { + let mut by: BTreeMap> = BTreeMap::new(); + for s in samples { + if !(s.wall_ms.is_finite() && s.wall_ms > 0.0) { + return Err(format!( + "{}: wall_ms {} is not a timing", + s.item_id, s.wall_ms + )); + } + by.entry(s.item_id.to_string()).or_default().push(s.wall_ms); + } + Ok(by + .into_iter() + .map(|(k, mut v)| (k, median(&mut v))) + .collect()) +} + +fn ci_of(mut stat: Vec) -> [f64; 2] { + stat.sort_by(f64::total_cmp); + [ + percentile_sorted(&stat, ALPHA / 2.0), + percentile_sorted(&stat, 1.0 - ALPHA / 2.0), + ] +} + +/// Unpaired percentile-bootstrap CI of the p95 of `x`. +fn p95_ci(x: &[f64]) -> [f64; 2] { + let mut rng = SplitMix64::new(SEED); + let mut buf = vec![0.0; x.len()]; + let stat = (0..BOOTSTRAP_RESAMPLES) + .map(|_| { + for b in &mut buf { + *b = x[rng.below(x.len())]; + } + percentile(&buf, P95) + }) + .collect(); + ci_of(stat) +} + +/// Build one tag's entry. A tag with no timings did not run and is not a +/// recorded tag (R-6); mixed apr binaries under one tag are refused. +pub fn record( + tag: &str, + cell: &str, + samples: &[Sample<'_>], + llama_cpp: &[Sample<'_>], + receipts_sha256: &str, +) -> Result { + let shas: std::collections::BTreeSet<&str> = samples.iter().map(|s| s.apr_sha256).collect(); + let apr_sha256 = match shas.len() { + 0 => { + return Err(format!( + "{tag} on {cell}: no warm timings; the tag did not run" + )) + } + 1 => shas.into_iter().next().unwrap_or_default().to_string(), + n => { + return Err(format!( + "{tag} on {cell}: {n} different apr binaries under one tag" + )) + } + }; + let items = per_item(samples)?; + let v: Vec = items.values().copied().collect(); + let llama_cpp_p95_ms = if llama_cpp.is_empty() { + None + } else { + let l: Vec = per_item(llama_cpp)?.into_values().collect(); + Some(percentile(&l, P95)) + }; + Ok(Entry { + schema: SCHEME.into(), + tag: tag.into(), + cell: cell.into(), + apr_sha256, + p95_ms: percentile(&v, P95), + p95_ci_ms: p95_ci(&v), + items, + llama_cpp_p95_ms, + receipts_sha256: receipts_sha256.into(), + }) +} + +/// ln Γ(x), Lanczos (g = 7, n = 9); x > 0. +fn ln_gamma(x: f64) -> f64 { + const C: [f64; 9] = [ + 0.999_999_999_999_809_9, + 676.520_368_121_885_1, + -1_259.139_216_722_402_8, + 771.323_428_777_653_1, + -176.615_029_162_140_6, + 12.507_343_278_686_905, + -0.138_571_095_265_720_12, + 9.984_369_578_019_572e-6, + 1.505_632_735_149_311_6e-7, + ]; + let x = x - 1.0; + let t = x + 7.5; + let s = C[1..] + .iter() + .enumerate() + .fold(C[0], |s, (i, c)| s + c / (x + i as f64 + 1.0)); + 0.5 * (2.0 * std::f64::consts::PI).ln() + (x + 0.5) * t.ln() - t + s.ln() +} + +fn not_tiny(v: f64) -> f64 { + if v.abs() < 1e-300 { + 1e-300 + } else { + v + } +} + +/// Continued fraction of the incomplete beta (modified Lentz). +fn beta_cf(a: f64, b: f64, x: f64) -> f64 { + let (qab, qap, qam) = (a + b, a + 1.0, a - 1.0); + let mut c = 1.0; + let mut d = not_tiny(1.0 - qab * x / qap).recip(); + let mut h = d; + for m in 1..=300 { + let m = f64::from(m); + let m2 = 2.0 * m; + for aa in [ + m * (b - m) * x / ((qam + m2) * (a + m2)), + -(a + m) * (qab + m) * x / ((a + m2) * (qap + m2)), + ] { + d = not_tiny(1.0 + aa * d).recip(); + c = not_tiny(1.0 + aa / c); + h *= d * c; + } + if (d * c - 1.0).abs() < 1e-15 { + break; + } + } + h +} + +/// Regularized incomplete beta I_x(a, b). +fn inc_beta(a: f64, b: f64, x: f64) -> f64 { + if x <= 0.0 { + return 0.0; + } + if x >= 1.0 { + return 1.0; + } + let front = + (ln_gamma(a + b) - ln_gamma(a) - ln_gamma(b) + a * x.ln() + b * (1.0 - x).ln()).exp(); + if x < (a + 1.0) / (a + b + 2.0) { + front * beta_cf(a, b, x) / a + } else { + 1.0 - front * beta_cf(b, a, 1.0 - x) / b + } +} + +/// Harrell–Davis weights for quantile `q` of `n` sorted values: a weighted +/// sum of every order statistic, so one noisy tail item cannot swing it the +/// way it swings the nearest-rank p95. +#[must_use] +pub fn harrell_davis_weights(n: usize, q: f64) -> Vec { + let (a, b) = ((n as f64 + 1.0) * q, (n as f64 + 1.0) * (1.0 - q)); + (1..=n) + .map(|i| inc_beta(a, b, i as f64 / n as f64) - inc_beta(a, b, (i - 1) as f64 / n as f64)) + .collect() +} + +fn hd(buf: &mut [f64], w: &[f64]) -> f64 { + buf.sort_by(f64::total_cmp); + buf.iter().zip(w).map(|(x, w)| x * w).sum() +} + +/// Harrell–Davis p95 ratio `new / reference` over paired items, and its +/// paired-bootstrap CI (items resampled together, so both sides see the +/// same items). +#[must_use] +pub fn paired_p95_ratio_ci(pairs: &[(f64, f64)]) -> Option<(f64, [f64; 2])> { + if pairs.is_empty() { + return None; + } + let w = harrell_davis_weights(pairs.len(), P95); + let (mut a, mut b): (Vec, Vec) = pairs.iter().copied().unzip(); + let point = hd(&mut a, &w) / hd(&mut b, &w); + let mut rng = SplitMix64::new(SEED); + let stat = (0..BOOTSTRAP_RESAMPLES) + .map(|_| { + for i in 0..pairs.len() { + let (n, r) = pairs[rng.below(pairs.len())]; + a[i] = n; + b[i] = r; + } + hd(&mut a, &w) / hd(&mut b, &w) + }) + .collect(); + Some((point, ci_of(stat))) +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)] +#[serde(rename_all = "lowercase")] +pub enum Andon { + Arming, + Green, + Red, +} + +/// One armed comparison: `tag` against the best earlier `reference`. +#[derive(Debug, Clone, PartialEq, Serialize)] +pub struct Comparison { + pub tag: String, + pub reference: String, + pub paired_items: usize, + pub p95_ratio: f64, + pub p95_ratio_ci: [f64; 2], + pub regression: bool, +} + +#[derive(Debug, Clone, PartialEq, Serialize)] +pub struct Verdict { + pub andon: Andon, + pub cell: Option, + pub tags: usize, + pub comparisons: Vec, + pub violations: Vec, + pub notes: Vec, +} + +/// Read the ratchet file of one cell and decide the andon. Violations (a +/// foreign schema or cell, a duplicate tag, no common items) are RED too: +/// the ratchet fails closed. +#[must_use] +pub fn check(entries: &[Entry]) -> Verdict { + let mut v = Verdict { + andon: Andon::Arming, + cell: entries.first().map(|e| e.cell.clone()), + tags: entries.len(), + comparisons: Vec::new(), + violations: Vec::new(), + notes: Vec::new(), + }; + let mut seen = std::collections::BTreeSet::new(); + for e in entries { + if e.schema != SCHEME { + v.violations + .push(format!("{}: schema {} is not {SCHEME}", e.tag, e.schema)); + } + if Some(&e.cell) != v.cell.as_ref() { + v.violations + .push(format!("{}: cell {} is not the file's cell", e.tag, e.cell)); + } + if !seen.insert(e.tag.as_str()) { + v.violations.push(format!("{}: tag recorded twice", e.tag)); + } + } + for (i, e) in entries.iter().enumerate().skip(ARM_AFTER) { + let best = entries[..i] + .iter() + .min_by(|a, b| a.p95_ms.total_cmp(&b.p95_ms)) + .unwrap_or(&entries[0]); + let pairs: Vec<(f64, f64)> = e + .items + .iter() + .filter_map(|(k, n)| best.items.get(k).map(|r| (*n, *r))) + .collect(); + if pairs.len() != e.items.len() || pairs.len() != best.items.len() { + v.notes.push(format!( + "{} vs {}: {} common items of {} / {}", + e.tag, + best.tag, + pairs.len(), + e.items.len(), + best.items.len() + )); + } + let Some((ratio, ci)) = paired_p95_ratio_ci(&pairs) else { + v.violations + .push(format!("{} vs {}: no common items", e.tag, best.tag)); + continue; + }; + v.comparisons.push(Comparison { + tag: e.tag.clone(), + reference: best.tag.clone(), + paired_items: pairs.len(), + p95_ratio: ratio, + p95_ratio_ci: ci, + regression: ci[0] > 1.0, + }); + } + for w in entries.windows(2) { + let gap = |e: &Entry| e.llama_cpp_p95_ms.map(|l| e.p95_ms / l); + if let (Some(g0), Some(g1)) = (gap(&w[0]), gap(&w[1])) { + if g1 > g0 { + v.notes.push(format!( + "{}: gap to llama.cpp grew {g0:.3}× → {g1:.3}× (§9.1: file it)", + w[1].tag + )); + } + } + } + v.andon = if !v.violations.is_empty() || v.comparisons.iter().any(|c| c.regression) { + Andon::Red + } else if entries.len() > ARM_AFTER { + Andon::Green + } else { + Andon::Arming + }; + v +} + +#[cfg(test)] +#[path = "ratchet_tests.rs"] +mod tests; diff --git a/crates/aprender-review-experiment/src/ratchet_tests.rs b/crates/aprender-review-experiment/src/ratchet_tests.rs new file mode 100644 index 0000000000..4f9b661819 --- /dev/null +++ b/crates/aprender-review-experiment/src/ratchet_tests.rs @@ -0,0 +1,200 @@ +use super::*; + +/// The pilot's item count: the dev split (~100) plus 30 fixed test items. +const N: usize = 130; + +/// Approximate N(0,1) from twelve uniforms, on the frozen generator. +fn gauss(rng: &mut SplitMix64) -> f64 { + (0..12) + .map(|_| (rng.next_u64() >> 11) as f64 / (1u64 << 53) as f64) + .sum::() + - 6.0 +} + +/// Item difficulty: log-normal wall time around 8 s with σ = 0.6, which is +/// wide (diff sizes vary by an order of magnitude); fixed across tags. +fn base() -> Vec { + let mut rng = SplitMix64::new(1); + (0..N) + .map(|_| (8000f64.ln() + 0.6 * gauss(&mut rng)).exp()) + .collect() +} + +/// One tag's run: every item × `scale`, with 3 % run-to-run noise. +fn tag(name: &str, seed: u64, scale: f64) -> Entry { + let mut rng = SplitMix64::new(seed); + let ids: Vec = (0..N).map(|i| format!("item-{i:03}")).collect(); + let ms: Vec = base() + .iter() + .map(|b| b * scale * (1.0 + 0.03 * gauss(&mut rng))) + .collect(); + let s: Vec> = ids + .iter() + .zip(&ms) + .map(|(id, &wall_ms)| Sample { + item_id: id, + apr_sha256: "aa", + wall_ms, + }) + .collect(); + record(name, "C4", &s, &[], "rr").expect("records") +} + +fn baseline() -> Vec { + vec![ + tag("v0.70.0", 10, 1.0), + tag("v0.70.1", 11, 1.0), + tag("v0.70.2", 12, 1.0), + ] +} + +#[test] +fn falsify_rxr_001_a_planted_ten_percent_p95_regression_is_red() { + let mut e = baseline(); + assert_eq!( + check(&e).andon, + Andon::Arming, + "3 tags only arm the ratchet" + ); + e.push(tag("v0.70.3", 13, 1.0)); + let same = check(&e); + assert_eq!(same.andon, Andon::Green, "{same:?}"); + e[3] = tag("v0.70.3", 13, 1.10); + let v = check(&e); + assert_eq!(v.andon, Andon::Red, "{v:?}"); + let c = &v.comparisons[0]; + assert!(c.regression && c.p95_ratio_ci[0] > 1.0 && c.paired_items == N); +} + +#[test] +fn falsify_rxr_001_a_gain_is_kept_the_reference_is_the_best_tag() { + let mut e = baseline(); + e[2] = tag("v0.70.2", 12, 0.80); + e.push(tag("v0.70.3", 13, 1.0)); + let v = check(&e); + assert_eq!(v.andon, Andon::Red, "giving back a 20 % gain is a rise"); + assert_eq!(v.comparisons[0].reference, "v0.70.2"); +} + +/// The finding behind the paired rule: at the pilot's n, the two unpaired +/// p95 CIs of a +10 % tag still overlap, so the §4 overlap rule calls it a +/// tie. The ratchet could not meet its own acceptance on that rule. +#[test] +fn falsify_rxr_002_the_unpaired_overlap_rule_misses_ten_percent() { + let a = tag("v0.70.2", 12, 1.0); + let b = tag("v0.70.3", 13, 1.10); + assert!(b.p95_ms > a.p95_ms); + assert!( + b.p95_ci_ms[0] <= a.p95_ci_ms[1], + "unpaired CIs {:?} vs {:?} unexpectedly separate", + a.p95_ci_ms, + b.p95_ci_ms + ); +} + +#[test] +fn falsify_rxr_003_the_ratchet_fails_closed() { + let mut dup = baseline(); + dup.push(tag("v0.70.1", 13, 1.0)); + assert_eq!(check(&dup).andon, Andon::Red, "duplicate tag"); + + let mut foreign = baseline(); + foreign[1].cell = "C1".into(); + assert_eq!(check(&foreign).andon, Andon::Red, "foreign cell"); + + let mut disjoint = baseline(); + let mut t = tag("v0.70.3", 13, 1.0); + t.items = t + .items + .into_iter() + .map(|(k, v)| (format!("x{k}"), v)) + .collect(); + disjoint.push(t); + let v = check(&disjoint); + assert_eq!(v.andon, Andon::Red, "no common items"); + assert!(v.violations.iter().any(|m| m.contains("no common items"))); + + let ok = Sample { + item_id: "a", + apr_sha256: "aa", + wall_ms: 1.0, + }; + assert!( + record("t", "C4", &[], &[], "r").is_err(), + "a tag that did not run" + ); + let other = Sample { + apr_sha256: "bb", + ..ok.clone() + }; + assert!( + record("t", "C4", &[ok.clone(), other], &[], "r").is_err(), + "two binaries" + ); + let nan = Sample { + wall_ms: f64::NAN, + ..ok + }; + assert!( + record("t", "C4", &[nan], &[], "r").is_err(), + "a non-finite timing" + ); +} + +#[test] +fn falsify_rxr_003_a_growing_llama_cpp_gap_is_noted() { + let mut e = baseline(); + e[0].llama_cpp_p95_ms = Some(e[0].p95_ms); + e[1].llama_cpp_p95_ms = Some(e[1].p95_ms / 1.2); + let v = check(&e); + assert!( + v.notes.iter().any(|n| n.contains("gap to llama.cpp grew")), + "{v:?}" + ); +} + +#[test] +fn an_item_timed_twice_counts_its_median() { + let s = |ms| Sample { + item_id: "a", + apr_sha256: "aa", + wall_ms: ms, + }; + let e = record("t", "C4", &[s(1.0), s(9.0), s(2.0)], &[], "r").expect("records"); + assert_eq!(e.items["a"], 2.0); +} + +#[test] +fn harrell_davis_weights_are_a_distribution_centred_on_the_tail() { + let w = harrell_davis_weights(N, 0.95); + assert!((w.iter().sum::() - 1.0).abs() < 1e-9); + assert!(w.iter().all(|x| *x >= 0.0)); + let x: Vec = (1..=N).map(|i| i as f64).collect(); + let q: f64 = x.iter().zip(&w).map(|(x, w)| x * w).sum(); + assert!( + (q - 0.95 * (N as f64 + 1.0)).abs() < 1.0, + "HD p95 of 1..N = {q}" + ); +} + +/// Rule 6: one seed is an anecdote. Over 10 independent runs of the planted +/// tag the ratchet catches +10 % every time and seldom alarms on an unchanged +/// tag (40/40 caught and 0/40 alarms when first measured over 40 runs, at the +/// modelled 3 % run-to-run noise per item). +#[test] +fn falsify_rxr_001_power_and_false_alarms_over_ten_runs() { + let e = baseline(); + let red = |scale: f64| { + (100..110) + .filter(|&s| { + let mut t = e.clone(); + t.push(tag("v0.70.3", s, scale)); + check(&t).andon == Andon::Red + }) + .count() + }; + let (caught, false_alarms) = (red(1.10), red(1.0)); + eprintln!("RXR power: +10% caught {caught}/10, unchanged red {false_alarms}/10"); + assert_eq!(caught, 10); + assert!(false_alarms <= 1, "{false_alarms}/10 false alarms"); +} diff --git a/crates/aprender-review-experiment/src/receipt.rs b/crates/aprender-review-experiment/src/receipt.rs index d43c131fce..925493f8c7 100644 --- a/crates/aprender-review-experiment/src/receipt.rs +++ b/crates/aprender-review-experiment/src/receipt.rs @@ -197,31 +197,8 @@ pub fn admissible(line: &str, expect: Expect<'_>) -> Result } } -fn problems(r: &Receipt, expect: Expect<'_>) -> Vec { - let mut bad = Vec::new(); - if r.schema != SCHEME { - bad.push(format!("schema {:?} != {SCHEME}", r.schema)); - } - named_problems(r, &mut bad); - sha_problems(r, &mut bad); - if r.prereg_sha != expect.prereg_sha { - bad.push("prereg_sha is not the locked one (exploratory data)".into()); - } - if r.corpus_version != expect.corpus_version { - bad.push(format!( - "corpus_version {:?} is not {:?}", - r.corpus_version, expect.corpus_version - )); - } - if r.decoding.temperature != 0.0 { - bad.push("decoding is not greedy (temperature != 0)".into()); - } - execution_problems(r, &mut bad); - bad -} - -/// Every identifying field is present and not the literal `unknown`. -fn named_problems(r: &Receipt, bad: &mut Vec) { +/// The identity fields that must be non-empty and not `unknown`. +fn check_named_fields(r: &Receipt, bad: &mut Vec) { let named = [ ("item_id", &r.item_id), ("cell", &r.cell), @@ -238,9 +215,10 @@ fn named_problems(r: &Receipt, bad: &mut Vec) { } } -/// Every digest is a sha256, except that a hosted arm's binary and weights -/// digests must be the literal `hosted`. -fn sha_problems(r: &Receipt, bad: &mut Vec) { +/// The sha256 fields: `apr_sha256`/`weights_sha256` must literally be +/// `hosted` for a hosted arm, and a real sha256 otherwise; the rest are +/// always shas. +fn check_shas(r: &Receipt, bad: &mut Vec) { let mut shas = vec![ ("item_sha256", &r.item_sha256), ("prompt_sha256", &r.prompt_sha256), @@ -267,8 +245,25 @@ fn sha_problems(r: &Receipt, bad: &mut Vec) { } } -/// An executed row carries its measurements, and any output is well-formed. -fn execution_problems(r: &Receipt, bad: &mut Vec) { +/// The locked prereg/corpus identity and the fixed decoding settings. +fn check_locks(r: &Receipt, expect: Expect<'_>, bad: &mut Vec) { + if r.prereg_sha != expect.prereg_sha { + bad.push("prereg_sha is not the locked one (exploratory data)".into()); + } + if r.corpus_version != expect.corpus_version { + bad.push(format!( + "corpus_version {:?} is not {:?}", + r.corpus_version, expect.corpus_version + )); + } + if r.decoding.temperature != 0.0 { + bad.push("decoding is not greedy (temperature != 0)".into()); + } +} + +/// What an executed row (one that produced a verdict, parsed or not) must +/// also carry, and the shape of its `output` block when present. +fn check_executed_fields(r: &Receipt, bad: &mut Vec) { if r.verdict.executed() { for (k, missing) in [ ("tokens", r.tokens.is_none()), @@ -288,6 +283,18 @@ fn execution_problems(r: &Receipt, bad: &mut Vec) { } } +fn problems(r: &Receipt, expect: Expect<'_>) -> Vec { + let mut bad = Vec::new(); + if r.schema != SCHEME { + bad.push(format!("schema {:?} != {SCHEME}", r.schema)); + } + check_named_fields(r, &mut bad); + check_shas(r, &mut bad); + check_locks(r, expect, &mut bad); + check_executed_fields(r, &mut bad); + bad +} + /// The model's answer with any `…` reasoning removed: the /// verdict is read after the last `` when one is present. #[must_use] diff --git a/crates/aprender-review-experiment/src/receipt_tests.rs b/crates/aprender-review-experiment/src/receipt_tests.rs index 1fd6b595ce..ec082686e8 100644 --- a/crates/aprender-review-experiment/src/receipt_tests.rs +++ b/crates/aprender-review-experiment/src/receipt_tests.rs @@ -216,7 +216,7 @@ fn output_path_and_sha_are_each_checked() { sha256: sha, }); let mut bad = Vec::new(); - execution_problems(&r, &mut bad); + check_executed_fields(&r, &mut bad); bad }; assert!(problems("raw/i.txt", h('6')).is_empty()); diff --git a/crates/aprender-review-experiment/src/replay.rs b/crates/aprender-review-experiment/src/replay.rs new file mode 100644 index 0000000000..d34bca73aa --- /dev/null +++ b/crates/aprender-review-experiment/src/replay.rs @@ -0,0 +1,644 @@ +//! PRM-S1: the `review-replay-v1` speed benchmark (PRM-001 v3 §2.4 and §9; +//! contract `review-replay-v1`). +//! +//! A frozen set of real quorum diffs (inputs only, never a lane's output) is +//! replayed through `apr serve` and through llama.cpp `d1d3c3396` on the same +//! GGUF and the same cell. This module builds the set, defines the receipt row +//! and folds rows into the §9 speed block. It does no I/O. +//! +//! - **Set.** Four strata by composed input tokens: `2k` (0, 2048], `8k` +//! (2048, 8192], `16k` (8192, 16384], `32k` (16384, 32768]. [`PER_STRATUM`] +//! items each. One item per `repo#PR`, one per diff sha. Every candidate that +//! hits the sealed test set (diff sha, hunk fingerprint or near-dup cluster) +//! is dropped. Selection inside a stratum is `sha256(seed ‖ diff_sha)` order, +//! so it is frozen by `(version, seed, candidates)`. A short stratum is an +//! error: the set is never padded. +//! - **Rows** (`review-replay-receipt-v1`), one per `(engine, item)`. +//! - **Summary.** Refuses a mixed set, a GGUF or cell mismatch, unequal item +//! coverage between engines, duplicate rows, any not-run row, or a missing +//! voter latency sample. Every ratio is `apr / llama_cpp`: TTFT wants ≤ 2.0, +//! prefill and decode want ≥ 1.0. A server that reports no timing yields +//! `null`, never a number. + +use std::collections::{BTreeMap, BTreeSet}; + +use serde::{Deserialize, Serialize}; + +use crate::contamination::Index; +use crate::corpus::sha256_hex; +use crate::receipt::Verdict; +use crate::stats::{percentile, percentile_sorted, SplitMix64}; + +pub const SCHEMA: &str = "review-replay-v1"; +pub const ROW_SCHEMA: &str = "review-replay-receipt-v1"; +pub const PER_STRATUM: usize = 50; +/// Bootstrap resamples for the p95 interval. +pub const BOOTSTRAP: usize = 2000; + +/// The first set version that pins the prompt (PRM-S1 v2): both engines are sent +/// `chat_template_kwargs.enable_thinking = false`, and a run is summarized only when +/// every item's prompt ids hash the same on both engines. v1 pinned neither; +/// llama.cpp `d1d3c3396` then decoded Qwen3.5 in thinking mode and apr did not, so v1 +/// runs are not comparable. +pub const V2: &str = "review-replay-v2"; + +/// Whether set `version` pins the prompt (see [`V2`]). +#[must_use] +pub fn pins_prompt(version: &str) -> bool { + version == V2 +} + +/// The chat request replayed for one item under set `version`: the REX-03 body, +/// plus `enable_thinking: false` when the version pins the prompt. +#[must_use] +pub fn replay_request(version: &str, model: &str, prompt: &str, diff: &str) -> serde_json::Value { + let mut body = crate::harness::request_body(model, prompt, diff); + if pins_prompt(version) { + body["chat_template_kwargs"] = serde_json::json!({ "enable_thinking": false }); + } + body +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)] +pub enum Stratum { + #[serde(rename = "2k")] + K2, + #[serde(rename = "8k")] + K8, + #[serde(rename = "16k")] + K16, + #[serde(rename = "32k")] + K32, +} + +impl Stratum { + pub const ALL: [Self; 4] = [Self::K2, Self::K8, Self::K16, Self::K32]; + + /// Inclusive upper bound in input tokens. + #[must_use] + pub fn upper(self) -> u64 { + match self { + Self::K2 => 2048, + Self::K8 => 8192, + Self::K16 => 16384, + Self::K32 => 32768, + } + } + + #[must_use] + pub fn as_str(self) -> &'static str { + match self { + Self::K2 => "2k", + Self::K8 => "8k", + Self::K16 => "16k", + Self::K32 => "32k", + } + } + + /// The stratum of an input of `tokens` tokens; `None` for 0 or > 32768. + #[must_use] + pub fn of(tokens: u64) -> Option { + if tokens == 0 { + return None; + } + Self::ALL.into_iter().find(|s| tokens <= s.upper()) + } +} + +/// A candidate diff, before sealing and stratification. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Candidate { + /// `owner/repo#N`. + pub group: String, + pub diff_sha256: String, + /// Composed prompt tokens, counted by the replay GGUF's tokenizer. + pub input_tokens: u64, + pub diff: String, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct Item { + pub group: String, + pub diff_sha256: String, + pub input_tokens: u64, + pub stratum: Stratum, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Set { + pub version: String, + pub seed: u64, + pub per_stratum: usize, + pub items: Vec, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum BuildError { + /// Nothing is sealed: the exclusion would prove nothing. + EmptySealedIndex, + /// Strata that could not be filled. + Short(Vec), +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct Short { + pub stratum: Stratum, + pub have: usize, + pub need: usize, +} + +fn rank_key(seed: u64, diff_sha: &str) -> String { + sha256_hex(format!("{seed}\u{0}{diff_sha}").as_bytes()) +} + +/// Build the frozen set (see module docs). +/// +/// # Errors +/// [`BuildError::EmptySealedIndex`] when `sealed` is empty, and +/// [`BuildError::Short`] listing every stratum with fewer than `per_stratum` +/// eligible items. +pub fn build( + version: &str, + seed: u64, + per_stratum: usize, + cands: &[Candidate], + sealed: &Index, +) -> Result { + if sealed.is_empty() { + return Err(BuildError::EmptySealedIndex); + } + // Deterministic order before the one-per-group / one-per-sha dedup, so + // input order never decides which duplicate survives. + let mut eligible: Vec<(String, Item)> = cands + .iter() + .filter(|c| { + sealed + .scan(&format!("{}\n{}", c.diff_sha256, c.diff), &c.group) + .is_empty() + }) + .filter_map(|c| { + Stratum::of(c.input_tokens).map(|stratum| { + ( + rank_key(seed, &c.diff_sha256), + Item { + group: c.group.clone(), + diff_sha256: c.diff_sha256.clone(), + input_tokens: c.input_tokens, + stratum, + }, + ) + }) + }) + .collect(); + eligible.sort_by(|a, b| a.0.cmp(&b.0).then_with(|| a.1.group.cmp(&b.1.group))); + let mut groups = BTreeSet::new(); + let mut shas = BTreeSet::new(); + let mut by: BTreeMap> = BTreeMap::new(); + for (_, it) in eligible { + if groups.contains(&it.group) || shas.contains(&it.diff_sha256) { + continue; + } + let bucket = by.entry(it.stratum).or_default(); + if bucket.len() < per_stratum { + groups.insert(it.group.clone()); + shas.insert(it.diff_sha256.clone()); + bucket.push(it); + } + } + let short: Vec = Stratum::ALL + .into_iter() + .filter_map(|s| { + let have = by.get(&s).map_or(0, Vec::len); + (have < per_stratum).then_some(Short { + stratum: s, + have, + need: per_stratum, + }) + }) + .collect(); + if !short.is_empty() { + return Err(BuildError::Short(short)); + } + Ok(Set { + version: version.to_string(), + seed, + per_stratum, + items: Stratum::ALL + .into_iter() + .flat_map(|s| by.remove(&s).unwrap_or_default()) + .collect(), + }) +} + +impl Set { + /// JSONL: a header line, then one item per line. + #[must_use] + pub fn render(&self) -> String { + let head: serde_json::Map = [ + ("schema", serde_json::Value::from(SCHEMA)), + ("version", self.version.as_str().into()), + ("seed", self.seed.into()), + ("per_stratum", self.per_stratum.into()), + ] + .into_iter() + .map(|(k, v)| (k.to_string(), v)) + .collect(); + let mut out = serde_json::Value::Object(head).to_string(); + out.push('\n'); + for it in &self.items { + out.push_str(&serde_json::to_string(it).unwrap_or_default()); + out.push('\n'); + } + out + } + + /// sha256 of [`Set::render`]; every row carries it. + #[must_use] + pub fn sha(&self) -> String { + sha256_hex(self.render().as_bytes()) + } + + /// Parse a rendered set. + /// + /// # Errors + /// A missing or foreign header, or a malformed item line. + pub fn parse(text: &str) -> Result { + let mut lines = text.lines(); + let head: serde_json::Value = lines + .next() + .and_then(|l| serde_json::from_str(l).ok()) + .ok_or("no header")?; + if head["schema"] != SCHEMA { + return Err(format!("schema is {}, want {SCHEMA}", head["schema"])); + } + let version = head["version"].as_str().ok_or("no version")?.to_string(); + let seed = head["seed"].as_u64().ok_or("no seed")?; + let per_stratum = head["per_stratum"] + .as_u64() + .and_then(|n| usize::try_from(n).ok()) + .ok_or("no per_stratum")?; + let items = lines + .enumerate() + .map(|(i, l)| serde_json::from_str(l).map_err(|e| format!("item {}: {e}", i + 1))) + .collect::>()?; + Ok(Self { + version, + seed, + per_stratum, + items, + }) + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum Engine { + Apr, + LlamaCpp, +} + +/// One `review-replay-receipt-v1` row. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct Row { + pub schema: String, + pub replay_version: String, + pub set_sha: String, + pub diff_sha256: String, + pub stratum: Stratum, + pub engine: Engine, + pub engine_version: String, + /// The apr tag under test (the same on the llama.cpp rows of that run). + pub apr_tag: String, + pub cell: String, + pub gguf_sha256: String, + pub wall_ms: f64, + /// Server-reported prompt (prefill) time: the TTFT of a non-streamed call. + pub prompt_ms: Option, + pub prompt_tps: Option, + pub decode_tps: Option, + pub input_tokens: Option, + pub output_tokens: Option, + pub peak_rss_mb: Option, + pub verdict: Verdict, + /// sha256 of the prompt ids this engine prefilled for the item (the ids as + /// decimal, comma-separated). Required from [`V2`] on. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub prompt_ids_sha256: Option, +} + +/// Engine A over engine B (`apr / llama_cpp`). `ratio` is null when either +/// side is null or B is zero. +#[derive(Debug, Clone, Copy, PartialEq, Serialize, Deserialize)] +pub struct Ratio { + pub apr: Option, + pub llama_cpp: Option, + pub ratio: Option, +} + +impl Ratio { + fn of(apr: Option, llama_cpp: Option) -> Self { + let ratio = match (apr, llama_cpp) { + (Some(a), Some(b)) if b > 0.0 => Some(a / b), + _ => None, + }; + Self { + apr, + llama_cpp, + ratio, + } + } +} + +/// The §9 speed block. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct Speed { + pub replay_version: String, + pub set_sha: String, + pub apr_tag: String, + pub cell: String, + pub gguf_sha256: String, + pub n_items: usize, + pub p50_s: f64, + pub p95_s: f64, + pub p95_ci: [f64; 2], + pub queue_budget_p95_s: f64, + pub queue_budget_lane: String, + pub within_budget: bool, + pub ttft_ms: Ratio, + pub prefill_tps: BTreeMap, + pub decode_tps: Ratio, + pub peak_rss_mb: Ratio, + pub parse_rate: f64, + pub verdict_identity_vs_prev_tag: Option, + /// sha256 over the per-item prompt-ids shas in item order: the ids both + /// engines prefilled. `None` for a set version that pins no prompt. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub prompt_ids_sha: Option, +} + +#[derive(Debug, Clone, PartialEq)] +pub enum SummaryError { + NoRows, + MixedSet, + Mismatch(&'static str), + DuplicateRow { + engine: Engine, + diff_sha256: String, + }, + CoverageDiffers, + NotRun { + engine: Engine, + diff_sha256: String, + }, + NoVoterLatency, + PrevSetDiffers, + PrevCoverageDiffers, + /// A prompt-pinning set whose row carries no prompt-ids sha. + NoPromptIds { + engine: Engine, + diff_sha256: String, + }, + /// The engines prefilled different ids for this item. + PromptIdsDiffer { + diff_sha256: String, + }, +} + +/// Per-engine rows keyed by diff sha. +type ByItem<'a> = BTreeMap<&'a str, &'a Row>; + +fn index(rows: &[Row]) -> Result<[ByItem<'_>; 2], SummaryError> { + let mut out = [BTreeMap::new(), BTreeMap::new()]; + for r in rows { + if let Verdict::NotRun(_) = r.verdict { + return Err(SummaryError::NotRun { + engine: r.engine, + diff_sha256: r.diff_sha256.clone(), + }); + } + let slot = usize::from(r.engine == Engine::LlamaCpp); + if out[slot].insert(r.diff_sha256.as_str(), r).is_some() { + return Err(SummaryError::DuplicateRow { + engine: r.engine, + diff_sha256: r.diff_sha256.clone(), + }); + } + } + Ok(out) +} + +/// For a prompt-pinning set: every row carries its prompt-ids sha and the two +/// engines agree on every item. Returns the sha over the per-item shas. +fn prompt_ids_sha( + version: &str, + apr: &ByItem<'_>, + llama: &ByItem<'_>, +) -> Result, SummaryError> { + if !pins_prompt(version) { + return Ok(None); + } + let sha_of = |r: &Row| { + r.prompt_ids_sha256 + .clone() + .ok_or_else(|| SummaryError::NoPromptIds { + engine: r.engine, + diff_sha256: r.diff_sha256.clone(), + }) + }; + let mut all = String::new(); + for (a, l) in apr.values().zip(llama.values()) { + let (sa, sl) = (sha_of(a)?, sha_of(l)?); + if sa != sl { + return Err(SummaryError::PromptIdsDiffer { + diff_sha256: a.diff_sha256.clone(), + }); + } + all.push_str(&sa); + all.push('\n'); + } + Ok(Some(sha256_hex(all.as_bytes()))) +} + +/// The sha a row records for prompt ids: sha256 of the ids as decimal, +/// comma-separated. +#[must_use] +pub fn ids_sha256(ids: &[u64]) -> String { + let text: Vec = ids.iter().map(u64::to_string).collect(); + sha256_hex(text.join(",").as_bytes()) +} + +fn median(xs: impl Iterator>) -> Option { + // A single null makes the engine's figure null: never a median of a subset. + let v: Option> = xs.collect(); + v.filter(|v| !v.is_empty()).map(|v| percentile(&v, 0.5)) +} + +fn p95_ci(wall_s: &[f64], seed: u64) -> [f64; 2] { + let mut rng = SplitMix64::new(seed); + let mut stats: Vec = (0..BOOTSTRAP) + .map(|_| { + let s: Vec = (0..wall_s.len()) + .map(|_| wall_s[rng.below(wall_s.len())]) + .collect(); + percentile(&s, 0.95) + }) + .collect(); + stats.sort_by(f64::total_cmp); + [ + percentile_sorted(&stats, 0.025), + percentile_sorted(&stats, 0.975), + ] +} + +/// Every row must share `first`'s set identity, GGUF, cell and apr tag. +fn check_consistent_rows(rows: &[Row], first: &Row) -> Result<(), SummaryError> { + for r in rows { + if r.set_sha != first.set_sha || r.replay_version != first.replay_version { + return Err(SummaryError::MixedSet); + } + if r.gguf_sha256 != first.gguf_sha256 { + return Err(SummaryError::Mismatch("gguf_sha256")); + } + if r.cell != first.cell { + return Err(SummaryError::Mismatch("cell")); + } + if r.apr_tag != first.apr_tag { + return Err(SummaryError::Mismatch("apr_tag")); + } + } + Ok(()) +} + +/// The largest lane p95 across every counted voter (the queue budget). +fn queue_budget(voters: &[(String, Vec)]) -> Result<(String, f64), SummaryError> { + if voters.is_empty() || voters.iter().any(|(_, v)| v.is_empty()) { + return Err(SummaryError::NoVoterLatency); + } + voters + .iter() + .map(|(lane, v)| (lane.clone(), percentile(v, 0.95))) + .max_by(|a, b| a.1.total_cmp(&b.1)) + .ok_or(SummaryError::NoVoterLatency) +} + +/// The engine-vs-engine ratios and the parse rate that make up most of +/// [`Speed`]: TTFT, per-stratum prefill, decode, peak RSS and parse rate. +struct Ratios { + ttft_ms: Ratio, + prefill_tps: BTreeMap, + decode_tps: Ratio, + peak_rss_mb: Ratio, + parse_rate: f64, +} + +fn compute_ratios(apr: &ByItem<'_>, llama: &ByItem<'_>) -> Ratios { + let pair = |f: fn(&Row) -> Option, keep: &dyn Fn(&Row) -> bool| { + Ratio::of( + median(apr.values().filter(|r| keep(r)).map(|r| f(r))), + median(llama.values().filter(|r| keep(r)).map(|r| f(r))), + ) + }; + let all = |_: &Row| true; + let prefill_tps = Stratum::ALL + .into_iter() + .filter(|s| apr.values().any(|r| r.stratum == *s)) + .map(|s| (s, pair(|r| r.prompt_tps, &|r: &Row| r.stratum == s))) + .collect(); + let peak = |m: &ByItem<'_>| -> Option { + let v: Option> = m.values().map(|r| r.peak_rss_mb).collect(); + v.and_then(|v| v.into_iter().reduce(f64::max)) + }; + let parsed = apr + .values() + .filter(|r| matches!(r.verdict, Verdict::Pass | Verdict::Fail)) + .count(); + Ratios { + ttft_ms: pair(|r| r.prompt_ms, &all), + prefill_tps, + decode_tps: pair(|r| r.decode_tps, &all), + peak_rss_mb: Ratio::of(peak(apr), peak(llama)), + parse_rate: parsed as f64 / apr.len() as f64, + } +} + +/// The fraction of `apr`'s items whose verdict matches `prev`'s apr rows on +/// the same set; `None` when there is no previous tag to compare against. +fn verdict_identity_vs_prev( + prev: Option<&[Row]>, + first: &Row, + apr: &ByItem<'_>, +) -> Result, SummaryError> { + let Some(prev) = prev else { + return Ok(None); + }; + let mut p = BTreeMap::new(); + for r in prev.iter().filter(|r| r.engine == Engine::Apr) { + if r.set_sha != first.set_sha { + return Err(SummaryError::PrevSetDiffers); + } + p.insert(r.diff_sha256.as_str(), r.verdict); + } + if !p.keys().eq(apr.keys()) { + return Err(SummaryError::PrevCoverageDiffers); + } + let same = apr + .values() + .filter(|r| p[r.diff_sha256.as_str()] == r.verdict) + .count(); + Ok(Some(same as f64 / apr.len() as f64)) +} + +/// Fold one run's rows into the §9 block. +/// +/// `voters` is `(lane, total seconds per review)` for every counted voter; +/// the queue budget is the largest lane p95. `prev` is the previous tag's +/// apr rows on the same set, for verdict identity. +/// +/// # Errors +/// See [`SummaryError`]; every refusal names what was inconsistent. +pub fn summarize( + rows: &[Row], + voters: &[(String, Vec)], + prev: Option<&[Row]>, + seed: u64, +) -> Result { + let first = rows.first().ok_or(SummaryError::NoRows)?; + check_consistent_rows(rows, first)?; + let [apr, llama] = index(rows)?; + if apr.is_empty() || !apr.keys().eq(llama.keys()) { + return Err(SummaryError::CoverageDiffers); + } + let prompt_ids_sha = prompt_ids_sha(&first.replay_version, &apr, &llama)?; + let (queue_budget_lane, queue_budget_p95_s) = queue_budget(voters)?; + + let wall_s: Vec = apr.values().map(|r| r.wall_ms / 1000.0).collect(); + let p95_s = percentile(&wall_s, 0.95); + let ratios = compute_ratios(&apr, &llama); + let verdict_identity_vs_prev_tag = verdict_identity_vs_prev(prev, first, &apr)?; + + Ok(Speed { + replay_version: first.replay_version.clone(), + set_sha: first.set_sha.clone(), + apr_tag: first.apr_tag.clone(), + cell: first.cell.clone(), + gguf_sha256: first.gguf_sha256.clone(), + n_items: apr.len(), + p50_s: percentile(&wall_s, 0.5), + p95_s, + p95_ci: p95_ci(&wall_s, seed), + queue_budget_p95_s, + queue_budget_lane, + within_budget: p95_s <= queue_budget_p95_s, + ttft_ms: ratios.ttft_ms, + prefill_tps: ratios.prefill_tps, + decode_tps: ratios.decode_tps, + peak_rss_mb: ratios.peak_rss_mb, + parse_rate: ratios.parse_rate, + verdict_identity_vs_prev_tag, + prompt_ids_sha, + }) +} + +#[cfg(test)] +#[path = "replay_tests.rs"] +mod tests; diff --git a/crates/aprender-review-experiment/src/replay_tests.rs b/crates/aprender-review-experiment/src/replay_tests.rs new file mode 100644 index 0000000000..3317b89d5e --- /dev/null +++ b/crates/aprender-review-experiment/src/replay_tests.rs @@ -0,0 +1,474 @@ +use super::*; +use crate::contamination::Index; +use crate::corpus::{hunk_fingerprints, sha256_hex, Sealed}; +use crate::receipt::NotRun; + +fn diff(tag: &str) -> String { + format!("diff --git a/{tag}.rs b/{tag}.rs\n--- a/{tag}.rs\n+++ b/{tag}.rs\n@@ -1,3 +1,4 @@\n fn {tag}() {{\n- old_{tag}();\n+ new_{tag}();\n+ more_{tag}();\n }}\n") +} + +fn cand(group: &str, tag: &str, tokens: u64) -> Candidate { + let d = diff(tag); + Candidate { + group: group.to_string(), + diff_sha256: sha256_hex(d.as_bytes()), + input_tokens: tokens, + diff: d, + } +} + +fn sealed_of(c: &Candidate) -> Sealed { + Sealed { + id: format!("G-{}", c.group), + diff_sha256: c.diff_sha256.clone(), + hunks: hunk_fingerprints(&c.diff), + } +} + +/// `n` fresh candidates per stratum, each its own group. +fn pool(n: usize) -> Vec { + let mut v = Vec::new(); + for s in Stratum::ALL { + for i in 0..n { + let tag = format!("{}_{i}", s.as_str()); + v.push(cand( + &format!("paiml/x#{tag}"), + &format!("f{tag}"), + s.upper() - i as u64, + )); + } + } + v +} + +fn unrelated_seal() -> Index { + Index::new(&[sealed_of(&cand("paiml/sealed#1", "sealedonly", 10))]) +} + +#[test] +fn stratum_bounds_are_inclusive_and_refuse_zero_and_overflow() { + assert_eq!(Stratum::of(0), None); + assert_eq!(Stratum::of(1), Some(Stratum::K2)); + assert_eq!(Stratum::of(2048), Some(Stratum::K2)); + assert_eq!(Stratum::of(2049), Some(Stratum::K8)); + assert_eq!(Stratum::of(8192), Some(Stratum::K8)); + assert_eq!(Stratum::of(8193), Some(Stratum::K16)); + assert_eq!(Stratum::of(16384), Some(Stratum::K16)); + assert_eq!(Stratum::of(16385), Some(Stratum::K32)); + assert_eq!(Stratum::of(32768), Some(Stratum::K32)); + assert_eq!(Stratum::of(32769), None); +} + +/// FALSIFY-REPLAY-001: a sealed item never enters the set, by diff sha, +/// by hunk or by a copy under another group. +#[test] +fn falsify_replay_001_sealed_items_are_excluded() { + let mut cands = pool(3); + let leak = cands[0].clone(); + // Same diff under another PR: the diff-sha / hunk hit must still drop it. + let mut copy = leak.clone(); + copy.group = "paiml/other#9".into(); + cands.push(copy); + let ix = Index::new(&[sealed_of(&leak)]); + let err = build("v1", 4354, 3, &cands, &ix).unwrap_err(); + assert_eq!( + err, + BuildError::Short(vec![Short { + stratum: Stratum::K2, + have: 2, + need: 3 + }]), + "only the sealed diff's stratum may come up short" + ); + let set = build("v1", 4354, 2, &cands, &ix).expect("2 per stratum fits"); + assert!(set.items.iter().all(|i| i.diff_sha256 != leak.diff_sha256)); +} + +#[test] +fn an_empty_sealed_index_is_refused() { + assert_eq!( + build("v1", 1, 1, &pool(1), &Index::default()), + Err(BuildError::EmptySealedIndex) + ); +} + +/// FALSIFY-REPLAY-002: a short stratum is an error, never padded. +#[test] +fn falsify_replay_002_short_strata_are_refused_not_padded() { + let mut cands = pool(2); + cands.retain(|c| Stratum::of(c.input_tokens) != Some(Stratum::K32)); + cands.push(cand("paiml/x#big", "big", 40_000)); + let err = build("v1", 1, 2, &cands, &unrelated_seal()).unwrap_err(); + assert_eq!( + err, + BuildError::Short(vec![Short { + stratum: Stratum::K32, + have: 0, + need: 2 + }]) + ); +} + +/// FALSIFY-REPLAY-003: one item per repo#PR and per diff sha, and the set +/// does not depend on candidate order. +#[test] +fn falsify_replay_003_one_per_group_and_order_free() { + let mut cands = pool(4); + // Two more diffs on an existing PR and a re-post of an existing diff. + let g = cands[0].group.clone(); + cands.push(cand(&g, "extra_a", 100)); + cands.push(cand(&g, "extra_b", 200)); + let mut repost = cands[1].clone(); + repost.group = "paiml/y#1".into(); + cands.push(repost); + let a = build("v1", 7, 4, &cands, &unrelated_seal()).expect("fits"); + cands.reverse(); + let b = build("v1", 7, 4, &cands, &unrelated_seal()).expect("fits"); + assert_eq!(a, b); + assert_eq!(a.sha(), b.sha()); + let groups: BTreeSet<_> = a.items.iter().map(|i| &i.group).collect(); + let shas: BTreeSet<_> = a.items.iter().map(|i| &i.diff_sha256).collect(); + assert_eq!(groups.len(), a.items.len()); + assert_eq!(shas.len(), a.items.len()); + assert_eq!(a.items.len(), 16); + for s in Stratum::ALL { + assert_eq!(a.items.iter().filter(|i| i.stratum == s).count(), 4); + } + let c = build("v1", 8, 4, &cands, &unrelated_seal()).expect("fits"); + assert_ne!(a.sha(), c.sha(), "the seed is part of the frozen identity"); +} + +#[test] +fn render_parse_round_trips_and_rejects_a_foreign_header() { + let set = build("v1", 3, 2, &pool(3), &unrelated_seal()).expect("fits"); + let text = set.render(); + assert_eq!(Set::parse(&text), Ok(set.clone())); + assert!(Set::parse(&text.replace(SCHEMA, "other-v1")).is_err()); + assert!(Set::parse("").is_err()); + assert_eq!(set.sha(), sha256_hex(text.as_bytes())); +} + +fn row(engine: Engine, i: usize, wall_ms: f64) -> Row { + let stratum = Stratum::ALL[i % 4]; + let (pt, dt) = match engine { + Engine::Apr => (500.0, 20.0), + Engine::LlamaCpp => (1000.0, 40.0), + }; + Row { + schema: ROW_SCHEMA.into(), + replay_version: "v1".into(), + set_sha: "s".repeat(64), + diff_sha256: format!("{i:064x}"), + stratum, + engine, + engine_version: "x".into(), + apr_tag: "v0.70.1".into(), + cell: "lambda-4090".into(), + gguf_sha256: "g".repeat(64), + wall_ms, + prompt_ms: Some(if engine == Engine::Apr { 300.0 } else { 100.0 }), + prompt_tps: Some(pt * (1.0 + (i % 4) as f64)), + decode_tps: Some(dt), + input_tokens: Some(1000), + output_tokens: Some(64), + peak_rss_mb: Some(if engine == Engine::Apr { + 3000.0 + i as f64 + } else { + 2000.0 + }), + verdict: if i % 5 == 0 { + Verdict::Unparsed + } else { + Verdict::Pass + }, + prompt_ids_sha256: None, + } +} + +fn run(n: usize) -> Vec { + let mut v: Vec = (0..n) + .map(|i| row(Engine::Apr, i, 1000.0 * (i + 1) as f64)) + .collect(); + v.extend((0..n).map(|i| row(Engine::LlamaCpp, i, 500.0))); + v +} + +fn voters() -> Vec<(String, Vec)> { + vec![ + ("haiku".into(), (1..=20).map(f64::from).collect()), + ("agy".into(), (1..=20).map(|x| f64::from(x) * 3.0).collect()), + ] +} + +/// FALSIFY-REPLAY-004: ratios are apr / llama_cpp and the queue budget is +/// the slowest counted voter's p95. +#[test] +fn falsify_replay_004_ratios_and_queue_budget() { + let s = summarize(&run(20), &voters(), None, 1).expect("valid run"); + assert_eq!(s.n_items, 20); + assert!((s.p50_s - 10.0).abs() < 1e-9, "p50 {}", s.p50_s); + assert!((s.p95_s - 19.0).abs() < 1e-9, "p95 {}", s.p95_s); + assert!( + s.p95_ci[0] <= s.p95_s && s.p95_s <= s.p95_ci[1], + "{:?}", + s.p95_ci + ); + assert_eq!(s.queue_budget_lane, "agy"); + assert!((s.queue_budget_p95_s - 57.0).abs() < 1e-9); + assert!(s.within_budget); + assert_eq!(s.ttft_ms.ratio, Some(3.0)); + assert_eq!(s.decode_tps.ratio, Some(0.5)); + assert_eq!(s.prefill_tps.len(), 4); + for (st, r) in &s.prefill_tps { + let k = 1.0 + Stratum::ALL.iter().position(|x| x == st).unwrap_or(9) as f64; + assert_eq!(r.apr, Some(500.0 * k), "{st:?}"); + assert_eq!(r.ratio, Some(0.5)); + } + assert_eq!(s.peak_rss_mb.apr, Some(3019.0)); + assert!((s.parse_rate - 0.8).abs() < 1e-9); + assert_eq!(s.verdict_identity_vs_prev_tag, None); + + let slow = vec![("haiku".to_string(), vec![5.0])]; + assert!( + !summarize(&run(20), &slow, None, 1) + .expect("valid") + .within_budget + ); +} + +#[test] +fn falsify_replay_004_a_null_server_timing_is_null_not_a_subset_median() { + let mut rows = run(8); + rows[9].prompt_ms = None; // one llama row + let s = summarize(&rows, &voters(), None, 1).expect("valid"); + assert_eq!(s.ttft_ms.llama_cpp, None); + assert_eq!(s.ttft_ms.ratio, None); + assert_eq!(s.ttft_ms.apr, Some(300.0)); +} + +/// FALSIFY-REPLAY-005: an inconsistent run never prints a number. +#[test] +fn falsify_replay_005_inconsistent_runs_are_refused() { + let v = voters(); + let base = run(8); + assert_eq!(summarize(&[], &v, None, 1), Err(SummaryError::NoRows)); + + let mut r = base.clone(); + r[3].set_sha = "t".repeat(64); + assert_eq!(summarize(&r, &v, None, 1), Err(SummaryError::MixedSet)); + let mut r = base.clone(); + r[3].replay_version = "v2".into(); + assert_eq!(summarize(&r, &v, None, 1), Err(SummaryError::MixedSet)); + let mut r = base.clone(); + r[9].gguf_sha256 = "h".repeat(64); + assert_eq!( + summarize(&r, &v, None, 1), + Err(SummaryError::Mismatch("gguf_sha256")) + ); + let mut r = base.clone(); + r[9].cell = "gx10".into(); + assert_eq!( + summarize(&r, &v, None, 1), + Err(SummaryError::Mismatch("cell")) + ); + let mut r = base.clone(); + r[9].apr_tag = "v0.70.0".into(); + assert_eq!( + summarize(&r, &v, None, 1), + Err(SummaryError::Mismatch("apr_tag")) + ); + + let mut r = base.clone(); + r.pop(); + assert_eq!( + summarize(&r, &v, None, 1), + Err(SummaryError::CoverageDiffers) + ); + let mut r = base.clone(); + r.push(r[0].clone()); + assert!(matches!( + summarize(&r, &v, None, 1), + Err(SummaryError::DuplicateRow { .. }) + )); + let mut r = base.clone(); + r[2].verdict = Verdict::NotRun(NotRun::ContextOverflow); + assert!(matches!( + summarize(&r, &v, None, 1), + Err(SummaryError::NotRun { + engine: Engine::Apr, + .. + }) + )); + let llama_only: Vec = base + .iter() + .filter(|r| r.engine == Engine::LlamaCpp) + .cloned() + .collect(); + assert_eq!( + summarize(&llama_only, &v, None, 1), + Err(SummaryError::CoverageDiffers) + ); + + assert_eq!( + summarize(&base, &[], None, 1), + Err(SummaryError::NoVoterLatency) + ); + let empty_lane = vec![("haiku".to_string(), vec![])]; + assert_eq!( + summarize(&base, &empty_lane, None, 1), + Err(SummaryError::NoVoterLatency) + ); +} + +/// FALSIFY-REPLAY-006: verdict identity compares the same set, item by item. +#[test] +fn falsify_replay_006_verdict_identity_vs_previous_tag() { + let now = run(10); + let mut prev: Vec = now.clone(); + for r in prev.iter_mut().filter(|r| r.engine == Engine::Apr).take(3) { + r.verdict = Verdict::Fail; + r.apr_tag = "v0.70.0".into(); + } + let s = summarize(&now, &voters(), Some(&prev), 1).expect("valid"); + assert_eq!(s.verdict_identity_vs_prev_tag, Some(0.7)); + // llama rows in `prev` are ignored. + let apr_prev: Vec = prev + .iter() + .filter(|r| r.engine == Engine::Apr) + .cloned() + .collect(); + assert_eq!( + summarize(&now, &voters(), Some(&apr_prev), 1) + .expect("valid") + .verdict_identity_vs_prev_tag, + Some(0.7) + ); + let mut other = apr_prev.clone(); + other[0].set_sha = "t".repeat(64); + assert_eq!( + summarize(&now, &voters(), Some(&other), 1), + Err(SummaryError::PrevSetDiffers) + ); + let mut fewer = apr_prev; + fewer.pop(); + assert_eq!( + summarize(&now, &voters(), Some(&fewer), 1), + Err(SummaryError::PrevCoverageDiffers) + ); +} + +#[test] +fn row_serializes_with_stable_engine_and_stratum_names() { + let v = serde_json::to_value(row(Engine::LlamaCpp, 1, 1.0)).expect("serializes"); + assert_eq!(v["engine"], "llama_cpp"); + assert_eq!(v["stratum"], "8k"); + assert_eq!(v["schema"], ROW_SCHEMA); +} + +/// FALSIFY-REPLAY-003: two PRs with many diffs each cannot fill a stratum of 3. +#[test] +fn falsify_replay_003_many_diffs_on_one_pr_count_once() { + let mut cands: Vec = pool(3) + .into_iter() + .filter(|c| Stratum::of(c.input_tokens) != Some(Stratum::K2)) + .collect(); + for i in 0..6 { + cands.push(cand("paiml/a#1", &format!("a{i}"), 100 + i)); + cands.push(cand("paiml/b#2", &format!("b{i}"), 200 + i)); + } + assert_eq!( + build("v1", 1, 3, &cands, &unrelated_seal()), + Err(BuildError::Short(vec![Short { + stratum: Stratum::K2, + have: 2, + need: 3 + }])) + ); +} + +/// FALSIFY-REPLAY-002: a stratum is never over-filled either. +#[test] +fn falsify_replay_002_each_stratum_holds_exactly_per_stratum() { + let set = build("v1", 1, 2, &pool(5), &unrelated_seal()).expect("fits"); + assert_eq!(set.items.len(), 8); + for s in Stratum::ALL { + assert_eq!(set.items.iter().filter(|i| i.stratum == s).count(), 2); + } +} + +#[test] +fn falsify_replay_004_ratio_and_median_edges() { + assert_eq!(Ratio::of(Some(0.5), Some(2.0)).ratio, Some(0.25)); + assert_eq!(Ratio::of(Some(1.0), Some(0.0)).ratio, None); + assert_eq!(Ratio::of(None, Some(2.0)).ratio, None); + let xs: Vec> = (1..=10).map(|x| Some(f64::from(x))).collect(); + assert_eq!(median(xs.into_iter()), Some(5.0)); + assert_eq!(median(std::iter::empty()), None); + assert_eq!(median([Some(1.0), None].into_iter()), None); +} + +#[test] +fn falsify_replay_004_p95_equal_to_the_budget_is_within() { + let exact = vec![("haiku".to_string(), vec![19.0])]; + let s = summarize(&run(20), &exact, None, 1).expect("valid"); + assert!((s.p95_s - s.queue_budget_p95_s).abs() < 1e-12); + assert!(s.within_budget); +} + +fn pinned_run(n: usize) -> Vec { + let mut rows = run(n); + for r in &mut rows { + r.replay_version = V2.into(); + let i = usize::from_str_radix(&r.diff_sha256, 16).expect("hex id"); + r.prompt_ids_sha256 = Some(ids_sha256(&[1, 2, i as u64])); + } + rows +} + +/// PRM-S1 v2: only a prompt-pinning version asks for thinking off, on every engine. +#[test] +fn falsify_replay_007_v2_request_turns_thinking_off_and_v1_is_unchanged() { + let v1 = replay_request("review-replay-v1", "m", "p", "d"); + assert_eq!(v1, crate::harness::request_body("m", "p", "d")); + let v2 = replay_request(V2, "m", "p", "d"); + assert_eq!(v2["chat_template_kwargs"]["enable_thinking"], false); + assert_eq!(v2["messages"], v1["messages"]); +} + +/// PRM-S1 v2: a pinned run whose engines prefilled different ids never prints +/// a number, and neither does one missing an ids sha. +#[test] +fn falsify_replay_007_v2_refuses_differing_or_missing_prompt_ids() { + let v = voters(); + let ok = summarize(&pinned_run(8), &v, None, 1).expect("ids agree"); + assert!(ok.prompt_ids_sha.is_some()); + + let mut r = pinned_run(8); + r[10].prompt_ids_sha256 = Some(ids_sha256(&[9])); + assert_eq!( + summarize(&r, &v, None, 1), + Err(SummaryError::PromptIdsDiffer { + diff_sha256: r[10].diff_sha256.clone() + }) + ); + let mut r = pinned_run(8); + r[3].prompt_ids_sha256 = None; + assert!(matches!( + summarize(&r, &v, None, 1), + Err(SummaryError::NoPromptIds { + engine: Engine::Apr, + .. + }) + )); + // v1 needs none and reports none. + assert_eq!( + summarize(&run(8), &v, None, 1).expect("v1").prompt_ids_sha, + None + ); +} + +#[test] +fn falsify_replay_007_ids_sha_is_order_sensitive() { + assert_ne!(ids_sha256(&[1, 2]), ids_sha256(&[2, 1])); + assert_ne!(ids_sha256(&[12]), ids_sha256(&[1, 2])); +} diff --git a/crates/aprender-review-experiment/src/score.rs b/crates/aprender-review-experiment/src/score.rs index 1c075ad37d..25fab6345e 100644 --- a/crates/aprender-review-experiment/src/score.rs +++ b/crates/aprender-review-experiment/src/score.rs @@ -5,9 +5,9 @@ //! scored as `NotRun{Inadmissible}`, so dropping or corrupting a row can only //! lower a score. The statistics themselves are the frozen `stats.rs`. -use crate::corpus::Item; +use crate::corpus::{Class, Item}; use crate::receipt::{admissible, localized, parse_verdict, Expect, NotRun, Receipt, Verdict}; -use crate::stats::{self, Paired}; +use crate::stats::{self, Paired, VsVoters}; use serde::Serialize; use std::collections::BTreeMap; @@ -40,6 +40,8 @@ impl Ratio { #[derive(Debug, Clone, PartialEq, Eq)] pub struct Scored { pub id: String, + /// Corpus class; H5 reads class R only (spec v2 §3). + pub class: Class, pub defect: bool, pub verdict: Verdict, /// A finding names the defect file (only set on a FAIL of a defect). @@ -161,6 +163,7 @@ where .map(|i| { got.remove(&i.id).unwrap_or_else(|| Scored { id: i.id.clone(), + class: i.class, defect: i.class.is_defect(), verdict: Verdict::NotRun(NotRun::Inadmissible), localized: false, @@ -188,6 +191,7 @@ where let Some(out) = &r.output else { return Ok(Scored { id: r.item_id.clone(), + class: item.class, defect, verdict: r.verdict, localized: false, @@ -213,6 +217,7 @@ where }; Ok(Scored { id: r.item_id.clone(), + class: item.class, defect, verdict, localized: verdict == Verdict::Fail && defect && localized(&text, &item.defect), @@ -220,16 +225,14 @@ where }) } -/// H1/H2 input: items whose verdict or output bytes differ between two runs, -/// and items that did not run on both (reported, not silently dropped). -#[must_use] -pub fn divergence(a: &[Scored], b: &[Scored]) -> (u64, u64) { +fn divergence_by(a: &[Scored], b: &[Scored], bytes: bool) -> (u64, u64) { let bm: BTreeMap<&str, &Scored> = b.iter().map(|s| (s.id.as_str(), s)).collect(); let (mut divergent, mut missing) = (0, 0); for s in a { match bm.get(s.id.as_str()) { Some(t) if s.verdict.executed() && t.verdict.executed() => { - divergent += u64::from(s.verdict != t.verdict || s.output_sha != t.output_sha); + let differs = s.verdict != t.verdict || (bytes && s.output_sha != t.output_sha); + divergent += u64::from(differs); } _ => missing += 1, } @@ -237,6 +240,22 @@ pub fn divergence(a: &[Scored], b: &[Scored]) -> (u64, u64) { (divergent, missing) } +/// H2 input (and the descriptive H1 byte count): items whose verdict or +/// output bytes differ between two runs, and items that did not run on both +/// (reported, not silently dropped). +#[must_use] +pub fn divergence(a: &[Scored], b: &[Scored]) -> (u64, u64) { + divergence_by(a, b, true) +} + +/// H1 input (spec v2 §3): items whose VERDICT differs across cells. Output +/// bytes are descriptive for H1 (`divergence`); a byte-only difference does +/// not reject it. +#[must_use] +pub fn verdict_divergence(a: &[Scored], b: &[Scored]) -> (u64, u64) { + divergence_by(a, b, false) +} + /// H4/H5 input: items where BOTH arms parsed a verdict, paired; the rest are /// dropped pairwise and counted (analysis plan). #[must_use] @@ -258,6 +277,52 @@ pub fn paired_parsed(a: &[Scored], b: &[Scored]) -> (Vec, u64) { (out, dropped) } +/// H5 input (spec v2 §3): `paired_parsed` restricted to items of `class` in +/// `a`. H5 decides on class R; class P is reported beside it (mutant confound). +#[must_use] +pub fn paired_parsed_class(a: &[Scored], b: &[Scored], class: Class) -> (Vec, u64) { + let only: Vec = a.iter().filter(|s| s.class == class).cloned().collect(); + paired_parsed(&only, b) +} + +/// H4 input (spec v2 §3): items where apr AND both voters parsed a verdict; +/// the rest are dropped and counted. +#[must_use] +pub fn vs_voters_parsed(apr: &[Scored], voters: [&[Scored]; 2]) -> (Vec, u64) { + let maps = voters.map(|v| { + v.iter() + .map(|s| (s.id.as_str(), s)) + .collect::>() + }); + let parsed = |v: Verdict| matches!(v, Verdict::Pass | Verdict::Fail); + let mut out = Vec::new(); + let mut dropped = 0; + for s in apr { + match (maps[0].get(s.id.as_str()), maps[1].get(s.id.as_str())) { + (Some(h), Some(g)) if parsed(s.verdict) && parsed(h.verdict) && parsed(g.verdict) => { + out.push(VsVoters { + defect: s.defect, + apr_fail: s.verdict == Verdict::Fail, + voter_fail: [h.verdict == Verdict::Fail, g.verdict == Verdict::Fail], + }); + } + _ => dropped += 1, + } + } + (out, dropped) +} + +/// κ input (spec v2 §3, descriptive): per-item error indicators of (a, b) +/// over items both lanes parsed. Error = not correct. +#[must_use] +pub fn error_pairs(a: &[Scored], b: &[Scored]) -> (Vec, Vec) { + paired_parsed(a, b) + .0 + .iter() + .map(|p| (p.a_fail != p.defect, p.b_fail != p.defect)) + .unzip() +} + /// H3 input: per-item correctness of (challenger, champion) over the /// champion's items; a missing challenger row is incorrect. #[must_use] diff --git a/crates/aprender-review-experiment/src/score_tests.rs b/crates/aprender-review-experiment/src/score_tests.rs index b9a425fbbb..bbfc6ac4a1 100644 --- a/crates/aprender-review-experiment/src/score_tests.rs +++ b/crates/aprender-review-experiment/src/score_tests.rs @@ -5,6 +5,7 @@ use crate::receipt::tests::{row, EXPECT}; fn s(id: &str, defect: bool, verdict: Verdict, localized: bool) -> Scored { Scored { id: id.into(), + class: if defect { Class::P } else { Class::G }, defect, verdict, localized, @@ -271,6 +272,56 @@ fn divergence_pairs_and_mcnemar_by_hand() { ); } +#[test] +fn falsify_h1_is_verdict_identity_bytes_are_descriptive() { + let a = fixture(); + let mut b = fixture(); + b[0].output_sha = Some("other".into()); // bytes only + assert_eq!( + verdict_divergence(&a, &b).0, + 0, + "a byte-only diff must not reject H1" + ); + assert_eq!( + divergence(&a, &b).0, + 1, + "H2/descriptive still sees the bytes" + ); + b[3].verdict = Verdict::Fail; + assert_eq!(verdict_divergence(&a, &b).0, 1, "a verdict diff rejects H1"); +} + +#[test] +fn h5_reads_class_r_only_and_h4_needs_three_parsed_lanes() { + let mut a = fixture(); + a[1].class = Class::R; // d2 + let (r, _) = paired_parsed_class(&a, &a, Class::R); + assert_eq!(r.len(), 1, "one parsed class-R item"); + let (p, _) = paired_parsed_class(&a, &a, Class::P); + assert!(p.iter().all(|x| x.defect) && !p.is_empty()); + + let (all, dropped) = paired_parsed(&a, &a); + let (three, dropped3) = vs_voters_parsed(&a, [&a, &a]); + assert_eq!((three.len(), dropped3), (all.len(), dropped)); + let mut agy = fixture(); + agy[0].verdict = Verdict::Unparsed; + let (three, dropped3) = vs_voters_parsed(&a, [&a, &agy]); + assert_eq!((three.len(), dropped3), (all.len() - 1, dropped + 1)); +} + +#[test] +fn error_pairs_mark_wrong_parsed_verdicts() { + let a = fixture(); + let mut b = fixture(); + b[0].verdict = Verdict::Pass; // d1 missed by b + let (ea, eb) = error_pairs(&a, &b); + assert_eq!(ea.len(), eb.len()); + assert_eq!( + eb.iter().filter(|e| **e).count(), + ea.iter().filter(|e| **e).count() + 1 + ); +} + #[test] fn ratio_value_is_k_over_n_and_none_only_for_empty() { assert_eq!(Ratio::new(3, 4).value(), Some(0.75)); diff --git a/crates/aprender-review-experiment/src/secret.rs b/crates/aprender-review-experiment/src/secret.rs new file mode 100644 index 0000000000..91301df9d8 --- /dev/null +++ b/crates/aprender-review-experiment/src/secret.rs @@ -0,0 +1,594 @@ +//! Secret scan for trace admission (contract `trace-admission-secret-v1`). +//! +//! Two independent scanners, and a row is a hit when EITHER fires: +//! +//! - **builtin**: hand-written detectors (AWS key id / secret, GitHub token, +//! PEM private key, Anthropic key, Slack token, JWT, high-entropy assignment). +//! It honours no allowlist, so the AWS documentation key `AKIA…EXAMPLE`, which +//! gitleaks allowlists, is still caught here. +//! - **gitleaks**: the upstream gitleaks v8.30.1 ruleset, vendored verbatim as +//! data (`data/gitleaks/`, MIT) and interpreted in-process: keyword prefilter, +//! regex, secret group, entropy floor, per-rule and global allowlists. +//! +//! Where gitleaks keys a rule or allowlist on a file PATH, the scan has none, so +//! it fails safe: a path-scoped rule runs on all content, and a path-conditioned +//! allowlist allows nothing. A rule with no regex (path-only) cannot fire on +//! content and is listed in [`PATH_ONLY_RULES`]. + +use std::sync::OnceLock; + +use regex::{Regex, RegexBuilder}; +use serde::Deserialize; + +/// The vendored ruleset, byte-for-byte as released upstream. +pub const GITLEAKS_TOML: &str = include_str!("../data/gitleaks/gitleaks-v8.30.1.toml"); +/// sha256 of [`GITLEAKS_TOML`]; a test pins it. +pub const GITLEAKS_SHA256: &str = + "e163e53b9e7e8a8511e77271e2b323ed057759542a6d988258afe3a1fa329caf"; +/// The scanner versions a `secret_scan` record carries. +pub const SCANNER_VERSIONS: &[&str] = &["builtin-1", "gitleaks-v8.30.1"]; +/// gitleaks rules that match on a path alone and so never fire on content. +pub const PATH_ONLY_RULES: &[&str] = &["pkcs12-file"]; + +/// Which scanner fired. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub enum Scanner { + Builtin, + Gitleaks, +} + +/// One rule that fired on the scanned text. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub struct Hit { + pub scanner: Scanner, + pub rule: &'static str, +} + +/// Every (scanner, rule) hit in `text` or in any JSON string value on one of +/// its lines (an escape such as `\u0041KIA…` cannot hide a key), sorted, each +/// at most once. +#[must_use] +pub fn scan(text: &str) -> Vec { + scan_without(text, &[]) +} + +/// [`scan`] with the named rules switched off: the ablation the recall +/// falsifier uses to show every rule is load-bearing for its canary. +#[must_use] +pub fn scan_without(text: &str, disabled: &[Hit]) -> Vec { + let texts = texts(text); + let on = |h: &Hit| !disabled.contains(h); + let mut hits: Vec = BUILTIN + .iter() + .filter_map(|&(rule, found)| { + let hit = Hit { + scanner: Scanner::Builtin, + rule, + }; + (on(&hit) && texts.iter().any(|t| found(t))).then_some(hit) + }) + .collect(); + let gl = gitleaks(); + for rule in &gl.rules { + let hit = Hit { + scanner: Scanner::Gitleaks, + rule: rule.id, + }; + if on(&hit) && texts.iter().any(|t| rule.fires(t, &gl.global)) { + hits.push(hit); + } + } + hits.sort(); + hits.dedup(); + hits +} + +/// The raw text plus every JSON string value found on its lines. +fn texts(text: &str) -> Vec { + let mut out = vec![text.to_string()]; + for line in text.lines() { + if let Ok(v) = serde_json::from_str::(line) { + collect_strings(&v, &mut out); + } + } + out +} + +fn collect_strings(v: &serde_json::Value, out: &mut Vec) { + match v { + serde_json::Value::String(s) => out.push(s.clone()), + serde_json::Value::Array(a) => a.iter().for_each(|x| collect_strings(x, out)), + serde_json::Value::Object(o) => o.values().for_each(|x| collect_strings(x, out)), + _ => {} + } +} + +/// Shannon entropy of `s` in bits per byte. +#[must_use] +pub fn entropy(s: &str) -> f64 { + let mut counts = [0usize; 256]; + s.bytes().for_each(|b| counts[usize::from(b)] += 1); + let n = s.len() as f64; + counts + .iter() + .filter(|&&c| c > 0) + .map(|&c| { + let p = c as f64 / n; + -p * p.log2() + }) + .sum() +} + +// --------------------------------------------------------------------------- +// builtin +// --------------------------------------------------------------------------- + +type Detector = fn(&str) -> bool; + +/// The builtin detectors, by rule name. +pub const BUILTIN: &[(&str, Detector)] = &[ + ("aws-access-key-id", aws_access_key_id), + ("aws-secret-access-key", aws_secret_access_key), + ("github-token", github_token), + ("private-key", private_key), + ("anthropic-key", anthropic_key), + ("slack-token", slack_token), + ("jwt", jwt), + ("high-entropy-assignment", high_entropy_assignment), +]; + +fn is_word(c: u8) -> bool { + c.is_ascii_alphanumeric() || c == b'_' +} + +/// Byte offsets just past each occurrence of `prefix` that does not start +/// inside a longer word. +fn after<'a>(text: &'a str, prefix: &'a str) -> impl Iterator + 'a { + let b = text.as_bytes(); + text.match_indices(prefix) + .filter(move |(i, _)| *i == 0 || !is_word(b[i - 1])) + .map(move |(i, _)| i + prefix.len()) +} + +/// Length of the run at `from` whose bytes satisfy `ok`. +fn run(text: &str, from: usize, ok: impl Fn(u8) -> bool) -> usize { + text.as_bytes()[from..] + .iter() + .take_while(|&&c| ok(c)) + .count() +} + +/// `AKIA`/`ASIA`/`ABIA`/`ACCA` + exactly 16 of `[A-Z0-9]`, as a whole word. +fn aws_access_key_id(text: &str) -> bool { + let key = |c: u8| c.is_ascii_uppercase() || c.is_ascii_digit(); + ["AKIA", "ASIA", "ABIA", "ACCA"] + .iter() + .any(|p| after(text, p).any(|i| run(text, i, is_word) == 16 && run(text, i, key) == 16)) +} + +/// `aws_secret_access_key` then `=`/`:`/quotes/space, then a 40-char key. +fn aws_secret_access_key(text: &str) -> bool { + let lower = text.to_ascii_lowercase(); + let key = |c: u8| c.is_ascii_alphanumeric() || c == b'/' || c == b'+'; + let found = after(&lower, "aws_secret_access_key").any(|i| { + let j = i + run(text, i, |c| { + matches!(c, b'=' | b':' | b'"' | b'\'' | b' ' | b'\t') + }); + j > i && run(text, j, key) == 40 + }); + found +} + +/// Classic `gh[pousr]_` + 36 alphanumerics, or a fine-grained `github_pat_`. +fn github_token(text: &str) -> bool { + ["ghp_", "gho_", "ghu_", "ghs_", "ghr_"] + .iter() + .any(|p| after(text, p).any(|i| run(text, i, |c| c.is_ascii_alphanumeric()) == 36)) + || after(text, "github_pat_").any(|i| run(text, i, is_word) >= 22) +} + +/// A PEM `-----BEGIN … PRIVATE KEY-----` armour line. +fn private_key(text: &str) -> bool { + text.match_indices("-----BEGIN ").any(|(i, m)| { + let rest = &text[i + m.len()..]; + rest.find("-----") + .is_some_and(|end| rest[..end].ends_with("PRIVATE KEY") && !rest[..end].contains('\n')) + }) +} + +fn anthropic_key(text: &str) -> bool { + after(text, "sk-ant-").any(|i| run(text, i, |c| is_word(c) || c == b'-') >= 20) +} + +fn slack_token(text: &str) -> bool { + ["xoxb-", "xoxa-", "xoxp-", "xoxr-", "xoxs-"] + .iter() + .any(|p| { + after(text, p).any(|i| run(text, i, |c| c.is_ascii_alphanumeric() || c == b'-') >= 10) + }) +} + +/// `eyJ.eyJ.`: a JWT whose header and payload are both +/// JSON objects, with a signature of at least 16 characters. +fn jwt(text: &str) -> bool { + let b64 = |c: u8| c.is_ascii_alphanumeric() || c == b'-' || c == b'_'; + after(text, "eyJ").any(|i| { + let h = run(text, i, b64); + let p = i + h; + if h < 10 || !text[p..].starts_with(".eyJ") { + return false; + } + let q = p + 4; + let body = run(text, q, b64); + let s = q + body; + body >= 10 && text[s..].starts_with('.') && run(text, s + 1, b64) >= 16 + }) +} + +/// `secret`/`token`/`password`/`api_key`/… assigned a value of at least 24 +/// characters whose entropy exceeds hex's 4 bits/char: random material, not a +/// hash, a path, or a call such as `token = self.next()`. +fn high_entropy_assignment(text: &str) -> bool { + const NAMES: &[&str] = &[ + "secret", + "token", + "password", + "passwd", + "api_key", + "apikey", + "access_key", + "private_key", + "client_secret", + ]; + let lower = text.to_ascii_lowercase(); + let value = |c: u8| c.is_ascii_alphanumeric() || matches!(c, b'+' | b'/' | b'_' | b'-' | b'='); + let found = NAMES.iter().any(|name| { + lower.match_indices(name).any(|(i, m)| { + let k = i + m.len() + run(&lower, i + m.len(), is_word); + let sep = run(text, k, |c| { + matches!(c, b'=' | b':' | b'"' | b'\'' | b' ' | b'\t' | b'>') + }); + if sep == 0 || !text[k..k + sep].contains(['=', ':']) { + return false; + } + let v = k + sep; + let len = run(text, v, value); + len >= 24 && entropy(&text[v..v + len]) > 4.2 + }) + }); + found +} + +// --------------------------------------------------------------------------- +// gitleaks as data +// --------------------------------------------------------------------------- + +#[derive(Deserialize)] +struct ConfigDef { + #[serde(default)] + allowlist: Option, + #[serde(default)] + rules: Vec, +} + +#[derive(Deserialize)] +#[serde(rename_all = "camelCase")] +struct RuleDef { + id: String, + regex: Option, + secret_group: Option, + entropy: Option, + #[serde(default)] + keywords: Vec, + #[serde(default)] + allowlists: Vec, +} + +#[derive(Deserialize)] +#[serde(rename_all = "camelCase")] +struct AllowDef { + #[serde(default)] + regexes: Vec, + #[serde(default)] + stopwords: Vec, + #[serde(default)] + paths: Vec, + regex_target: Option, + condition: Option, +} + +#[derive(Clone, Copy, PartialEq, Eq)] +enum Target { + Secret, + Match, + Line, +} + +struct Allow { + regexes: Vec, + stopwords: Vec, + target: Target, + /// `condition = "AND"` with paths: a path we do not have is a condition we + /// cannot meet, so this allowlist allows nothing. + inert: bool, +} + +impl Allow { + fn allows(&self, secret: &str, whole: &str, line: &str) -> bool { + if self.inert { + return false; + } + let target = match self.target { + Target::Secret => secret, + Target::Match => whole, + Target::Line => line, + }; + let lower = secret.to_lowercase(); + self.regexes.iter().any(|r| r.is_match(target)) + || self.stopwords.iter().any(|w| lower.contains(w.as_str())) + } +} + +/// One compiled gitleaks rule. +pub struct Rule { + pub id: &'static str, + regex: Regex, + secret_group: Option, + entropy: Option, + keywords: Vec, + allows: Vec, +} + +impl Rule { + fn fires(&self, text: &str, global: &Allow) -> bool { + if !self.keywords.is_empty() { + let lower = text.to_lowercase(); + if !self.keywords.iter().any(|k| lower.contains(k.as_str())) { + return false; + } + } + self.regex.captures_iter(text).any(|caps| { + let whole = caps.get(0).map_or("", |m| m.as_str()); + let secret = self + .secret_group + .and_then(|g| caps.get(g)) + .or_else(|| { + caps.iter() + .skip(1) + .flatten() + .find(|m| !m.as_str().is_empty()) + }) + .map_or(whole, |m| m.as_str()); + if let Some(floor) = self.entropy { + if entropy(secret) <= floor { + return false; + } + if self.id.starts_with("generic") && !secret.bytes().any(|b| b.is_ascii_digit()) { + return false; + } + } + let start = caps.get(0).map_or(0, |m| m.start()); + let end = caps.get(0).map_or(0, |m| m.end()); + let ls = text[..start].rfind('\n').map_or(0, |i| i + 1); + let le = text[end..].find('\n').map_or(text.len(), |i| end + i); + let line = &text[ls..le]; + !global.allows(secret, whole, line) + && !self.allows.iter().any(|a| a.allows(secret, whole, line)) + }) + } +} + +/// The compiled ruleset. +pub struct Gitleaks { + pub rules: Vec, + global: Allow, + /// Rules whose regex failed to compile: must be empty (a test asserts it). + pub failed: Vec<(String, String)>, +} + +/// The vendored ruleset, compiled once. +/// +/// # Panics +/// The vendored TOML does not parse; the sha-pin test catches that first. +pub fn gitleaks() -> &'static Gitleaks { + static GL: OnceLock = OnceLock::new(); + GL.get_or_init(|| compile(GITLEAKS_TOML).expect("vendored gitleaks toml parses")) +} + +fn build(pattern: &str) -> Result { + RegexBuilder::new(&go_to_rust(pattern)) + .size_limit(1 << 28) + .dfa_size_limit(1 << 28) + .build() +} + +fn allow(def: AllowDef, failed: &mut Vec<(String, String)>, owner: &str) -> Allow { + let target = match def.regex_target.as_deref() { + Some("match") => Target::Match, + Some("line") => Target::Line, + _ => Target::Secret, + }; + let and = def.condition.as_deref() == Some("AND"); + let regexes = def + .regexes + .iter() + .filter_map(|p| { + build(p) + .map_err(|e| failed.push((format!("{owner} allowlist"), e.to_string()))) + .ok() + }) + .collect(); + Allow { + regexes, + stopwords: def.stopwords.iter().map(|w| w.to_lowercase()).collect(), + target, + inert: and && !def.paths.is_empty(), + } +} + +fn compile(toml_text: &str) -> Result { + let def: ConfigDef = toml::from_str(toml_text)?; + let mut failed = Vec::new(); + let global = def.allowlist.map_or( + Allow { + regexes: Vec::new(), + stopwords: Vec::new(), + target: Target::Secret, + inert: true, + }, + |a| allow(a, &mut failed, "global"), + ); + let mut rules = Vec::new(); + for r in def.rules { + let Some(pattern) = r.regex else { continue }; + let id: &'static str = Box::leak(r.id.into_boxed_str()); + match build(&pattern) { + Ok(regex) => rules.push(Rule { + id, + regex, + secret_group: r.secret_group, + entropy: r.entropy, + keywords: r.keywords.iter().map(|k| k.to_lowercase()).collect(), + allows: r + .allowlists + .into_iter() + .map(|a| allow(a, &mut failed, id)) + .collect(), + }), + Err(e) => failed.push((id.to_string(), e.to_string())), + } + } + Ok(Gitleaks { + rules, + global, + failed, + }) +} + +/// Push one escape sequence (`\` plus the escaped char, which may be +/// multi-byte UTF-8) onto `out` and advance `i` past it. +fn push_escape(p: &str, i: &mut usize, out: &mut String) { + let n = p[*i + 1..].chars().next().map_or(1, char::len_utf8); + out.push_str(&p[*i..*i + 1 + n]); + *i += 1 + n; +} + +/// A `[` that just opened a character class: `[^]` and `[]` are pushed +/// verbatim (Go RE2 allows a class with an immediate `]`, which Rust's parser +/// would otherwise read as closing the class one byte early). Returns `true` +/// when it fully consumed the bracket (the caller should `continue`); `false` +/// means neither special case applied and the `[` still needs its common, +/// single-character push. +fn push_class_open(b: &[u8], i: &mut usize, out: &mut String) -> bool { + if b.get(*i + 1) == Some(&b'^') { + out.push_str("[^"); + *i += 2; + if b.get(*i) == Some(&b']') { + out.push(']'); + *i += 1; + } + return true; + } + if b.get(*i + 1) == Some(&b']') { + out.push_str("[]"); + *i += 2; + return true; + } + false +} + +/// Cursor state for [`go_to_rust`], threaded through one character at a time +/// by [`GoToRust::step`]. +struct GoToRust { + i: usize, + class: bool, + prev_atom: bool, +} + +impl GoToRust { + /// A `\` escape: pushed verbatim, `prev_atom` set. Returns `true` when it + /// consumed the byte at `self.i` (the caller should move on). + fn step_escape(&mut self, p: &str, c: u8, out: &mut String) -> bool { + if c != b'\\' || self.i + 1 >= p.len() { + return false; + } + push_escape(p, &mut self.i, out); + self.prev_atom = true; + true + } + + /// A `{` outside a character class that is not the start of a valid + /// repetition: Go RE2 reads it as a literal, so Rust needs it escaped. + fn step_brace(&mut self, p: &str, c: u8, out: &mut String) -> bool { + if c != b'{' || (self.prev_atom && is_repetition(&p[self.i..])) { + return false; + } + out.push_str("\\{"); + self.i += 1; + self.prev_atom = true; + true + } + + /// One character that reached the common path: whatever it is, it is + /// pushed as-is and `prev_atom` is updated for the next `{`/repetition + /// check. + fn step_plain(&mut self, p: &str, c: u8, out: &mut String) { + self.prev_atom = self.class || !matches!(c, b'(' | b'|'); + let n = p[self.i..].chars().next().map_or(1, char::len_utf8); + out.push_str(&p[self.i..self.i + n]); + self.i += n; + } + + /// Advance past exactly one input character (possibly multi-byte), + /// pushing its Rust-regex translation onto `out`. + fn step(&mut self, p: &str, b: &[u8], out: &mut String) { + let c = b[self.i]; + if self.step_escape(p, c, out) { + return; + } + if self.class { + self.class = c != b']'; + } else if c == b'[' { + self.class = true; + if push_class_open(b, &mut self.i, out) { + return; + } + } else if self.step_brace(p, c, out) { + return; + } + self.step_plain(p, c, out); + } +} + +/// Go RE2 takes a `{` that does not open a valid repetition as a literal; Rust +/// rejects it. Escape those, outside character classes; nothing else differs +/// for this ruleset (the compile-all test is the proof). +fn go_to_rust(p: &str) -> String { + let b = p.as_bytes(); + let mut out = String::with_capacity(p.len() + 8); + let mut st = GoToRust { + i: 0, + class: false, + prev_atom: false, + }; + while st.i < b.len() { + st.step(p, b, &mut out); + } + out +} + +/// `{n}`, `{n,}` or `{n,m}` at the start of `s`. +fn is_repetition(s: &str) -> bool { + let Some(end) = s.find('}') else { + return false; + }; + let inner = &s[1..end]; + let (lo, hi) = inner.split_once(',').unwrap_or((inner, "0")); + !lo.is_empty() + && lo.bytes().all(|c| c.is_ascii_digit()) + && hi.bytes().all(|c| c.is_ascii_digit()) +} + +#[cfg(test)] +#[path = "secret_tests.rs"] +pub(crate) mod tests; diff --git a/crates/aprender-review-experiment/src/secret_tests.rs b/crates/aprender-review-experiment/src/secret_tests.rs new file mode 100644 index 0000000000..a414a609a0 --- /dev/null +++ b/crates/aprender-review-experiment/src/secret_tests.rs @@ -0,0 +1,359 @@ +//! Falsifiers for `trace-admission-secret-v1` (PRA-001 T6, G-SEC / G-SEC-FN). +//! +//! Every canary is BUILT at run time, from pieces or from a seeded generator, +//! so no key-shaped literal sits in the source for a repo scanner to flag. + +// `json!` expands to an `unwrap` of an infallible `to_value`. +#![allow(clippy::disallowed_methods)] + +use super::*; +use sha2::{Digest, Sha256}; + +/// A splitmix64 stream: the same seed yields the same canaries on every run. +struct Gen(u64); + +impl Gen { + fn next(&mut self) -> u64 { + self.0 = self.0.wrapping_add(0x9E37_79B9_7F4A_7C15); + let mut z = self.0; + z = (z ^ (z >> 30)).wrapping_mul(0xBF58_476D_1CE4_E5B9); + z = (z ^ (z >> 27)).wrapping_mul(0x94D0_49BB_1331_11EB); + z ^ (z >> 31) + } + + /// `n` characters drawn from `alphabet`. + fn take(&mut self, alphabet: &str, n: usize) -> String { + let a = alphabet.as_bytes(); + (0..n) + .map(|_| char::from(a[(self.next() % a.len() as u64) as usize])) + .collect() + } +} + +const UPPER: &str = "ABCDEFGHIJKLMNOPQRSTUVWXYZ"; +const B32: &str = "ABCDEFGHIJKLMNOPQRSTUVWXYZ234567"; +const ALNUM: &str = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"; +const LOWER_NUM: &str = "abcdefghijklmnopqrstuvwxyz0123456789"; +const HEX: &str = "0123456789abcdef"; +const DIGITS: &str = "0123456789"; + +fn b64url() -> String { + [ALNUM, "-_"].concat() +} + +fn b64() -> String { + [ALNUM, "+/"].concat() +} + +fn hit(scanner: Scanner, rule: &'static str) -> Hit { + Hit { scanner, rule } +} + +/// The AWS documentation key: gitleaks allowlists it (`.+EXAMPLE$`), which is +/// exactly why the builtin scanner must catch it on its own. +fn planted_aws_key() -> String { + ["AKIA", "IOSFODNN7", "EXAMPLE"].concat() +} + +/// The 20 canaries, each labelled with the ONE rule it is planted for. +pub(crate) fn canaries() -> Vec<(Hit, String)> { + use Scanner::{Builtin as B, Gitleaks as G}; + let mut g = Gen(0x5EC0_0001); + let pem = |g: &mut Gen, kind: &str| { + let body = (0..4) + .map(|_| g.take(&b64(), 64)) + .collect::>() + .join("\n"); + format!("-----BEGIN {kind}PRIVATE KEY-----\n{body}\n-----END {kind}PRIVATE KEY-----\n") + }; + let jwt = |g: &mut Gen| { + format!( + "Authorization: Bearer eyJ{}.eyJ{}.{}\n", + g.take(ALNUM, 30), + g.take(ALNUM, 60), + g.take(&b64url(), 43) + ) + }; + vec![ + // builtin + ( + hit(B, "aws-access-key-id"), + format!("export AWS_ACCESS_KEY_ID={}", planted_aws_key()), + ), + ( + hit(B, "aws-secret-access-key"), + format!( + "aws configure set aws_secret_access_key \"{}\"", + g.take(&b64(), 40) + ), + ), + ( + hit(B, "github-token"), + ["gho_", &g.take(ALNUM, 36)].concat(), + ), + (hit(B, "private-key"), pem(&mut g, "OPENSSH ")), + ( + hit(B, "anthropic-key"), + ["sk-ant-", "admin01-", &g.take(ALNUM, 40)].concat(), + ), + ( + hit(B, "slack-token"), + ["xoxp-", &g.take(DIGITS, 11), "-", &g.take(ALNUM, 24)].concat(), + ), + (hit(B, "jwt"), jwt(&mut g)), + ( + hit(B, "high-entropy-assignment"), + format!("client_secret: \"{}\"", g.take(ALNUM, 32)), + ), + // gitleaks + ( + hit(G, "aws-access-token"), + format!("key {}{} ", "ASIA", g.take(B32, 16)), + ), + (hit(G, "github-pat"), ["ghp_", &g.take(ALNUM, 36)].concat()), + ( + hit(G, "github-fine-grained-pat"), + ["github_pat_", &g.take(ALNUM, 82)].concat(), + ), + (hit(G, "private-key"), pem(&mut g, "")), + (hit(G, "jwt"), jwt(&mut g)), + ( + hit(G, "generic-api-key"), + format!("api_token = \"{}{}\"", g.take(ALNUM, 30), g.take(DIGITS, 2)), + ), + ( + hit(G, "slack-bot-token"), + [ + "xoxb-", + &g.take(DIGITS, 12), + "-", + &g.take(DIGITS, 12), + "-", + &g.take(ALNUM, 24), + ] + .concat(), + ), + ( + hit(G, "stripe-access-token"), + format!("{}{} ", "sk_live_", g.take(ALNUM, 24)), + ), + ( + hit(G, "gitlab-pat"), + ["glpat-", &g.take(ALNUM, 20)].concat(), + ), + ( + hit(G, "npm-access-token"), + format!("{}{} ", "npm_", g.take(LOWER_NUM, 36)), + ), + ( + hit(G, "gcp-api-key"), + format!("{}{} ", "AIza", g.take(&b64url(), 35)), + ), + ( + hit(G, "digitalocean-pat"), + format!("{}{} ", "dop_v1_", g.take(HEX, 64)), + ), + ] +} + +/// The canary set covers what G-SEC names: AWS, GitHub PAT, private key, JWT, +/// high-entropy generic; 20 of them, each labelled with a distinct rule. +#[test] +fn canary_set_is_twenty_distinct_labelled_rules() { + let c = canaries(); + assert_eq!(c.len(), 20); + let mut labels: Vec = c.iter().map(|(h, _)| *h).collect(); + labels.sort(); + labels.dedup(); + assert_eq!(labels.len(), 20, "each canary names a distinct rule"); + for (h, _) in &c { + match h.scanner { + Scanner::Builtin => assert!(BUILTIN.iter().any(|(r, _)| r == &h.rule)), + Scanner::Gitleaks => assert!(gitleaks().rules.iter().any(|r| r.id == h.rule)), + } + } + assert!(c[0].1.contains(&planted_aws_key())); +} + +/// FALSIFY-TAS-001 (G-SEC): 100% canary recall. Each canary fires the rule it is +/// labelled with, bare and inside a captured JSONL row. +#[test] +fn falsify_tas_001_every_canary_fires_its_labelled_rule() { + for (label, text) in canaries() { + let row = serde_json::json!({"lane": "haiku-4-5", "output": text}).to_string(); + for t in [&text, &row] { + assert!(scan(t).contains(&label), "{label:?} missed: {t}"); + } + } +} + +/// FALSIFY-TAS-002 (G-SEC-FN): remove one rule and the canary planted for it +/// goes RED: its scanner then reports NOTHING on it, so no other rule of that +/// scanner is quietly carrying the recall check above. +#[test] +fn falsify_tas_002_removing_a_rule_turns_its_canary_red() { + for (label, text) in canaries() { + let rest: Vec = scan_without(&text, &[label]) + .into_iter() + .filter(|h| h.scanner == label.scanner) + .collect(); + assert_eq!( + rest, + Vec::::new(), + "{label:?} with its rule removed: {text}" + ); + } +} + +/// FALSIFY-TAS-003: the AWS documentation key is allowlisted by gitleaks and +/// caught by the builtin scanner; the union quarantines it. Removing the +/// builtin rule leaves NOTHING to catch it: the union is load-bearing. +#[test] +fn falsify_tas_003_aws_doc_key_needs_the_builtin_scanner() { + let text = format!("export AWS_ACCESS_KEY_ID={}", planted_aws_key()); + let aws = hit(Scanner::Builtin, "aws-access-key-id"); + assert_eq!(scan(&text), vec![aws]); + assert!(scan_without(&text, &[aws]).is_empty()); +} + +/// The vendored ruleset is the released v8.30.1 file, byte for byte. +#[test] +fn vendored_gitleaks_toml_is_sha_pinned() { + let got = format!("{:x}", Sha256::digest(GITLEAKS_TOML.as_bytes())); + assert_eq!(got, GITLEAKS_SHA256); +} + +/// Every gitleaks rule with a regex compiles, and the only rules without one +/// are the listed path-only rules: no rule is silently dropped. +#[test] +fn every_gitleaks_rule_compiles() { + let gl = gitleaks(); + assert!(gl.failed.is_empty(), "failed to compile: {:?}", gl.failed); + let declared = GITLEAKS_TOML + .lines() + .filter(|l| l.trim() == "[[rules]]") + .count(); + assert_eq!(declared, 222); + assert_eq!(gl.rules.len() + PATH_ONLY_RULES.len(), declared); + for id in PATH_ONLY_RULES { + assert!(gl.rules.iter().all(|r| r.id != *id)); + assert!(GITLEAKS_TOML.contains(&format!("id = \"{id}\""))); + } +} + +/// Go's literal `{` becomes an escaped one; real repetitions are untouched. +#[test] +fn go_regex_translation_case_table() { + for (go, rust) in [ + (r"^\$(?:\d+|{\d+})$", r"^\$(?:\d+|\{\d+})$"), + (r"a{2,5}", r"a{2,5}"), + (r"a{2}", r"a{2}"), + (r"[{]x", r"[{]x"), + (r"{{x}}", r"\{\{x}}"), + (r"a{,5}", r"a\{,5}"), + (r"\{\d{0,2}}", r"\{\d{0,2}}"), + ] { + assert_eq!(go_to_rust(go), rust, "{go}"); + assert!(Regex::new(&go_to_rust(go)).is_ok(), "{go}"); + } +} + +/// Builtin case table: must-match rows fire exactly their rule; must-not-match +/// rows fire nothing in the builtin scanner. +#[test] +fn builtin_case_table() { + let builtin = |t: &str| -> Vec<&'static str> { + scan(t) + .into_iter() + .filter(|h| h.scanner == Scanner::Builtin) + .map(|h| h.rule) + .collect() + }; + let hits = [ + (planted_aws_key(), "aws-access-key-id"), + (["ASIA", "Y34FZKBOKMUTVV7A"].concat(), "aws-access-key-id"), + (["ghp_", &"a".repeat(36)].concat(), "github-token"), + (["github_pat_", &"B".repeat(40)].concat(), "github-token"), + ( + ["-----BEGIN RSA ", "PRIVATE KEY-----"].concat(), + "private-key", + ), + (["-----BEGIN ", "PRIVATE KEY-----"].concat(), "private-key"), + ( + ["sk-ant-", "api03-", &"x".repeat(40)].concat(), + "anthropic-key", + ), + ( + ["xoxb-", "123456789012-", &"q".repeat(24)].concat(), + "slack-token", + ), + ]; + let escaped = format!("{{\"output\": \"\\u0041{}\"}}", &planted_aws_key()[1..]); + assert!( + !escaped.contains("AKIA"), + "the fixture must hide the literal" + ); + assert_eq!(builtin(&escaped), vec!["aws-access-key-id"]); + for (text, kind) in &hits { + assert_eq!(builtin(text), vec![*kind], "must match: {text}"); + } + let misses = [ + "AKIA".to_string(), + ["AKIA", "iosfodnn7example"].concat(), + ["XAKIA", "IOSFODNN7EXAMPLE"].concat(), + ["AKIA", "IOSFODNN7EXAMPLEX"].concat(), + "ghp_short".to_string(), + "-----BEGIN PUBLIC KEY-----".to_string(), + "the aws_secret_access_key field must be set".to_string(), + "sk-ant is the Anthropic key prefix".to_string(), + "eyJhbGciOiJIUzI1NiJ9 alone is a header".to_string(), + format!("token = {}", "0123456789abcdef".repeat(4)), + ]; + for text in &misses { + assert!(builtin(text).is_empty(), "must not match: {text}"); + } +} + +/// Ordinary review traffic fires neither scanner: quarantine is for secrets, +/// not for every row that mentions a token. +#[test] +fn ordinary_review_rows_are_clean() { + let upper = Gen(7).take(UPPER, 8); + for text in [ + "LGTM: the bound check is inclusive now.".to_string(), + "let token = lexer.next_token()?;".to_string(), + "+ password: String,\n- passwd: &'static str,".to_string(), + format!("sha256 {}", "0123456789abcdef".repeat(4)), + "fn secret_scan(rows: &str) -> Vec {".to_string(), + format!("see crates/aprender-serve/src/api/{upper}.rs"), + "--- a/src/lib.rs\n+++ b/src/lib.rs\n@@ -1,3 +1,3 @@\n-x\n+y".to_string(), + ] { + assert!(scan(&text).is_empty(), "{:?} fired on: {text}", scan(&text)); + } +} + +/// gitleaks semantics the canaries alone do not pin: the entropy floor drops a +/// low-entropy match, and an allowlist conditioned (AND) on a path we do not +/// have allows nothing, so the scan fails safe. +#[test] +fn gitleaks_semantics_case_table() { + let gl_rules = |t: &str| -> Vec<&'static str> { + scan(t) + .into_iter() + .filter(|h| h.scanner == Scanner::Gitleaks) + .map(|h| h.rule) + .collect() + }; + // `ghp_aaaa…` matches github-pat's regex but not its entropy floor of 3. + let flat = ["ghp_", &"a".repeat(36)].concat(); + assert!(!gl_rules(&flat).contains(&"github-pat"), "{flat}"); + // generic-api-key's `SRC…="…"` allowlist applies only to BitBake paths + // (condition AND); with no path it must not suppress the hit. + let mut g = Gen(0x5EC0_0002); + let bitbake = format!( + "SRC_API_TOKEN = \"{}{}\"", + g.take(ALNUM, 30), + g.take(DIGITS, 2) + ); + assert!(gl_rules(&bitbake).contains(&"generic-api-key"), "{bitbake}"); +} diff --git a/crates/aprender-review-experiment/src/sparse_logits.rs b/crates/aprender-review-experiment/src/sparse_logits.rs new file mode 100644 index 0000000000..017fcc560f --- /dev/null +++ b/crates/aprender-review-experiment/src/sparse_logits.rs @@ -0,0 +1,418 @@ +//! PRA-001 T11: the `sparse-logits-v1` blob (spec PRA-001 §2.7). +//! +//! Per output token: the sampled id, the top-k ids with their log-softmax at +//! temperature 1.0 (before any sampling filter), the full-vocab logsumexp, and +//! the retained mass `M = Σ exp(logprobs) ≤ 1`. Storing `M` is the point: a +//! top-k cache without its residual mass is a biased teacher estimate, and +//! the tail-bucket correction of a sparse forward-KL objective needs it. +//! +//! Layout: `MAGIC`, a u32-LE header length, the header JSON, then one zstd +//! frame holding the columns (all LE): sampled ids `n×u32`, ids `n·k×u32`, +//! logprobs `n·k×f16`, logsumexp `n×f32`, mass `n×f16`. The decoder is +//! strict: an unknown header key, a short column, a trailing byte, or a mass +//! that disagrees with its own logprobs is an error, never a best effort. + +use half::f16; +use serde::{Deserialize, Serialize}; + +pub const SCHEME: &str = "sparse-logits-v1"; +/// Magic prefix; the trailing newline keeps `head -c` output readable. +pub const MAGIC: &[u8; 8] = b"SPLOGV1\n"; +/// PRA-001 §2.7 default `k` `[A]`. +pub const DEFAULT_K: u8 = 20; +/// zstd level for the column frame. +const ZSTD_LEVEL: i32 = 3; +/// f16 has an 11-bit significand: Σ of k rounded exps can drift this much +/// from the rounded mass. Anything larger is a producer defect. +const MASS_TOLERANCE: f32 = 1.0 / 256.0; + +/// How the token set was chosen. `RsSample` is reserved (§2.7) so the 27B +/// teacher can write importance-sampled sets later without a schema change. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub enum Mode { + Topk, + RsSample, +} + +/// Per-blob header (§2.7). +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Header { + pub schema: String, + pub mode: Mode, + pub model_sha256: String, + pub tokenizer_sha256: String, + pub apr_tag: String, + /// The backend that actually served the row (`served_by`). + pub backend: String, + /// Sampling temperature of the recorded run. Logprobs are always T = 1.0. + pub temperature_of_record: f32, + pub k: u8, + pub n_tokens: u32, +} + +/// One output token's sparse distribution. +#[derive(Debug, Clone, PartialEq)] +pub struct TokenLogits { + pub token_id_sampled: u32, + /// Top-k ids, descending by logprob. u32: Qwen's vocab exceeds 2^16. + pub ids: Vec, + /// log-softmax at T = 1.0, same order as `ids`. + pub logprobs: Vec, + pub logsumexp_full: f32, + /// Retained mass `M = Σ exp(logprobs)`. + pub topk_mass: f16, +} + +impl TokenLogits { + /// Build from a full-vocab logit row at T = 1.0. `None` when `k` is 0, + /// exceeds the vocab, or a logit is not finite. + #[must_use] + pub fn from_full_logits(logits: &[f32], k: u8, token_id_sampled: u32) -> Option { + let k = usize::from(k); + if k == 0 || k > logits.len() || logits.iter().any(|x| !x.is_finite()) { + return None; + } + let max = logits.iter().copied().fold(f32::NEG_INFINITY, f32::max); + let sum: f64 = logits.iter().map(|&x| f64::from(x - max).exp()).sum(); + let lse = f64::from(max) + sum.ln(); + let mut order: Vec = (0..logits.len()).collect(); + // Descending logit, ascending id on ties: deterministic. + order.sort_by(|&a, &b| logits[b].total_cmp(&logits[a]).then(a.cmp(&b))); + order.truncate(k); + let lp: Vec = order.iter().map(|&i| f64::from(logits[i]) - lse).collect(); + let mass: f64 = lp.iter().map(|x| x.exp()).sum(); + Some(Self { + token_id_sampled, + ids: order + .iter() + .map(|&i| u32::try_from(i).ok()) + .collect::>()?, + logprobs: lp.iter().map(|&x| f16::from_f64(x)).collect(), + logsumexp_full: lse as f32, + topk_mass: f16::from_f64(mass.min(1.0)), + }) + } + + /// Residual (tail) mass `1 − M`, clamped at 0. + #[must_use] + pub fn residual_mass(&self) -> f32 { + (1.0 - self.topk_mass.to_f32()).max(0.0) + } + + fn check(&self, k: usize, at: usize) -> Result<(), String> { + if self.ids.len() != k || self.logprobs.len() != k { + return Err(format!( + "token {at}: {} ids / {} logprobs, header k = {k}", + self.ids.len(), + self.logprobs.len() + )); + } + if !self.logsumexp_full.is_finite() { + return Err(format!("token {at}: logsumexp_full is not finite")); + } + let mut sum = 0.0_f32; + for lp in &self.logprobs { + let v = lp.to_f32(); + if !v.is_finite() || v > 0.0 { + return Err(format!( + "token {at}: logprob {v} is not a finite log-probability" + )); + } + sum += v.exp(); + } + let m = self.topk_mass.to_f32(); + if !(0.0..=1.0 + MASS_TOLERANCE).contains(&m) { + return Err(format!("token {at}: topk_mass {m} outside [0, 1]")); + } + if (sum - m).abs() > MASS_TOLERANCE { + return Err(format!("token {at}: topk_mass {m} != Σexp(logprobs) {sum}")); + } + Ok(()) + } +} + +/// PRM C11 capture: the `choices[0].logprobs.content` of an `apr serve` +/// `/v1/chat/completions` body (#4026, `top_logprobs = k`) as one +/// [`TokenLogits`] per output token, ready for [`encode`]. +/// +/// Refused, never guessed: a body without `logsumexp_full` (a hosted or +/// foreign server — its tail mass is unknowable, and hosted output never +/// enters the corpus), fewer than `k` alternatives, an entry without +/// `token_id`, or logprobs that are not a descending, finite log-softmax. +pub fn from_chat_completion(body: &serde_json::Value, k: u8) -> Result, String> { + let content = body + .pointer("/choices/0/logprobs/content") + .and_then(serde_json::Value::as_array) + .ok_or("no choices[0].logprobs.content: the request did not ask for logprobs")?; + content + .iter() + .enumerate() + .map(|(at, entry)| { + token_from_serve_entry(entry, usize::from(k)).map_err(|e| format!("token {at}: {e}")) + }) + .collect() +} + +fn serve_id(v: &serde_json::Value) -> Result { + v.get("token_id") + .and_then(serde_json::Value::as_u64) + .and_then(|id| u32::try_from(id).ok()) + .ok_or_else(|| "no u32 token_id".to_string()) +} + +fn token_from_serve_entry(entry: &serde_json::Value, k: usize) -> Result { + if k == 0 { + return Err("k = 0".into()); + } + let lse = entry + .get("logsumexp_full") + .and_then(serde_json::Value::as_f64) + .filter(|x| x.is_finite()) + .ok_or("no finite logsumexp_full: not an apr serve #4026 body")?; + let top = entry + .get("top_logprobs") + .and_then(serde_json::Value::as_array) + .ok_or("no top_logprobs")?; + if top.len() < k { + return Err(format!("{} alternatives, k = {k}", top.len())); + } + let mut ids = Vec::with_capacity(k); + let mut lps = Vec::with_capacity(k); + for alt in &top[..k] { + let lp = alt + .get("logprob") + .and_then(serde_json::Value::as_f64) + .filter(|x| x.is_finite() && *x <= 0.0) + .ok_or("an alternative's logprob is not a finite value ≤ 0")?; + if lps.last().is_some_and(|&prev: &f64| lp > prev) { + return Err("top_logprobs are not descending".into()); + } + ids.push(serve_id(alt)?); + lps.push(lp); + } + let mass: f64 = lps.iter().map(|x| x.exp()).sum(); + Ok(TokenLogits { + token_id_sampled: serve_id(entry)?, + ids, + logprobs: lps.iter().map(|&x| f16::from_f64(x)).collect(), + logsumexp_full: lse as f32, + topk_mass: f16::from_f64(mass.min(1.0)), + }) +} + +fn check_all(h: &Header, tokens: &[TokenLogits]) -> Result<(), String> { + if h.schema != SCHEME { + return Err(format!("schema {} is not {SCHEME}", h.schema)); + } + if h.k == 0 { + return Err("k = 0".into()); + } + if usize::try_from(h.n_tokens).ok() != Some(tokens.len()) { + return Err(format!( + "header n_tokens {} != {} tokens", + h.n_tokens, + tokens.len() + )); + } + if !h.temperature_of_record.is_finite() || h.temperature_of_record < 0.0 { + return Err("temperature_of_record is not a finite value ≥ 0".into()); + } + for (name, v) in [ + ("model_sha256", &h.model_sha256), + ("tokenizer_sha256", &h.tokenizer_sha256), + ] { + if v.len() != 64 || !v.bytes().all(|b| b.is_ascii_hexdigit()) { + return Err(format!("{name} is not a sha256 hex digest")); + } + } + tokens + .iter() + .enumerate() + .try_for_each(|(i, t)| t.check(usize::from(h.k), i)) +} + +/// Encode one blob. Refuses a header/token set the decoder would reject, so +/// a blob that exists is a blob that reads. +pub fn encode(h: &Header, tokens: &[TokenLogits]) -> Result, String> { + check_all(h, tokens)?; + let head = serde_json::to_vec(h).map_err(|e| e.to_string())?; + let mut cols = Vec::with_capacity(tokens.len() * (usize::from(h.k) * 6 + 10)); + for t in tokens { + cols.extend_from_slice(&t.token_id_sampled.to_le_bytes()); + } + for t in tokens { + t.ids + .iter() + .for_each(|x| cols.extend_from_slice(&x.to_le_bytes())); + } + for t in tokens { + t.logprobs + .iter() + .for_each(|x| cols.extend_from_slice(&x.to_bits().to_le_bytes())); + } + for t in tokens { + cols.extend_from_slice(&t.logsumexp_full.to_le_bytes()); + } + for t in tokens { + cols.extend_from_slice(&t.topk_mass.to_bits().to_le_bytes()); + } + let body = zstd::bulk::compress(&cols, ZSTD_LEVEL).map_err(|e| e.to_string())?; + let len = u32::try_from(head.len()).map_err(|_| "header too large".to_string())?; + let mut out = Vec::with_capacity(MAGIC.len() + 4 + head.len() + body.len()); + out.extend_from_slice(MAGIC); + out.extend_from_slice(&len.to_le_bytes()); + out.extend_from_slice(&head); + out.extend_from_slice(&body); + Ok(out) +} + +/// Uncompressed column bytes for `n` tokens at `k` (the M6 "raw" figure). +#[must_use] +pub fn raw_column_bytes(n: usize, k: usize) -> usize { + n * (4 + k * (4 + 2) + 4 + 2) +} + +struct Cursor<'a>(&'a [u8]); + +impl Cursor<'_> { + fn take(&mut self) -> Result<[u8; N], String> { + if self.0.len() < N { + return Err("column data is short".into()); + } + let (a, b) = self.0.split_at(N); + self.0 = b; + a.try_into().map_err(|_| "column data is short".to_string()) + } +} + +/// Decode and validate one blob. +pub fn decode(blob: &[u8]) -> Result<(Header, Vec), String> { + let rest = blob.strip_prefix(MAGIC.as_slice()).ok_or("bad magic")?; + if rest.len() < 4 { + return Err("truncated header length".into()); + } + let (len, rest) = rest.split_at(4); + let len = u32::from_le_bytes(len.try_into().map_err(|_| "header length")?) as usize; + if rest.len() < len { + return Err("truncated header".into()); + } + let (head, body) = rest.split_at(len); + let h: Header = serde_json::from_slice(head).map_err(|e| format!("header: {e}"))?; + let (n, k) = (h.n_tokens as usize, usize::from(h.k)); + let want = raw_column_bytes(n, k); + let cols = zstd::bulk::decompress(body, want + 1).map_err(|e| format!("columns: {e}"))?; + if cols.len() != want { + return Err(format!( + "columns hold {} bytes, header implies {want}", + cols.len() + )); + } + let mut c = Cursor(&cols); + let mut sampled = Vec::with_capacity(n); + for _ in 0..n { + sampled.push(u32::from_le_bytes(c.take()?)); + } + let mut ids = Vec::with_capacity(n * k); + for _ in 0..n * k { + ids.push(u32::from_le_bytes(c.take()?)); + } + let mut lps = Vec::with_capacity(n * k); + for _ in 0..n * k { + lps.push(f16::from_bits(u16::from_le_bytes(c.take()?))); + } + let mut lse = Vec::with_capacity(n); + for _ in 0..n { + lse.push(f32::from_le_bytes(c.take()?)); + } + let mut tokens = Vec::with_capacity(n); + for (i, (&s, &l)) in sampled.iter().zip(&lse).enumerate() { + tokens.push(TokenLogits { + token_id_sampled: s, + ids: ids[i * k..(i + 1) * k].to_vec(), + logprobs: lps[i * k..(i + 1) * k].to_vec(), + logsumexp_full: l, + topk_mass: f16::from_bits(u16::from_le_bytes(c.take()?)), + }); + } + check_all(&h, &tokens)?; + Ok((h, tokens)) +} + +/// Where the CAS stores the blob with this sha under `root` +/// (`blobs/sha256/ab/cd/.zst`, the layout the data card reads). +#[must_use] +pub fn cas_path(root: &std::path::Path, sha: &str) -> std::path::PathBuf { + root.join("blobs/sha256") + .join(&sha[..2]) + .join(&sha[2..4]) + .join(format!("{sha}.zst")) +} + +/// What one C11 capture stored. +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +pub struct Captured { + /// sha256 of the raw blob: the row's `logits_sha`. + pub logits_sha: String, + pub n_tokens: u32, + pub k: u8, + pub raw_bytes: u64, + pub stored_bytes: u64, +} + +/// PRM C11: turn an `apr serve` `/v1/chat/completions` body into a +/// `sparse-logits-v1` blob in the CAS under `root`, and return its sha. +/// +/// `h.n_tokens` is taken from the body. Nothing is written unless the blob +/// decodes back to exactly what was captured, and the stored frame is read +/// back and re-hashed before the sha is returned. An existing blob with the +/// same sha is kept (the store is content-addressed), after the same check. +/// +/// # Errors +/// The body is refused by [`from_chat_completion`] or [`encode`], or the +/// store cannot be written or does not read back. +pub fn capture( + root: &std::path::Path, + h: &Header, + body: &serde_json::Value, +) -> Result { + let tokens = from_chat_completion(body, h.k)?; + let h = Header { + n_tokens: u32::try_from(tokens.len()).map_err(|_| "too many tokens".to_string())?, + ..h.clone() + }; + let blob = encode(&h, &tokens)?; + if decode(&blob)? != (h.clone(), tokens) { + return Err("encoded blob does not decode to the capture".into()); + } + let sha = crate::corpus::sha256_hex(&blob); + let path = cas_path(root, &sha); + if !path.exists() { + let frame = zstd::bulk::compress(&blob, ZSTD_LEVEL).map_err(|e| e.to_string())?; + let dir = path.parent().ok_or("cas path has no parent")?; + std::fs::create_dir_all(dir).map_err(|e| format!("{}: {e}", dir.display()))?; + let tmp = path.with_extension(format!("zst.tmp{}", std::process::id())); + std::fs::write(&tmp, &frame).map_err(|e| format!("{}: {e}", tmp.display()))?; + std::fs::rename(&tmp, &path).map_err(|e| format!("{}: {e}", path.display()))?; + } + let stored = std::fs::read(&path).map_err(|e| format!("{}: {e}", path.display()))?; + let raw = zstd::stream::decode_all(stored.as_slice()).map_err(|e| e.to_string())?; + if crate::corpus::sha256_hex(&raw) != sha { + return Err(format!( + "{}: stored blob does not hash to {sha}", + path.display() + )); + } + Ok(Captured { + logits_sha: sha, + n_tokens: h.n_tokens, + k: h.k, + raw_bytes: blob.len() as u64, + stored_bytes: stored.len() as u64, + }) +} + +#[cfg(test)] +#[path = "sparse_logits_tests.rs"] +mod tests; diff --git a/crates/aprender-review-experiment/src/sparse_logits_tests.rs b/crates/aprender-review-experiment/src/sparse_logits_tests.rs new file mode 100644 index 0000000000..e95be94fb2 --- /dev/null +++ b/crates/aprender-review-experiment/src/sparse_logits_tests.rs @@ -0,0 +1,384 @@ +// `json!` expands to an `unwrap` of an infallible `to_value`. +#![allow(clippy::disallowed_methods)] + +use super::*; + +const SHA: &str = "70dffe5625e1849661303b82bb1fdf5254ab1d4560e2fe3eacc147e5d2588bba"; + +fn header(k: u8, n: u32) -> Header { + Header { + schema: SCHEME.into(), + mode: Mode::Topk, + model_sha256: SHA.into(), + tokenizer_sha256: SHA.into(), + apr_tag: "v0.70.0-rc.1".into(), + backend: "gx10-cuda".into(), + temperature_of_record: 0.7, + k, + n_tokens: n, + } +} + +/// A deterministic vocab row bigger than 2^16 so id width is exercised. +fn row(seed: u32, vocab: usize) -> Vec { + (0..vocab) + .map(|i| { + let x = (i as u32) + .wrapping_mul(2_654_435_761) + .wrapping_add(seed.wrapping_mul(40_503)); + f32::from((x >> 20) as u16) / 256.0 + }) + .collect() +} + +fn tokens(n: u32, vocab: usize) -> Vec { + (0..n) + .map(|s| { + TokenLogits::from_full_logits(&row(s, vocab), DEFAULT_K, s * 7).expect("finite row") + }) + .collect() +} + +/// FALSIFY-SPL-001: header and every column survive a round trip exactly. +#[test] +fn falsify_spl_001_round_trip_is_exact() { + let t = tokens(16, 70_000); + let h = header(DEFAULT_K, 16); + let (h2, t2) = decode(&encode(&h, &t).expect("encode")).expect("decode"); + assert_eq!(h2, h); + assert_eq!(t2, t); + assert!( + t.iter().flat_map(|x| &x.ids).any(|&id| id > 65_535), + "fixture must carry ids > u16" + ); +} + +/// FALSIFY-SPL-002: the stored mass is Σexp(logprobs) of the top-k of the +/// full softmax, and 1 − M is the true tail mass. +#[test] +fn falsify_spl_002_residual_mass_is_the_true_tail() { + let logits = [3.0_f32, 1.0, 0.5, 0.0, -1.0, -2.0]; + let t = TokenLogits::from_full_logits(&logits, 2, 0).expect("row"); + let z: f64 = logits.iter().map(|&x| f64::from(x).exp()).sum(); + let tail = (f64::from(0.5_f32).exp() + 1.0 + (-1.0_f64).exp() + (-2.0_f64).exp()) / z; + assert_eq!(t.ids, vec![0, 1]); + assert!( + (f64::from(t.residual_mass()) - tail).abs() < 1e-3, + "{} vs {tail}", + t.residual_mass() + ); + assert!((f64::from(t.logsumexp_full) - z.ln()).abs() < 1e-5); + assert!(t.logprobs.iter().all(|lp| lp.to_f32() <= 0.0)); + let (_, back) = decode(&encode(&header(2, 1), &[t.clone()]).expect("encode")).expect("decode"); + assert_eq!(back[0].residual_mass(), t.residual_mass()); +} + +/// FALSIFY-SPL-003: a mass that disagrees with its own logprobs is refused +/// at BOTH ends — a producer cannot write it, a reader cannot accept it. +#[test] +fn falsify_spl_003_inconsistent_mass_is_refused() { + let mut t = tokens(1, 1_000); + t[0].topk_mass = f16::from_f32(1.0); + assert!(encode(&header(DEFAULT_K, 1), &t).is_err()); + let good = tokens(1, 1_000); + let mut blob = encode(&header(DEFAULT_K, 1), &good).expect("encode"); + // Re-encode the columns with a forged mass, header untouched. + let hl = u32::from_le_bytes(blob[8..12].try_into().expect("len")) as usize; + let mut cols = zstd::bulk::decompress(&blob[12 + hl..], 1 << 20).expect("cols"); + let n = cols.len(); + cols[n - 2..].copy_from_slice(&f16::from_f32(1.0).to_bits().to_le_bytes()); + blob.truncate(12 + hl); + blob.extend(zstd::bulk::compress(&cols, 3).expect("zstd")); + assert!(decode(&blob) + .expect_err("forged mass") + .contains("topk_mass")); +} + +/// FALSIFY-SPL-004: the header is closed and the columns are exact-length. +#[test] +fn falsify_spl_004_decoder_is_strict() { + let t = tokens(2, 500); + let blob = encode(&header(DEFAULT_K, 2), &t).expect("encode"); + let hl = u32::from_le_bytes(blob[8..12].try_into().expect("len")) as usize; + let head = std::str::from_utf8(&blob[12..12 + hl]).expect("utf8"); + // Unknown key. + let extra = head.replacen('{', "{\"extra\":1,", 1); + let mut b = MAGIC.to_vec(); + b.extend((extra.len() as u32).to_le_bytes()); + b.extend(extra.as_bytes()); + b.extend(&blob[12 + hl..]); + assert!(decode(&b).is_err(), "unknown header key accepted"); + // n_tokens lies about the column length. + assert!(encode(&header(DEFAULT_K, 3), &t).is_err()); + let lie = head.replace("\"n_tokens\":2", "\"n_tokens\":1"); + let mut b = MAGIC.to_vec(); + b.extend((lie.len() as u32).to_le_bytes()); + b.extend(lie.as_bytes()); + b.extend(&blob[12 + hl..]); + assert!(decode(&b).is_err(), "column overrun accepted"); + // Truncation anywhere and a bad magic. + for cut in [0, 7, 11, 12 + hl - 1, blob.len() - 1] { + assert!(decode(&blob[..cut]).is_err(), "truncated at {cut} accepted"); + } + let mut bad = blob.clone(); + bad[0] = b'X'; + assert!(decode(&bad).is_err()); + // A trailing byte after the column frame. + let mut long = blob.clone(); + long.push(0); + assert!(decode(&long).is_err(), "trailing byte accepted"); + // Exactly one extra column byte inside a valid zstd frame. + let mut cols = zstd::bulk::decompress(&blob[12 + hl..], 1 << 20).expect("cols"); + cols.push(0); + let mut b = blob[..12 + hl].to_vec(); + b.extend(zstd::bulk::compress(&cols, 3).expect("zstd")); + assert!(decode(&b).is_err(), "one extra column byte accepted"); +} + +#[test] +fn from_full_logits_rejects_degenerate_rows() { + assert!(TokenLogits::from_full_logits(&[1.0, 2.0], 0, 0).is_none()); + assert!(TokenLogits::from_full_logits(&[1.0, 2.0], 3, 0).is_none()); + assert!(TokenLogits::from_full_logits(&[1.0, f32::NAN], 1, 0).is_none()); + assert!(TokenLogits::from_full_logits(&[1.0, f32::INFINITY], 1, 0).is_none()); + // k == vocab: all the mass is kept, residual ≈ 0. + let t = TokenLogits::from_full_logits(&[0.0, 0.0, 0.0], 3, 0).expect("row"); + assert!(t.residual_mass() < 1e-3); + assert_eq!(t.ids, vec![0, 1, 2], "ties break by ascending id"); +} + +#[test] +fn header_rejects_non_sha_and_bad_schema() { + let t = tokens(1, 100); + let mut h = header(DEFAULT_K, 1); + h.model_sha256 = "abc".into(); + assert!(encode(&h, &t).is_err()); + let mut h = header(DEFAULT_K, 1); + h.schema = "sparse-logits-v0".into(); + assert!(encode(&h, &t).is_err()); + let mut h = header(DEFAULT_K, 1); + h.temperature_of_record = f32::NAN; + assert!(encode(&h, &t).is_err()); +} + +/// M6: the RAW figure is exact (130 B/token at k = 20; §2.7 estimated ≈128). +/// The encoded figure here is from a SYNTHETIC, highly regular fixture and is +/// a codec smoke only — it is not M6. M6's encoded bytes/token are measured +/// on real `apr serve` rows once top-k capture exists. +#[test] +fn m6_raw_bytes_per_token_at_k20() { + assert_eq!(raw_column_bytes(1, 20), 130); + let n = 256; + let blob = encode(&header(DEFAULT_K, n), &tokens(n, 151_936)).expect("encode"); + let raw = raw_column_bytes(n as usize, 20); + // Column frame + header is bounded by raw + zstd frame overhead + header. + assert!( + blob.len() < raw + 1024, + "encoded {} > raw {raw} + 1 KiB", + blob.len() + ); +} + +/// A body shaped as `apr serve` writes it for `top_logprobs = k` (#4026): +/// `logprob = logit − logsumexp_full` over the full row, most likely first. +fn serve_body(rows: &[Vec], k: usize, sampled: &[u32]) -> serde_json::Value { + let content: Vec = rows + .iter() + .zip(sampled) + .map(|(r, &s)| { + let t = TokenLogits::from_full_logits(r, 1, s).expect("finite row"); + let lse = f64::from(t.logsumexp_full); + let mut order: Vec = (0..r.len()).collect(); + order.sort_by(|&a, &b| r[b].total_cmp(&r[a]).then(a.cmp(&b))); + let top: Vec = order[..k] + .iter() + .map(|&i| serde_json::json!({"token_id": i, "logprob": f64::from(r[i]) - lse})) + .collect(); + serde_json::json!({ + "token_id": s, + "logprob": f64::from(r[s as usize]) - lse, + "top_logprobs": top, + "logsumexp_full": lse, + }) + }) + .collect(); + serde_json::json!({"choices": [{"logprobs": {"content": content}}]}) +} + +/// PRM C11: a serve body captures to the same record the full row gives, and +/// the capture encodes and decodes. +#[test] +fn c11_serve_capture_matches_the_full_row_and_round_trips() { + let rows: Vec> = (0..3).map(|s| row(s, 70_000)).collect(); + let sampled = [5_u32, 69_999, 12]; + let got = from_chat_completion(&serve_body(&rows, 20, &sampled), DEFAULT_K).expect("capture"); + assert_eq!(got.len(), 3); + for ((g, r), &s) in got.iter().zip(&rows).zip(&sampled) { + let want = TokenLogits::from_full_logits(r, DEFAULT_K, s).expect("row"); + assert_eq!(g.token_id_sampled, s); + assert_eq!(g.ids, want.ids); + for (a, b) in g.logprobs.iter().zip(&want.logprobs) { + assert!((a.to_f32() - b.to_f32()).abs() <= 1e-2, "{a} vs {b}"); + } + assert_eq!(g.logsumexp_full, want.logsumexp_full); + assert!((g.topk_mass.to_f32() - want.topk_mass.to_f32()).abs() <= MASS_TOLERANCE); + } + let blob = encode(&header(DEFAULT_K, 3), &got).expect("encode"); + assert_eq!(decode(&blob).expect("decode").1, got); +} + +/// PRM C11: every body the encoder could not honestly record is refused by name. +#[test] +fn c11_serve_capture_refuses_what_it_cannot_record() { + let good = serve_body(&[row(1, 64)], 4, &[3]); + let entry = "/choices/0/logprobs/content/0"; + let cases: &[(&str, fn(&mut serde_json::Value), &str)] = &[ + ( + "no logprobs", + |b| *b = serde_json::json!({"choices": [{}]}), + "did not ask", + ), + ( + "hosted body", + |b| { + b.pointer_mut("/choices/0/logprobs/content/0") + .expect("e") + .as_object_mut() + .expect("o") + .remove("logsumexp_full"); + }, + "logsumexp_full", + ), + ( + "too few", + |b| { + b.pointer_mut("/choices/0/logprobs/content/0/top_logprobs") + .expect("t") + .as_array_mut() + .expect("a") + .truncate(2); + }, + "2 alternatives, k = 4", + ), + ( + "not descending", + |b| { + b.pointer_mut("/choices/0/logprobs/content/0/top_logprobs") + .expect("t") + .as_array_mut() + .expect("a") + .swap(0, 1); + }, + "descending", + ), + ( + "positive logprob", + |b| { + b["choices"][0]["logprobs"]["content"][0]["top_logprobs"][0]["logprob"] = + serde_json::json!(0.5); + }, + "≤ 0", + ), + ( + "no token_id", + |b| { + b.pointer_mut("/choices/0/logprobs/content/0") + .expect("e") + .as_object_mut() + .expect("o") + .remove("token_id"); + }, + "token_id", + ), + ]; + assert!( + from_chat_completion(&good, 4).is_ok(), + "control: the unplanted body captures" + ); + assert!(good.pointer(entry).is_some()); + for (name, plant, want) in cases { + let mut b = good.clone(); + plant(&mut b); + let err = from_chat_completion(&b, 4).expect_err(name); + assert!(err.contains(want), "{name}: {err}"); + } + assert!(from_chat_completion(&good, 0) + .expect_err("k=0") + .contains("k = 0")); +} + +/// A scratch CAS root, removed on drop. +struct Root(std::path::PathBuf); + +impl Root { + fn new(tag: &str) -> Self { + let p = std::env::temp_dir().join(format!("spl-{tag}-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&p); + Self(p) + } +} + +impl Drop for Root { + fn drop(&mut self) { + let _ = std::fs::remove_dir_all(&self.0); + } +} + +/// FALSIFY-SPL-005 (PRM C11): a serve body captured into the CAS reads back as +/// the same tokens, keeps its residual mass, and is addressed by the sha of the +/// raw blob; a second capture is idempotent, and a stored blob that no longer +/// hashes to its name is refused. +#[test] +fn falsify_spl_005_capture_round_trips_through_the_cas() { + let root = Root::new("005"); + let rows: Vec> = (0..4).map(|s| row(s, 70_000)).collect(); + let body = serve_body(&rows, 20, &[5, 69_999, 12, 7]); + let want = from_chat_completion(&body, DEFAULT_K).expect("control"); + let got = capture(&root.0, &header(DEFAULT_K, 0), &body).expect("capture"); + assert_eq!((got.n_tokens, got.k), (4, DEFAULT_K)); + let path = cas_path(&root.0, &got.logits_sha); + let stored = std::fs::read(&path).expect("stored"); + assert_eq!(stored.len() as u64, got.stored_bytes); + assert_eq!( + crate::datacard::zstd_content_size(&stored), + Some(got.raw_bytes), + "the frame declares its raw size for the data card" + ); + let raw = zstd::stream::decode_all(stored.as_slice()).expect("frame"); + assert_eq!(crate::corpus::sha256_hex(&raw), got.logits_sha); + let (h, toks) = decode(&raw).expect("decode"); + assert_eq!(h.n_tokens, 4); + assert_eq!(toks, want); + for (t, r) in toks.iter().zip(&rows) { + let full = TokenLogits::from_full_logits(r, DEFAULT_K, 0).expect("row"); + let tail = 1.0 - t.topk_mass.to_f32(); + assert!( + (tail - (1.0 - full.topk_mass.to_f32())).abs() <= 1e-3, + "residual mass" + ); + } + assert_eq!( + capture(&root.0, &header(DEFAULT_K, 0), &body).expect("again"), + got + ); + std::fs::write(&path, zstd::bulk::compress(b"forged", 3).expect("z")).expect("plant"); + let err = capture(&root.0, &header(DEFAULT_K, 0), &body).expect_err("forged store"); + assert!(err.contains("does not hash"), "{err}"); +} + +/// FALSIFY-SPL-005: a body the codec refuses writes nothing to the CAS. +#[test] +fn falsify_spl_005_a_refused_body_writes_nothing() { + let root = Root::new("005-refused"); + let mut body = serve_body(&[row(1, 64)], 4, &[3]); + body.pointer_mut("/choices/0/logprobs/content/0") + .expect("e") + .as_object_mut() + .expect("o") + .remove("logsumexp_full"); + assert!(capture(&root.0, &header(4, 0), &body).is_err()); + assert!( + !root.0.join("blobs").exists(), + "a refused capture left a blob" + ); +} diff --git a/crates/aprender-review-experiment/src/split_guard.rs b/crates/aprender-review-experiment/src/split_guard.rs new file mode 100644 index 0000000000..f4125af875 --- /dev/null +++ b/crates/aprender-review-experiment/src/split_guard.rs @@ -0,0 +1,222 @@ +//! PRM-C9 `trace-split-guard-v1`: the train/val/test split (gate G-DUP; +//! contract `trace-split-guard-v1`, FALSIFY-TSG-001..004). +//! +//! - **Unit**: `repo#PR`. Every row of a PR lands in one split. +//! - **Time-ordered**: a PR's split comes from its first-seen time against +//! the plan's `val_from` / `test_from` dates. +//! - **Embargo**: a PR first seen less than [`EMBARGO_DAYS`] before a +//! boundary is in no split, so nothing in a later split is within 14 d of +//! an earlier one. +//! - **Closure**: PRs that share a [`crate::dedup`] cluster are one component, +//! split by the component's earliest time: a copy of a train diff opened +//! later is train too, never test. +//! - **Sealed**: a component holding a near-dup of a sealed item is +//! [`Split::Sealed`], i.e. refused from every pool. +//! +//! [`audit`] recomputes nothing: it checks any assignment for PRs or +//! clusters that straddle splits. G-DUP requires both lists empty. + +use std::collections::{BTreeMap, BTreeSet, HashMap}; + +use serde::{Deserialize, Serialize}; + +use crate::dedup::{self, Item}; + +pub const SCHEME: &str = "trace-split-guard-v1"; + +/// Minimum gap between a PR's first-seen time and the next split's start. +pub const EMBARGO_DAYS: i64 = 14; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub enum Split { + Train, + Val, + Test, + /// Near-dup of a sealed item: in no pool. + Sealed, +} + +/// One trace diff to place. +#[derive(Debug, Clone, Copy)] +pub struct Unit<'a> { + /// `diff_sha256`. + pub id: &'a str, + pub repo: &'a str, + pub pr: u64, + /// RFC 3339 UTC. + pub at: &'a str, + pub diff: &'a str, +} + +/// Split boundaries, `YYYY-MM-DD`. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Plan { + pub val_from: String, + pub test_from: String, +} + +/// The `split_guard` a trace row carries. `split: None` is embargoed. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Guard { + pub group_key: String, + pub split: Option, + pub dedup_cluster: String, +} + +/// Days since 1970-01-01 of a `YYYY-MM-DD` prefix; `None` if it is not one. +#[must_use] +pub fn day(s: &str) -> Option { + let b = s.as_bytes(); + if b.len() < 10 || b[4] != b'-' || b[7] != b'-' { + return None; + } + let num = |r: std::ops::Range| s.get(r)?.parse::().ok(); + let (y, m, d) = (num(0..4)?, num(5..7)?, num(8..10)?); + if !(1..=12).contains(&m) || !(1..=31).contains(&d) { + return None; + } + // Howard Hinnant's days_from_civil. + let y = if m <= 2 { y - 1 } else { y }; + let era = y.div_euclid(400); + let yoe = y - era * 400; + let doy = (153 * (m + if m > 2 { -3 } else { 9 }) + 2) / 5 + d - 1; + let doe = yoe * 365 + yoe / 4 - yoe / 100 + doy; + Some(era * 146_097 + doe - 719_468) +} + +/// The `YYYY-MM-DD` of a [`day`] number (Hinnant's civil_from_days). +#[must_use] +pub fn date(days: i64) -> String { + let z = days + 719_468; + let era = z.div_euclid(146_097); + let doe = z - era * 146_097; + let yoe = (doe - doe / 1460 + doe / 36_524 - doe / 146_096) / 365; + let doy = doe - (365 * yoe + yoe / 4 - yoe / 100); + let mp = (5 * doy + 2) / 153; + let d = doy - (153 * mp + 2) / 5 + 1; + let m = if mp < 10 { mp + 3 } else { mp - 9 }; + let y = yoe + era * 400 + i64::from(m <= 2); + format!("{y:04}-{m:02}-{d:02}") +} + +/// Assign each unit its guard, in input order. A plan whose val window is +/// shorter than the embargo, or an unreadable date, is an error. +pub fn assign(units: &[Unit], sealed: &[(&str, &str)], plan: &Plan) -> Result, String> { + let (v, t) = ( + day(&plan.val_from).ok_or("val_from is not a date")?, + day(&plan.test_from).ok_or("test_from is not a date")?, + ); + if t - v <= EMBARGO_DAYS { + return Err(format!( + "val window {} d is not longer than the {EMBARGO_DAYS} d embargo", + t - v + )); + } + let days = units + .iter() + .map(|u| day(u.at).ok_or_else(|| format!("{}: at {:?} is not a date", u.id, u.at))) + .collect::, String>>()?; + // Sealed items sort before every unit (empty `at`), so they name their clusters. + let items: Vec = sealed + .iter() + .map(|&(id, diff)| Item { id, at: "", diff }) + .chain(units.iter().map(|u| Item { + id: u.id, + at: u.at, + diff: u.diff, + })) + .collect(); + let all = dedup::clusters(&items); + let (sealed_ids, clusters) = all.split_at(sealed.len()); + let sealed_ids: BTreeSet<&String> = sealed_ids.iter().collect(); + // Components: units joined by a shared PR or a shared cluster. + let keys: Vec = units + .iter() + .map(|u| format!("{}#{}", u.repo, u.pr)) + .collect(); + let mut parent: Vec = (0..units.len()).collect(); + let mut first: HashMap<&str, usize> = HashMap::new(); + for i in 0..units.len() { + for k in [keys[i].as_str(), clusters[i].as_str()] { + let j = *first.entry(k).or_insert(i); + let (ri, rj) = (root(&mut parent, i), root(&mut parent, j)); + parent[ri.max(rj)] = ri.min(rj); + } + } + let mut comp: BTreeMap = BTreeMap::new(); + for i in 0..units.len() { + let r = root(&mut parent, i); + let e = comp.entry(r).or_insert((days[i], false)); + e.0 = e.0.min(days[i]); + e.1 |= sealed_ids.contains(&clusters[i]); + } + Ok((0..units.len()) + .map(|i| { + let (d, is_sealed) = comp[&root(&mut parent, i)]; + let split = if is_sealed { + Some(Split::Sealed) + } else if d < v - EMBARGO_DAYS { + Some(Split::Train) + } else if d < v { + None + } else if d < t - EMBARGO_DAYS { + Some(Split::Val) + } else if d < t { + None + } else { + Some(Split::Test) + }; + Guard { + group_key: keys[i].clone(), + split, + dedup_cluster: clusters[i].clone(), + } + }) + .collect()) +} + +fn root(parent: &mut [usize], mut i: usize) -> usize { + while parent[i] != i { + parent[i] = parent[parent[i]]; + i = parent[i]; + } + i +} + +/// What straddles splits in an assignment. G-DUP holds when both are empty. +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize)] +pub struct Audit { + pub straddling_groups: Vec, + pub cross_split_clusters: Vec, +} + +/// Check any assignment. Embargoed members (`None`) are in no pool, so they +/// never make a straddle. +#[must_use] +pub fn audit(guards: &[Guard]) -> Audit { + let mut by_group: BTreeMap<&str, BTreeSet> = BTreeMap::new(); + let mut by_cluster: BTreeMap<&str, BTreeSet> = BTreeMap::new(); + for g in guards { + if let Some(s) = g.split { + by_group.entry(&g.group_key).or_default().insert(s); + by_cluster.entry(&g.dedup_cluster).or_default().insert(s); + } + } + let straddles = |m: BTreeMap<&str, BTreeSet>| -> Vec { + m.into_iter() + .filter(|(_, s)| s.len() > 1) + .map(|(k, _)| k.to_owned()) + .collect() + }; + Audit { + straddling_groups: straddles(by_group), + cross_split_clusters: straddles(by_cluster), + } +} + +#[cfg(test)] +#[path = "split_guard_tests.rs"] +mod tests; diff --git a/crates/aprender-review-experiment/src/split_guard_tests.rs b/crates/aprender-review-experiment/src/split_guard_tests.rs new file mode 100644 index 0000000000..98b98a048a --- /dev/null +++ b/crates/aprender-review-experiment/src/split_guard_tests.rs @@ -0,0 +1,144 @@ +use super::*; + +/// Segments `from..to` of a synthetic diff; each segment's literals are its own. +fn segs(from: u32, to: u32) -> String { + (from..to) + .flat_map(|k| { + (0..4).map(move |j| { + format!("+ let v = w.get({k}).map(|x| x * {k}{j}).unwrap_or({j});\n") + }) + }) + .collect() +} + +/// The same change re-indented and renamed. +fn perturb(d: &str) -> String { + d.replace(" ", " ").replace("w.get", "table.get") +} + +fn plan() -> Plan { + Plan { + val_from: "2026-09-01".into(), + test_from: "2026-10-01".into(), + } +} + +fn unit<'a>(id: &'a str, pr: u64, at: &'a str, diff: &'a str) -> Unit<'a> { + Unit { + id, + repo: "paiml/aprender", + pr, + at, + diff, + } +} + +fn splits(g: &[Guard]) -> Vec> { + g.iter().map(|g| g.split).collect() +} + +#[test] +fn falsify_tsg_001_a_pr_never_straddles_splits() { + let (a, b) = (segs(1, 6), segs(20, 25)); + // PR 1 opened in the train window; its second round lands in test time. + let u = [ + unit("a", 1, "2026-08-01T00:00:00Z", &a), + unit("b", 1, "2026-10-05T00:00:00Z", &b), + ]; + let g = assign(&u, &[], &plan()).expect("assigns"); + assert_eq!(splits(&g), [Some(Split::Train), Some(Split::Train)]); + assert_eq!(audit(&g), Audit::default()); + let mut bad = g.clone(); + bad[1].split = Some(Split::Test); + assert_eq!(audit(&bad).straddling_groups, ["paiml/aprender#1"]); +} + +#[test] +fn falsify_tsg_002_a_near_dup_cluster_closes_to_its_earliest_members_split() { + let (a, b, c) = (segs(1, 6), segs(20, 25), segs(40, 45)); + let (a2, b2) = (perturb(&a), perturb(&b)); + let u = [ + // A chain: PR 1 {a} ~ PR 2 {a', b} ~ PR 3 {b'}; PR 4 unrelated. + unit("b2", 3, "2026-10-09T00:00:00Z", &b2), + unit("a", 1, "2026-08-01T00:00:00Z", &a), + unit("a2", 2, "2026-10-05T00:00:00Z", &a2), + unit("b", 2, "2026-10-05T00:00:00Z", &b), + unit("c", 4, "2026-10-06T00:00:00Z", &c), + ]; + let g = assign(&u, &[], &plan()).expect("assigns"); + let train = Some(Split::Train); + assert_eq!(splits(&g), [train, train, train, train, Some(Split::Test)]); + assert_eq!(g[2].dedup_cluster, "a"); + assert_eq!(audit(&g), Audit::default()); + // A per-row time split leaks the copy into test, and the audit sees it. + let mut naive = g.clone(); + naive[2].split = Some(Split::Test); + naive[2].group_key = "paiml/aprender#9".into(); + assert_eq!(audit(&naive).cross_split_clusters, ["a"]); +} + +#[test] +fn falsify_tsg_003_the_embargo_is_at_least_14_days() { + let d: Vec = (0..5).map(|k| segs(10 * k + 1, 10 * k + 6)).collect(); + let u = [ + unit("t", 1, "2026-08-17T23:59:59Z", &d[0]), // 15 d before val: train + unit("e1", 2, "2026-08-18T00:00:00Z", &d[1]), // 14 d before val: embargoed + unit("v", 3, "2026-09-01T00:00:00Z", &d[2]), + unit("e2", 4, "2026-09-30T00:00:00Z", &d[3]), + unit("x", 5, "2026-10-01T00:00:00Z", &d[4]), + ]; + let g = assign(&u, &[], &plan()).expect("assigns"); + assert_eq!( + splits(&g), + [ + Some(Split::Train), + None, + Some(Split::Val), + None, + Some(Split::Test) + ] + ); + // An embargoed member never makes a straddle. + let mut one = g.clone(); + one[1].group_key = one[0].group_key.clone(); + assert_eq!(audit(&one), Audit::default()); + // A val window no longer than the embargo is refused, as is a bad date. + let short = Plan { + val_from: "2026-09-01".into(), + test_from: "2026-09-15".into(), + }; + assert!(assign(&u, &[], &short).is_err()); + let bad = [unit("t", 1, "yesterday", &d[0])]; + assert!(assign(&bad, &[], &plan()).is_err()); +} + +#[test] +fn falsify_tsg_004_a_perturbed_sealed_item_is_refused_with_its_whole_pr() { + let (s, b) = (segs(1, 6), segs(20, 25)); + let leak = perturb(&s); + let c = segs(40, 45); + let u = [ + unit("leak", 1, "2026-08-01T00:00:00Z", &leak), + unit("b", 1, "2026-08-02T00:00:00Z", &b), + unit("c", 2, "2026-08-03T00:00:00Z", &c), + ]; + let g = assign(&u, &[("sealed-7", &s)], &plan()).expect("assigns"); + let sealed = Some(Split::Sealed); + assert_eq!(splits(&g), [sealed, sealed, Some(Split::Train)]); + assert_eq!(g[0].dedup_cluster, "sealed-7"); +} + +#[test] +fn day_matches_the_civil_calendar() { + let cases = [ + ("1970-01-01", Some(0)), + ("2000-03-01", Some(11_017)), + ("2026-09-25T12:00:00Z", Some(20_721)), + ("2024-02-29", Some(19_782)), + ("2026-13-01", None), + ("26-09-25", None), + ]; + for (s, want) in cases { + assert_eq!(day(s), want, "{s}"); + } +} diff --git a/crates/aprender-review-experiment/src/stats.rs b/crates/aprender-review-experiment/src/stats.rs index 3068b1d4b0..4a56319b4b 100644 --- a/crates/aprender-review-experiment/src/stats.rs +++ b/crates/aprender-review-experiment/src/stats.rs @@ -1,4 +1,4 @@ -//! REX-001 frozen analysis code (spec §2.3, §3, §5.4). +//! PRM-001 (PROMETHEUS) frozen analysis code (spec v3 §2.7, §3, §5.3; was REX-001). //! //! This file is part of the pre-registration bundle (`prereg.rs`): its sha256 is //! folded into the prereg sha that every receipt carries. Changing a single byte @@ -13,6 +13,14 @@ //! resamples, SplitMix64 seeded (the generator is written out here so its //! stream can never change under a dependency bump). One-sided bootstrap p = //! fraction of resamples whose difference is < 0. +//! - v2 H4: the bootstrap statistic is `p(apr) − min(p(Haiku), p(agy))` with +//! the min computed inside each resample (`bootstrap_vs_min_voter`). The +//! decision is the Holm-adjusted one-sided p; the 95 % CI is report-only. +//! - v2 H5: recall on class R only; class-P recall and precision are +//! descriptive (mutant confound). +//! - v3 H7 (a gate outside the Holm family, δ-free): Cohen's κ on per-item +//! errors (`error_kappa`); holds iff κ(apr, each voter) ≤ the max voter–voter +//! κ on the same items (`h7_holds`). //! - Holm step-down over the family H1–H6 at overall alpha = 0.05. The count //! and threshold tests (H1, H2, H6) enter the family as p = 0 when rejected //! and p = 1 when not: they are deterministic rules, not sampling statistics. @@ -194,6 +202,122 @@ pub fn paired_bootstrap_diff( }) } +/// One H4 item scored by apr and the two existing voters (Haiku, agy). +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct VsVoters { + /// Ground truth: the item carries a defect (class P or R). + pub defect: bool, + /// apr said FAIL. + pub apr_fail: bool, + /// The two voting lanes said FAIL, in a fixed order (Haiku, agy). + pub voter_fail: [bool; 2], +} + +fn rate_of(items: &[VsVoters], idx: &[usize], lane: Option, which: Rate) -> Option { + let pick = |it: &VsVoters| lane.map_or(it.apr_fail, |v| it.voter_fail[v]); + let (mut num, mut den) = (0u64, 0u64); + for &i in idx { + let it = &items[i]; + let fail = pick(it); + match which { + Rate::Precision if fail => { + den += 1; + num += u64::from(it.defect); + } + Rate::Recall if it.defect => { + den += 1; + num += u64::from(fail); + } + _ => {} + } + } + (den > 0).then(|| num as f64 / den as f64) +} + +/// `rate(apr) − min(rate(Haiku), rate(agy))` on one index set. +fn diff_vs_min(items: &[VsVoters], idx: &[usize], which: Rate) -> Option { + let a = rate_of(items, idx, None, which)?; + let lo = rate_of(items, idx, Some(0), which)?.min(rate_of(items, idx, Some(1), which)?); + Some(a - lo) +} + +/// H4 (spec v2 §3): paired bootstrap of `p(apr) − min(p(Haiku), p(agy))`. +/// The min is taken INSIDE each resample, so the weaker voter is chosen on +/// that resample and the selection is part of the sampling distribution. The +/// decision is the Holm-adjusted `p_below_zero`; the CI is report-only. +#[must_use] +pub fn bootstrap_vs_min_voter( + items: &[VsVoters], + which: Rate, + resamples: usize, + seed: u64, + alpha: f64, +) -> Option { + let all: Vec = (0..items.len()).collect(); + let estimate = diff_vs_min(items, &all, which)?; + let mut rng = SplitMix64::new(seed); + let mut diffs = Vec::with_capacity(resamples); + let mut undefined = 0usize; + let mut idx = vec![0usize; items.len()]; + for _ in 0..resamples { + for slot in &mut idx { + *slot = rng.below(items.len()); + } + match diff_vs_min(items, &idx, which) { + Some(d) => diffs.push(d), + None => undefined += 1, + } + } + if diffs.is_empty() { + return None; + } + diffs.sort_by(f64::total_cmp); + let below = diffs.iter().filter(|d| **d < 0.0).count(); + Some(BootstrapDiff { + estimate, + ci: ( + percentile_sorted(&diffs, alpha / 2.0), + percentile_sorted(&diffs, 1.0 - alpha / 2.0), + ), + p_below_zero: below as f64 / diffs.len() as f64, + undefined, + }) +} + +/// Cohen's κ between two lanes' per-item ERROR indicators (PRM-001 v3 §2.7, +/// H7): 1 = they miss the same items, 0 = overlap at chance. +/// `None` when the lengths differ, the sample is empty, or chance agreement +/// is 1 (both lanes constant), where κ is undefined. +#[must_use] +pub fn error_kappa(a_err: &[bool], b_err: &[bool]) -> Option { + if a_err.len() != b_err.len() || a_err.is_empty() { + return None; + } + let n = a_err.len() as f64; + let agree = a_err.iter().zip(b_err).filter(|(x, y)| x == y).count() as f64 / n; + let pa = a_err.iter().filter(|x| **x).count() as f64 / n; + let pb = b_err.iter().filter(|x| **x).count() as f64 / n; + let chance = pa * pb + (1.0 - pa) * (1.0 - pb); + (chance < 1.0).then(|| (agree - chance) / (1.0 - chance)) +} + +/// H7 independence gate (PRM-001 v3 §3; outside the Holm family, δ-free): +/// the local lane's κ_err with EVERY voter must not exceed the largest κ_err +/// among voter–voter pairs on the same items — a tie-breaker may not be more +/// correlated with a voter than the voters already are with each other. +/// `None` (undecided, never "holds") when either side is empty or any κ is +/// undefined. +#[must_use] +pub fn h7_holds(qwen_vs_voter: &[Option], voter_vs_voter: &[Option]) -> Option { + if qwen_vs_voter.is_empty() || voter_vs_voter.is_empty() { + return None; + } + let pairs: Option> = voter_vs_voter.iter().copied().collect(); + let ceiling = pairs?.into_iter().fold(f64::NEG_INFINITY, f64::max); + let qwen: Option> = qwen_vs_voter.iter().copied().collect(); + Some(qwen?.into_iter().all(|k| k <= ceiling)) +} + /// Nearest-rank percentile of an ascending slice. `q` in (0, 1]. #[must_use] pub fn percentile_sorted(sorted: &[f64], q: f64) -> f64 { @@ -387,6 +511,100 @@ mod tests { assert!(close(p.estimate, 0.0, 0.0) && close(p.ci.0, 0.0, 0.0)); } + /// Three lanes, equal point precision 10/12, false positives on + /// disjoint good items: which voter is weaker flips between resamples. + fn tied_voters() -> Vec { + (0..40) + .map(|i| { + let defect = i < 20; + let caught = i < 10; + VsVoters { + defect, + apr_fail: caught || i == 20 || i == 21, + voter_fail: [caught || i == 22 || i == 23, caught || i == 24 || i == 25], + } + }) + .collect() + } + + #[test] + fn falsify_h4_min_is_taken_inside_each_resample() { + let items = tied_voters(); + let inside = + bootstrap_vs_min_voter(&items, Rate::Precision, 2000, 4354, ALPHA).expect("defined"); + // The v1 rule: pick the weaker voter ONCE on the full sample (a tie → + // voter 0), then bootstrap against it. Same seed → same resamples. + let fixed: Vec = items + .iter() + .map(|t| Paired { + defect: t.defect, + a_fail: t.apr_fail, + b_fail: t.voter_fail[0], + }) + .collect(); + let once = + paired_bootstrap_diff(&fixed, Rate::Precision, 2000, 4354, ALPHA).expect("defined"); + assert!(close(inside.estimate, 0.0, 1e-12) && close(once.estimate, 0.0, 1e-12)); + // a − min(v0, v1) ≥ a − v0 on every resample, strictly on those where + // v1 is the weaker: the two rules must give different p. + assert!( + inside.p_below_zero < once.p_below_zero, + "{inside:?} vs {once:?}" + ); + // Voter order is irrelevant under min(). + let swapped: Vec = items + .iter() + .map(|t| VsVoters { + voter_fail: [t.voter_fail[1], t.voter_fail[0]], + ..*t + }) + .collect(); + assert_eq!( + bootstrap_vs_min_voter(&swapped, Rate::Precision, 2000, 4354, ALPHA), + Some(inside) + ); + } + + #[test] + fn error_kappa_matches_hand_computed() { + // agree 3/4, pa = .5, pb = .25, chance = .5 → κ = .5 + let k = error_kappa(&[true, true, false, false], &[true, false, false, false]); + assert!(close(k.expect("defined"), 0.5, 1e-12)); + assert_eq!(error_kappa(&[true, false], &[true, false]), Some(1.0)); + assert_eq!( + error_kappa(&[false, false], &[false, false]), + None, + "chance = 1" + ); + assert_eq!( + error_kappa(&[true], &[true, false]), + None, + "length mismatch" + ); + assert_eq!(error_kappa(&[], &[]), None); + } + + #[test] + fn falsify_h7_is_max_pair_and_undecided_on_gaps() { + let pairs = [Some(0.30), Some(0.10), Some(0.20)]; + assert_eq!( + h7_holds(&[Some(0.30), Some(0.05)], &pairs), + Some(true), + "≤ max pair" + ); + assert_eq!( + h7_holds(&[Some(0.31), Some(0.05)], &pairs), + Some(false), + "one voter above max" + ); + // Not the min, not the mean: 0.25 is above both and still holds. + assert_eq!(h7_holds(&[Some(0.25)], &pairs), Some(true)); + assert_eq!(h7_holds(&[None, Some(0.0)], &pairs), None); + assert_eq!(h7_holds(&[Some(0.0)], &[Some(0.3), None]), None); + assert_eq!(h7_holds(&[], &pairs), None); + assert_eq!(h7_holds(&[Some(0.0)], &[]), None); + } + #[test] fn interference_rule() { let alone = [10.0, 10.2, 9.8, 10.1, 9.9]; diff --git a/crates/aprender-review-experiment/src/terms.rs b/crates/aprender-review-experiment/src/terms.rs new file mode 100644 index 0000000000..376a71404f --- /dev/null +++ b/crates/aprender-review-experiment/src/terms.rs @@ -0,0 +1,360 @@ +//! Provider-terms tagging for `agent-trace-v1` rows (PRA-001 T12, spec §2.10; +//! contract `agent-trace-terms-v1`). +//! +//! Every captured row carries `provider`, an exact `model_id`, `access_channel` +//! and a `terms_ref` snapshot, so any pool can be rebuilt without a given +//! provider or channel in one call (P-TERMS). This records which terms governed +//! a row; it is not legal advice, and the operator's S-6 ruling governs use. +//! +//! A row whose tags are missing or inconsistent is REFUSED by every rebuild, even +//! one that excludes nothing: an untagged row cannot be shown not to come from an +//! excluded source. + +use serde_json::Value; + +/// Who served the lane (`provider` in §2.2). +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Provider { + Anthropic, + Google, + Local, +} + +impl Provider { + pub const ALL: [Provider; 3] = [Provider::Anthropic, Provider::Google, Provider::Local]; + + #[must_use] + pub fn as_str(self) -> &'static str { + match self { + Provider::Anthropic => "anthropic", + Provider::Google => "google", + Provider::Local => "local", + } + } + + /// The exact wire spelling; anything else is unknown. + #[must_use] + pub fn parse(s: &str) -> Option { + Self::ALL.into_iter().find(|p| p.as_str() == s) + } +} + +/// How the lane was reached (`access_channel` in §2.2); terms differ by channel. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Channel { + AnthropicApi, + ClaudeCodeCli, + Antigravity, + GeminiApi, + Vertex, + AprServe, +} + +impl Channel { + pub const ALL: [Channel; 6] = [ + Channel::AnthropicApi, + Channel::ClaudeCodeCli, + Channel::Antigravity, + Channel::GeminiApi, + Channel::Vertex, + Channel::AprServe, + ]; + + #[must_use] + pub fn as_str(self) -> &'static str { + match self { + Channel::AnthropicApi => "anthropic-api", + Channel::ClaudeCodeCli => "claude-code-cli", + Channel::Antigravity => "antigravity", + Channel::GeminiApi => "gemini-api", + Channel::Vertex => "vertex", + Channel::AprServe => "apr-serve", + } + } + + /// The exact wire spelling; anything else is unknown. + #[must_use] + pub fn parse(s: &str) -> Option { + Self::ALL.into_iter().find(|c| c.as_str() == s) + } + + /// The only provider this channel can serve. + #[must_use] + pub fn provider(self) -> Provider { + match self { + Channel::AnthropicApi | Channel::ClaudeCodeCli => Provider::Anthropic, + Channel::Antigravity | Channel::GeminiApi | Channel::Vertex => Provider::Google, + Channel::AprServe => Provider::Local, + } + } +} + +/// The governing-terms snapshot taken at capture. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct TermsRef { + pub url: String, + pub effective_date: String, + pub fetched_at: String, +} + +/// The four terms fields of one row, validated. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Tags { + pub provider: Provider, + pub model_id: String, + pub channel: Channel, + pub terms: TermsRef, +} + +/// One reason a row's terms tags do not hold. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum Gap { + /// The line is not a JSON object. + NotJson, + /// A required field is absent, null or not a string/object. + Missing(&'static str), + UnknownProvider(String), + UnknownChannel(String), + /// The channel cannot serve the stated provider. + ChannelProvider { + channel: Channel, + provider: Provider, + }, + /// A family alias or placeholder, not an exact model id. + AliasModel(String), + /// A local model is named by its weights sha256 and nothing else. + LocalModelNotSha(String), + TermsUrl(String), + TermsEffectiveDate(String), + TermsFetchedAt(String), +} + +/// The validated tags of `row`, or every gap found. +/// +/// # Errors +/// Returns the gaps when any tag is missing, unknown or inconsistent. +pub fn tags(row: &Value) -> Result> { + let mut gaps = Vec::new(); + let text = |key: &'static str, gaps: &mut Vec| { + let s = row.get(key).and_then(Value::as_str); + if s.is_none() { + gaps.push(Gap::Missing(key)); + } + s + }; + let provider = text("provider", &mut gaps).and_then(|s| { + Provider::parse(s).or_else(|| { + gaps.push(Gap::UnknownProvider(s.to_string())); + None + }) + }); + let channel = text("access_channel", &mut gaps).and_then(|s| { + Channel::parse(s).or_else(|| { + gaps.push(Gap::UnknownChannel(s.to_string())); + None + }) + }); + let model_id = text("model_id", &mut gaps); + if let (Some(p), Some(c)) = (provider, channel) { + if c.provider() != p { + gaps.push(Gap::ChannelProvider { + channel: c, + provider: p, + }); + } + } + if let Some(m) = model_id { + if let Some(gap) = model_gap(m, provider) { + gaps.push(gap); + } + } + let terms = terms_ref(row.get("terms_ref"), &mut gaps); + match (provider, model_id, channel, terms) { + (Some(provider), Some(m), Some(channel), Some(terms)) if gaps.is_empty() => Ok(Tags { + provider, + model_id: m.to_string(), + channel, + terms, + }), + _ => Err(gaps), + } +} + +/// An exact id carries a version (a digit); a local id is a weights sha256. +fn model_gap(m: &str, provider: Option) -> Option { + let placeholder = m.is_empty() || m.eq_ignore_ascii_case("unknown"); + if provider == Some(Provider::Local) && !placeholder { + let hex = m.strip_prefix("sha256:").unwrap_or(m); + let is_sha = hex.len() == 64 && hex.bytes().all(|b| matches!(b, b'0'..=b'9' | b'a'..=b'f')); + return (!is_sha).then(|| Gap::LocalModelNotSha(m.to_string())); + } + let versioned = m.bytes().any(|b| b.is_ascii_digit()) && !m.contains(char::is_whitespace); + (placeholder || !versioned).then(|| Gap::AliasModel(m.to_string())) +} + +fn terms_ref(v: Option<&Value>, gaps: &mut Vec) -> Option { + let Some(obj) = v.and_then(Value::as_object) else { + gaps.push(Gap::Missing("terms_ref")); + return None; + }; + let field = |key: &'static str, gaps: &mut Vec| { + let s = obj.get(key).and_then(Value::as_str).map(str::to_string); + if s.is_none() { + gaps.push(Gap::Missing(key)); + } + s + }; + let url = field("url", gaps); + let effective_date = field("effective_date", gaps); + let fetched_at = field("fetched_at", gaps); + let before = gaps.len(); + if let Some(u) = &url { + if !u.starts_with("https://") || u.len() == "https://".len() { + gaps.push(Gap::TermsUrl(u.clone())); + } + } + if let Some(d) = &effective_date { + if !is_date(d) { + gaps.push(Gap::TermsEffectiveDate(d.clone())); + } + } + if let Some(t) = &fetched_at { + if !is_utc_timestamp(t) { + gaps.push(Gap::TermsFetchedAt(t.clone())); + } + } + Some(TermsRef { + url: url?, + effective_date: effective_date?, + fetched_at: fetched_at?, + }) + .filter(|_| gaps.len() == before) +} + +/// `YYYY-MM-DD` with a month 01-12 and a day 01-31. +fn is_date(s: &str) -> bool { + let b = s.as_bytes(); + let digits = |r: std::ops::Range| { + b[r.clone()] + .iter() + .all(u8::is_ascii_digit) + .then(|| num(&b[r])) + }; + if b.len() != 10 || b[4] != b'-' || b[7] != b'-' { + return false; + } + matches!( + (digits(0..4), digits(5..7), digits(8..10)), + (Some(_), Some(1..=12), Some(1..=31)) + ) +} + +/// RFC 3339 in UTC: `YYYY-MM-DDTHH:MM:SS[.frac]Z`. +fn is_utc_timestamp(s: &str) -> bool { + let Some(rest) = s.strip_suffix('Z') else { + return false; + }; + let (date, time) = match rest.split_once('T') { + Some(p) => p, + None => return false, + }; + let (hms, frac) = time.split_once('.').unwrap_or((time, "0")); + let t = hms.as_bytes(); + let two = |i: usize| { + t[i..i + 2] + .iter() + .all(u8::is_ascii_digit) + .then(|| num(&t[i..i + 2])) + }; + is_date(date) + && t.len() == 8 + && t[2] == b':' + && t[5] == b':' + && matches!( + (two(0), two(3), two(6)), + (Some(0..=23), Some(0..=59), Some(0..=60)) + ) + && !frac.is_empty() + && frac.bytes().all(|c| c.is_ascii_digit()) +} + +fn num(digits: &[u8]) -> u32 { + digits.iter().fold(0, |n, d| n * 10 + u32::from(d - b'0')) +} + +/// What a pool rebuild leaves out. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct Exclude { + pub providers: Vec, + pub channels: Vec, +} + +impl Exclude { + /// The public-release default (§2.11): gold-and-local only, so every hosted + /// provider is excluded unless the operator overrides it at a §8 STOP. + #[must_use] + pub fn public_release() -> Self { + Self { + providers: vec![Provider::Anthropic, Provider::Google], + channels: Vec::new(), + } + } + + /// Parse `--exclude-provider

` / `--exclude-channel `, each repeatable. + /// + /// # Errors + /// An unknown flag, a missing value, or an unknown provider or channel. + pub fn from_args(args: &[&str]) -> Result { + let mut ex = Self::default(); + let mut it = args.iter(); + while let Some(&flag) = it.next() { + let value = it.next().ok_or_else(|| format!("{flag}: missing value"))?; + match flag { + "--exclude-provider" => ex.providers.push( + Provider::parse(value).ok_or_else(|| format!("unknown provider: {value}"))?, + ), + "--exclude-channel" => ex.channels.push( + Channel::parse(value).ok_or_else(|| format!("unknown channel: {value}"))?, + ), + other => return Err(format!("unknown argument: {other}")), + } + } + Ok(ex) + } + + fn excludes(&self, t: &Tags) -> bool { + self.providers.contains(&t.provider) || self.channels.contains(&t.channel) + } +} + +/// A rebuilt pool: rows kept, rows left out by the filter, rows refused for gaps. +#[derive(Debug, Default)] +pub struct Rebuild<'a> { + pub kept: Vec<&'a str>, + pub excluded: Vec<&'a str>, + pub refused: Vec<(&'a str, Vec)>, +} + +/// Rebuild a pool from a JSONL batch, leaving out every excluded source. +/// Rows are passed through byte-for-byte, in input order. +#[must_use] +pub fn rebuild<'a>(rows: &'a str, ex: &Exclude) -> Rebuild<'a> { + let mut r = Rebuild::default(); + for line in rows.lines().filter(|l| !l.trim().is_empty()) { + let checked = serde_json::from_str::(line) + .ok() + .filter(Value::is_object) + .ok_or_else(|| vec![Gap::NotJson]) + .and_then(|v| tags(&v)); + match checked { + Ok(t) if ex.excludes(&t) => r.excluded.push(line), + Ok(_) => r.kept.push(line), + Err(gaps) => r.refused.push((line, gaps)), + } + } + r +} + +#[cfg(test)] +#[path = "terms_tests.rs"] +mod tests; diff --git a/crates/aprender-review-experiment/src/terms_tests.rs b/crates/aprender-review-experiment/src/terms_tests.rs new file mode 100644 index 0000000000..e338fc2bca --- /dev/null +++ b/crates/aprender-review-experiment/src/terms_tests.rs @@ -0,0 +1,352 @@ +use super::*; + +const TERMS: &str = r#"{"url":"https://www.anthropic.com/legal/commercial-terms","effective_date":"2025-06-17","fetched_at":"2026-09-25T17:00:00Z"}"#; +const LOCAL_SHA: &str = "3f1a9c0b5e2d4f6a8b7c9d0e1f2a3b4c5d6e7f8091a2b3c4d5e6f708192a3b4c"; + +fn row(id: &str, provider: &str, model: &str, channel: &str, terms: &str) -> String { + format!( + r#"{{"schema":"agent-trace-v1","trace_id":"{id}","provider":"{provider}","model_id":"{model}","access_channel":"{channel}","terms_ref":{terms}}}"# + ) +} + +fn terms(url: &str, effective: &str, fetched: &str) -> String { + format!(r#"{{"url":"{url}","effective_date":"{effective}","fetched_at":"{fetched}"}}"#) +} + +/// One row per channel, all correctly tagged. +fn mixed_batch() -> String { + [ + row( + "t1", + "anthropic", + "claude-sonnet-5", + "claude-code-cli", + TERMS, + ), + row( + "t2", + "anthropic", + "claude-haiku-4-5-20251001", + "anthropic-api", + TERMS, + ), + row("t3", "google", "gemini-3.1-pro-high", "antigravity", TERMS), + row("t4", "google", "gemini-3.1-pro", "gemini-api", TERMS), + row("t5", "google", "gemini-3.1-pro", "vertex", TERMS), + row("t6", "local", LOCAL_SHA, "apr-serve", TERMS), + ] + .join("\n") +} + +fn ids(rows: &[&str]) -> Vec { + rows.iter() + .map(|r| { + let v: Value = serde_json::from_str(r).expect("row is JSON"); + v["trace_id"].as_str().expect("trace_id").to_string() + }) + .collect() +} + +#[test] +fn falsify_terms_001_pool_rebuild_excluding_a_provider() { + let batch = mixed_batch(); + let ex = Exclude::from_args(&["--exclude-provider", "google"]).expect("valid args"); + let r = rebuild(&batch, &ex); + assert!(r.refused.is_empty(), "every row is tagged: {:?}", r.refused); + assert_eq!(ids(&r.kept), ["t1", "t2", "t6"]); + assert_eq!(ids(&r.excluded), ["t3", "t4", "t5"]); + for kept in &r.kept { + let t = tags(&serde_json::from_str(kept).expect("json")).expect("tagged"); + assert_ne!( + t.provider, + Provider::Google, + "a google row survived: {kept}" + ); + assert!( + batch.lines().any(|l| l == *kept), + "kept row rewritten: {kept}" + ); + } +} + +#[test] +fn falsify_terms_002_channel_exclusion_is_narrower_than_provider() { + let batch = mixed_batch(); + let ex = Exclude::from_args(&["--exclude-channel", "antigravity"]).expect("valid args"); + let r = rebuild(&batch, &ex); + assert_eq!(ids(&r.excluded), ["t3"]); + assert_eq!(ids(&r.kept), ["t1", "t2", "t4", "t5", "t6"]); +} + +#[test] +fn falsify_terms_003_identity_case_table() { + let https = "https://x.example/terms"; + let at = "2026-09-25T17:00:00Z"; + let prefixed = format!("sha256:{LOCAL_SHA}"); + let upper = LOCAL_SHA.to_uppercase(); + // (trace_id, provider, model_id, channel, terms_ref, must be tagged?) + let cases: Vec<(&str, &str, &str, &str, String, bool)> = vec![ + ( + "ok-a", + "anthropic", + "claude-opus-5-5", + "claude-code-cli", + TERMS.into(), + true, + ), + ( + "ok-g", + "google", + "gemini-3.1-pro-high", + "antigravity", + TERMS.into(), + true, + ), + ("ok-l", "local", LOCAL_SHA, "apr-serve", TERMS.into(), true), + ("ok-l2", "local", &prefixed, "apr-serve", TERMS.into(), true), + ( + "alias", + "anthropic", + "claude-sonnet", + "claude-code-cli", + TERMS.into(), + false, + ), + ( + "bare", + "anthropic", + "sonnet", + "claude-code-cli", + TERMS.into(), + false, + ), + ( + "gem-alias", + "google", + "gemini-pro", + "gemini-api", + TERMS.into(), + false, + ), + ( + "unk-model", + "anthropic", + "unknown", + "claude-code-cli", + TERMS.into(), + false, + ), + ( + "empty-model", + "anthropic", + "", + "claude-code-cli", + TERMS.into(), + false, + ), + ( + "unk-prov", + "openai", + "gpt-5", + "anthropic-api", + TERMS.into(), + false, + ), + ( + "unk-chan", + "anthropic", + "claude-opus-5-5", + "web", + TERMS.into(), + false, + ), + ( + "mismatch", + "google", + "gemini-3.1-pro", + "claude-code-cli", + TERMS.into(), + false, + ), + ( + "local-alias", + "local", + "qwen3.5-4b", + "apr-serve", + TERMS.into(), + false, + ), + ( + "local-short", + "local", + "sha256:3f1a9c", + "apr-serve", + TERMS.into(), + false, + ), + ( + "local-upper", + "local", + &upper, + "apr-serve", + TERMS.into(), + false, + ), + ( + "no-terms", + "anthropic", + "claude-opus-5-5", + "claude-code-cli", + "null".into(), + false, + ), + ( + "http", + "anthropic", + "claude-opus-5-5", + "claude-code-cli", + terms("http://x.example/terms", "2025-06-17", at), + false, + ), + ( + "bad-date", + "anthropic", + "claude-opus-5-5", + "claude-code-cli", + terms(https, "June 17", at), + false, + ), + ( + "bad-month", + "anthropic", + "claude-opus-5-5", + "claude-code-cli", + terms(https, "2025-13-17", at), + false, + ), + ( + "bad-fetch", + "anthropic", + "claude-opus-5-5", + "claude-code-cli", + terms(https, "2025-06-17", "yesterday"), + false, + ), + ( + "fetch-no-zone", + "anthropic", + "claude-opus-5-5", + "claude-code-cli", + terms(https, "2025-06-17", "2026-09-25T17:00:00"), + false, + ), + // Terms announced ahead of their effective date are still a valid snapshot. + ( + "fetch-before-effective", + "anthropic", + "claude-opus-5-5", + "claude-code-cli", + terms(https, "2026-10-01", at), + true, + ), + ]; + for (id, p, m, c, t, want) in &cases { + let v: Value = serde_json::from_str(&row(id, p, m, c, t)).expect("case is JSON"); + let got = tags(&v); + assert_eq!(got.is_ok(), *want, "case {id} -> {got:?}"); + if let Err(gaps) = got { + assert!(!gaps.is_empty(), "case {id}: a refusal names its gaps"); + } + } + let missing: Value = serde_json::from_str( + r#"{"schema":"agent-trace-v1","trace_id":"missing","model_id":"claude-opus-5-5","access_channel":"claude-code-cli"}"#, + ) + .expect("json"); + let gaps = tags(&missing).expect_err("no provider, no terms_ref"); + assert!(gaps.contains(&Gap::Missing("provider")), "{gaps:?}"); + assert!(gaps.contains(&Gap::Missing("terms_ref")), "{gaps:?}"); +} + +#[test] +fn falsify_terms_004_untagged_row_is_refused_never_kept() { + let batch = format!( + "{}\n{}\n\n", + row( + "good", + "anthropic", + "claude-opus-5-5", + "claude-code-cli", + TERMS + ), + row("alias", "google", "gemini-pro", "gemini-api", TERMS), + ); + // Excluding anthropic must not let an untagged row into the pool. + let ex = Exclude::from_args(&["--exclude-provider", "anthropic"]).expect("valid"); + let r = rebuild(&batch, &ex); + assert!(r.kept.is_empty(), "kept: {:?}", r.kept); + assert_eq!(ids(&r.excluded), ["good"]); + assert_eq!(r.refused.len(), 1); + assert!(!r.refused[0].1.is_empty(), "a refusal names its gaps"); + // With nothing excluded the untagged row is still refused. + let r = rebuild(&batch, &Exclude::default()); + assert_eq!(ids(&r.kept), ["good"]); + assert_eq!(r.refused.len(), 1); + // A line that is not JSON is refused too, never dropped silently. + let r = rebuild("not json\n", &Exclude::default()); + assert!(r.kept.is_empty()); + assert_eq!(r.refused.len(), 1); +} + +#[test] +fn exclude_args_reject_unknown_values_and_flags() { + assert!(Exclude::from_args(&["--exclude-provider", "openai"]).is_err()); + assert!(Exclude::from_args(&["--exclude-channel", "web"]).is_err()); + assert!(Exclude::from_args(&["--exclude-provider"]).is_err()); + assert!(Exclude::from_args(&["--include-provider", "google"]).is_err()); + let ex = Exclude::from_args(&[ + "--exclude-provider", + "anthropic", + "--exclude-channel", + "vertex", + "--exclude-provider", + "google", + ]) + .expect("valid"); + assert_eq!(ex.providers, [Provider::Anthropic, Provider::Google]); + assert_eq!(ex.channels, [Channel::Vertex]); + assert_eq!(Exclude::from_args(&[]).expect("empty"), Exclude::default()); +} + +#[test] +fn public_release_keeps_only_local_rows() { + let batch = mixed_batch(); + let r = rebuild(&batch, &Exclude::public_release()); + assert_eq!(ids(&r.kept), ["t6"]); + assert_eq!(r.excluded.len(), 5); +} + +#[test] +fn every_channel_maps_to_its_provider() { + let table = [ + ("anthropic-api", Provider::Anthropic), + ("claude-code-cli", Provider::Anthropic), + ("antigravity", Provider::Google), + ("gemini-api", Provider::Google), + ("vertex", Provider::Google), + ("apr-serve", Provider::Local), + ]; + for (name, p) in table { + let c = Channel::parse(name).expect("known channel"); + assert_eq!(c.provider(), p, "{name}"); + assert_eq!(c.as_str(), name); + } + assert_eq!(table.len(), Channel::ALL.len()); + for (name, p) in [ + ("anthropic", Provider::Anthropic), + ("google", Provider::Google), + ("local", Provider::Local), + ] { + assert_eq!(Provider::parse(name), Some(p)); + } + assert_eq!(Provider::parse("Anthropic"), None); +} diff --git a/crates/aprender-review-experiment/src/workload.rs b/crates/aprender-review-experiment/src/workload.rs new file mode 100644 index 0000000000..8ed4244b5e --- /dev/null +++ b/crates/aprender-review-experiment/src/workload.rs @@ -0,0 +1,200 @@ +//! Review workload profile (PRM-001 v3 PRM-S2, §0.2 and §2.4; contract +//! `workload-profile-v1`, FALSIFY-WLP-001..004). +//! +//! The profile reads `agent-trace-v1` rows and reports, per lane, the +//! distribution of `tokens.input` and `tokens.output`. It decides §0.2's claim +//! that review is prefill-heavy (2k–32k tokens in, a short verdict out) and +//! sets the replay strata weights over distinct diffs. + +use std::collections::BTreeMap; + +use serde_json::{json, Value}; + +use crate::stats::percentile; + +/// §2.4 replay strata upper bounds, in input tokens. +pub const STRATA: [u64; 4] = [2000, 8000, 16000, 32000]; + +/// Per-lane token quantiles over the measured rows. +#[derive(Debug, Clone, PartialEq)] +pub struct LaneProfile { + pub lane: String, + pub n: usize, + pub input_p50: f64, + pub input_p95: f64, + pub output_p50: f64, + pub output_p95: f64, +} + +impl LaneProfile { + /// §0.2 "a diff of 2k–32k tokens goes in". + #[must_use] + pub fn in_band(&self) -> bool { + self.input_p50 >= STRATA[0] as f64 && self.input_p95 <= STRATA[3] as f64 + } + + /// §0.2 "prefill-heavy": the long output is shorter than the typical input. + #[must_use] + pub fn prefill_heavy(&self) -> bool { + self.output_p95 < self.input_p50 + } +} + +/// The profile of one set of trace rows. +#[derive(Debug, Clone, PartialEq)] +pub struct Profile { + pub lanes: Vec, + /// Rows that could not be measured (capture failed, unparsed, no tokens). + pub excluded: usize, + /// Distinct `diff_sha256` work units with a measurement. + pub units: usize, + /// Units over the top stratum; not weighted. + pub over_32k: usize, + /// Weight per stratum bound, over the in-range units. + pub strata: Vec<(u64, f64)>, + /// §0.2: `None` when no lane reached `min_n`. + pub claim: Option, + /// Why the claim is not confirmed (S-code lines), empty when it is. + pub refusals: Vec, +} + +/// Build the profile of JSONL `rows`; lanes with fewer than `min_n` measured +/// rows are reported but do not decide the claim. +#[must_use] +pub fn profile(rows: &str, min_n: usize) -> Profile { + let mut by_lane: BTreeMap, Vec)> = BTreeMap::new(); + let mut size: BTreeMap = BTreeMap::new(); + let mut excluded = 0; + for line in rows.lines().filter(|l| !l.trim().is_empty()) { + let Some((lane, diff, input, output)) = measured(line) else { + excluded += 1; + continue; + }; + let e = by_lane.entry(lane).or_default(); + e.0.push(input as f64); + e.1.push(output as f64); + let s = size.entry(diff).or_insert(0); + *s = (*s).max(input); + } + let lanes: Vec = by_lane + .into_iter() + .map(|(lane, (i, o))| LaneProfile { + lane, + n: i.len(), + input_p50: percentile(&i, 0.5), + input_p95: percentile(&i, 0.95), + output_p50: percentile(&o, 0.5), + output_p95: percentile(&o, 0.95), + }) + .collect(); + let (claim, refusals) = decide(&lanes, min_n); + let over_32k = size.values().filter(|&&s| s > STRATA[3]).count(); + let in_range = size.len() - over_32k; + let strata = STRATA + .iter() + .enumerate() + .map(|(k, &b)| { + let lo = if k == 0 { 0 } else { STRATA[k - 1] }; + let c = size.values().filter(|&&s| s > lo && s <= b).count(); + let w = if in_range == 0 { + 0.0 + } else { + c as f64 / in_range as f64 + }; + (b, w) + }) + .collect(); + Profile { + lanes, + excluded, + units: size.len(), + over_32k, + strata, + claim, + refusals, + } +} + +/// `(lane, diff_sha256, input, output)` of a measured row; `None` for a failed +/// capture, a line that is not a row, or a row without positive input and an +/// output count. +fn measured(line: &str) -> Option<(String, String, u64, u64)> { + let v: Value = serde_json::from_str(line).ok()?; + if v["trace_status"].as_str() == Some("capture_failed") { + return None; + } + let input = v["tokens"]["input"].as_u64().filter(|&n| n > 0)?; + let output = v["tokens"]["output"].as_u64()?; + Some(( + v["lane"].as_str()?.to_string(), + v["diff_sha256"].as_str()?.to_string(), + input, + output, + )) +} + +fn decide(lanes: &[LaneProfile], min_n: usize) -> (Option, Vec) { + let deciding: Vec<&LaneProfile> = lanes.iter().filter(|l| l.n >= min_n).collect(); + if deciding.is_empty() { + return ( + None, + vec![format!( + "S-14 §0.2 undecided: no lane has min_n = {min_n} measured rows" + )], + ); + } + let mut refusals = Vec::new(); + for l in deciding { + if !l.in_band() { + refusals.push(format!( + "§0.2 falsified by {}: input p50 {} / p95 {} outside 2000–32000", + l.lane, l.input_p50, l.input_p95 + )); + } + if !l.prefill_heavy() { + refusals.push(format!( + "§0.2 falsified by {}: output p95 {} ≥ input p50 {}", + l.lane, l.output_p95, l.input_p50 + )); + } + } + (Some(refusals.is_empty()), refusals) +} + +/// The receipt's `workload` block (§7 template) plus the claim, strata and +/// exclusions it was decided on. +#[must_use] +#[allow(clippy::disallowed_methods)] // `json!` expands to an `unwrap` of an infallible `to_value` +pub fn render(p: &Profile) -> Value { + let pair = |f: fn(&LaneProfile) -> [f64; 2]| -> serde_json::Map { + p.lanes + .iter() + .map(|l| (l.lane.clone(), json!(f(l)))) + .collect() + }; + let strata: serde_json::Map = p + .strata + .iter() + .map(|(b, w)| (format!("{}k", b / 1000), json!(w))) + .collect(); + let n: serde_json::Map = p + .lanes + .iter() + .map(|l| (l.lane.clone(), json!(l.n))) + .collect(); + json!({"workload": { + "input_tokens_p50_p95": pair(|l| [l.input_p50, l.input_p95]), + "output_tokens_p50_p95": pair(|l| [l.output_p50, l.output_p95]), + "n": n, + "claim_0_2": p.claim, + "refusals": p.refusals, + "strata": strata, + "units": p.units, + "over_32k": p.over_32k, + "excluded": p.excluded, + }}) +} + +#[cfg(test)] +#[path = "workload_tests.rs"] +mod tests; diff --git a/crates/aprender-review-experiment/src/workload_tests.rs b/crates/aprender-review-experiment/src/workload_tests.rs new file mode 100644 index 0000000000..eeeb470be6 --- /dev/null +++ b/crates/aprender-review-experiment/src/workload_tests.rs @@ -0,0 +1,170 @@ +// `json!` expands to an `unwrap` of an infallible `to_value`. +#![allow(clippy::disallowed_methods)] + +use super::*; + +fn row(lane: &str, diff: &str, input: u64, output: u64) -> String { + json!({"lane": lane, "diff_sha256": diff, "trace_status": "ok", + "tokens": {"input": input, "output": output}}) + .to_string() +} + +fn jsonl(rows: &[String]) -> String { + rows.iter().map(|r| format!("{r}\n")).collect() +} + +fn lane<'a>(p: &'a Profile, name: &str) -> &'a LaneProfile { + p.lanes + .iter() + .find(|l| l.lane == name) + .unwrap_or_else(|| panic!("lane {name} missing from {:?}", p.lanes)) +} + +/// 20 rows of one lane: inputs 1000, 2000, …, 20000 and outputs 10, 20, …, 200. +fn ramp(name: &str) -> Vec { + (1..=20u64) + .map(|i| row(name, &format!("d{i}"), 1000 * i, 10 * i)) + .collect() +} + +/// FALSIFY-WLP-001: quantiles are nearest rank on measured rows; a failed +/// capture, an unparseable line, a zero input and a missing output are +/// excluded and counted, and move nothing. +#[test] +fn falsify_wlp_001_quantiles_are_nearest_rank_on_measured_rows_only() { + let clean = ramp("qwen"); + let p = profile(&jsonl(&clean), 1); + let q = lane(&p, "qwen"); + assert_eq!(q.n, 20); + // nearest rank: p50 = rank 10, p95 = rank 19 + assert_eq!((q.input_p50, q.input_p95), (10_000.0, 19_000.0)); + assert_eq!((q.output_p50, q.output_p95), (100.0, 190.0)); + assert_eq!(p.excluded, 0); + + let mut dirty = clean.clone(); + dirty.push( + json!({"lane": "qwen", "diff_sha256": "x1", "trace_status": "capture_failed", + "tokens": {"input": 1, "output": 1}}) + .to_string(), + ); + dirty.push("{not json".into()); + dirty.push(row("qwen", "x2", 0, 0)); + dirty.push(json!({"lane": "qwen", "diff_sha256": "x3", "tokens": {"input": 5}}).to_string()); + dirty.push(json!({"lane": "qwen", "diff_sha256": "x4"}).to_string()); + let d = profile(&jsonl(&dirty), 1); + assert_eq!(d.excluded, 5); + assert_eq!(lane(&d, "qwen"), q, "an excluded row moved the quantiles"); + assert_eq!(d.units, p.units); +} + +/// FALSIFY-WLP-002: §0.2 is decided per lane on lanes with min_n rows. +#[test] +fn falsify_wlp_002_claim_is_decided_per_lane_and_refused_when_thin() { + // ramp: input p50 10000 ≥ 2000, p95 19000 ≤ 32000, output p95 190 < 10000 + let mut rows = ramp("qwen"); + rows.extend(ramp("sonnet-5")); + let p = profile(&jsonl(&rows), 20); + assert!(lane(&p, "qwen").in_band() && lane(&p, "qwen").prefill_heavy()); + assert_eq!(p.claim, Some(true)); + assert!(p.refusals.is_empty(), "{:?}", p.refusals); + + // a decode-heavy lane: output p95 ≥ input p50 falsifies, naming the lane + let mut chatty = rows.clone(); + chatty.extend((1..=20u64).map(|i| row("gemini", &format!("d{i}"), 4000, 5000))); + let c = profile(&jsonl(&chatty), 20); + assert!(lane(&c, "gemini").in_band() && !lane(&c, "gemini").prefill_heavy()); + assert_eq!(c.claim, Some(false)); + assert!( + c.refusals.iter().any(|r| r.contains("gemini")), + "{:?}", + c.refusals + ); + // the edge: output p95 == input p50 is not prefill-heavy + let edge = profile(&jsonl(&[row("e", "d", 3000, 3000)]), 1); + assert!(!lane(&edge, "e").prefill_heavy()); + assert_eq!(edge.claim, Some(false)); + + // out of band, both ends; the bounds are inclusive + for (input, band) in [(1999, false), (2000, true), (32_000, true), (32_001, false)] { + let b = profile(&jsonl(&[row("b", "d", input, 10)]), 1); + assert_eq!(lane(&b, "b").in_band(), band, "input {input}"); + assert_eq!(b.claim, Some(band), "input {input}"); + } + + // a thin decode-heavy lane does not decide + let mut thin = rows.clone(); + thin.push(row("haiku", "d1", 3000, 9000)); + let t = profile(&jsonl(&thin), 20); + assert_eq!(lane(&t, "haiku").n, 1); + assert_eq!(t.claim, Some(true)); + + // no lane at min_n: undecided, refused, never confirmed + let u = profile(&jsonl(&rows), 21); + assert_eq!(u.claim, None); + assert!( + u.refusals.iter().any(|r| r.contains("min_n")), + "{:?}", + u.refusals + ); + let e = profile("", 1); + assert_eq!(e.claim, None); + assert!(e.lanes.is_empty()); +} + +/// FALSIFY-WLP-003: strata weights are over distinct diffs at their largest +/// lane measurement. +#[test] +fn falsify_wlp_003_strata_weights_are_over_distinct_diffs() { + let rows = [ + // one diff, three lanes: sized by its largest measurement, 8001 → 16k + row("qwen", "a", 7000, 10), + row("sonnet-5", "a", 8001, 10), + row("gemini", "a", 7500, 10), + // inclusive bounds + row("qwen", "b", 2000, 10), + row("qwen", "c", 2001, 10), + row("qwen", "d", 32_000, 10), + // over the top stratum: counted, not weighted + row("qwen", "e", 40_000, 10), + row("sonnet-5", "e", 1000, 10), + ]; + let p = profile(&jsonl(&rows), 1); + assert_eq!(p.units, 5); + assert_eq!(p.over_32k, 1); + assert_eq!( + p.strata, + vec![(2000, 0.25), (8000, 0.25), (16000, 0.25), (32000, 0.25)] + ); + let sum: f64 = p.strata.iter().map(|s| s.1).sum(); + assert!((sum - 1.0).abs() < 1e-12); + + // a lane count does not weight a diff + let mut many = rows.to_vec(); + many.extend((0..10).map(|i| row(&format!("l{i}"), "b", 2000, 10))); + assert_eq!(profile(&jsonl(&many), 1).strata, p.strata); + + // nothing in range: no weights, not NaN + let o = profile(&jsonl(&[row("q", "z", 50_000, 10)]), 1); + assert_eq!(o.over_32k, 1); + assert!(o.strata.iter().all(|s| s.1 == 0.0), "{:?}", o.strata); +} + +/// FALSIFY-WLP-004: the rendered block matches the v3 §7 receipt template. +#[test] +fn falsify_wlp_004_receipt_block_matches_the_v3_template() { + let p = profile(&jsonl(&ramp("qwen")), 20); + let v = render(&p); + assert_eq!( + v["workload"]["input_tokens_p50_p95"]["qwen"], + json!([10_000.0, 19_000.0]) + ); + assert_eq!( + v["workload"]["output_tokens_p50_p95"]["qwen"], + json!([100.0, 190.0]) + ); + assert_eq!(v["workload"]["claim_0_2"], json!(true)); + assert_eq!(v["workload"]["excluded"], json!(0)); + assert_eq!(v["workload"]["strata"]["16k"], json!(0.4)); + let u = render(&profile(&jsonl(&ramp("qwen")), 99)); + assert_eq!(u["workload"]["claim_0_2"], Value::Null); +} diff --git a/crates/aprender-serve/src/api/chat_logprobs.rs b/crates/aprender-serve/src/api/chat_logprobs.rs new file mode 100644 index 0000000000..053ff822d6 --- /dev/null +++ b/crates/aprender-serve/src/api/chat_logprobs.rs @@ -0,0 +1,91 @@ +//! OpenAI `logprobs` / `top_logprobs` on `/v1/chat/completions` (#4026). +//! +//! Only the session engine records per-step logprobs, so only the Qwen3.5 arm +//! answers them. Every other arm, and the streaming path, refuses the request +//! by name: a reply without the logprobs it was asked for would read as a +//! model that has none. +//! +//! Each `content` entry is OpenAI's shape (`token`, `logprob`, `bytes`, +//! `top_logprobs`) plus three fields PRM C11 (`sparse-logits-v1`) needs and +//! OpenAI does not carry: `token_id`, the top ids, and `logsumexp_full`, the +//! full-vocab log-sum-exp at T = 1.0 from which the retained mass and the +//! residual tail follow. All values are from the forward's logits, before any +//! repetition penalty or sampler. + +use super::ChatCompletionRequest; +use crate::gguf::StepLogprobs; + +/// OpenAI's ceiling on `top_logprobs`. +pub(crate) const MAX_TOP_LOGPROBS: usize = 20; + +/// `Ok(None)` when the request asks for no logprobs, `Ok(Some(n))` for `n` +/// alternatives per token, `Err` for a request that cannot be answered as +/// asked. +pub(crate) fn requested_top_logprobs(req: &ChatCompletionRequest) -> Result, String> { + let wanted = req.logprobs.unwrap_or(false); + match (wanted, req.top_logprobs) { + (false, None) => Ok(None), + (false, Some(_)) => Err("`top_logprobs` requires `logprobs: true` (#4026)".to_string()), + (true, Some(n)) if n > MAX_TOP_LOGPROBS => Err(format!( + "`top_logprobs` is {n}; at most {MAX_TOP_LOGPROBS} are supported (#4026)" + )), + (true, _) if req.stream => Err( + "`logprobs` on a streamed chat completion is not recorded yet; send `stream: false` \ + (#4026)" + .to_string(), + ), + (true, n) => Ok(Some(n.unwrap_or(0))), + } +} + +/// The refusal every arm but the session engine returns for `logprobs: true`. +pub(crate) fn unsupported_backend_reason(architecture: Option<&str>) -> String { + format!( + "`logprobs` are recorded only by the Qwen3.5 session engine; this model ({}) is served \ + by a backend that cannot record them, so the request is refused rather than answered \ + without them (#4026)", + architecture.unwrap_or("unknown architecture") + ) +} + +/// The `choices[].logprobs` object for the generated `steps`. +/// +/// `steps` must already be trimmed to the ids the reply reports; `n_top` of +/// each step's recorded alternatives are emitted. +pub(crate) fn chat_logprobs_json( + steps: &[StepLogprobs], + n_top: usize, + decode: &dyn Fn(u32) -> String, +) -> serde_json::Value { + let content: Vec = steps + .iter() + .map(|s| { + let top: Vec = s + .top + .iter() + .take(n_top) + .map(|t| { + serde_json::json!({ + "token": decode(t.token_id), + "token_id": t.token_id, + "logprob": t.logprob, + "bytes": serde_json::Value::Null, + }) + }) + .collect(); + serde_json::json!({ + "token": decode(s.chosen), + "token_id": s.chosen, + "logprob": s.chosen_logprob, + "bytes": serde_json::Value::Null, + "top_logprobs": top, + "logsumexp_full": s.logsumexp_full, + }) + }) + .collect(); + serde_json::json!({ "content": content }) +} + +#[cfg(test)] +#[path = "chat_logprobs_tests.rs"] +mod tests; diff --git a/crates/aprender-serve/src/api/chat_logprobs_tests.rs b/crates/aprender-serve/src/api/chat_logprobs_tests.rs new file mode 100644 index 0000000000..199c2d37e4 --- /dev/null +++ b/crates/aprender-serve/src/api/chat_logprobs_tests.rs @@ -0,0 +1,105 @@ +use super::*; +use crate::gguf::TopLogprob; + +fn req(logprobs: Option, top: Option, stream: bool) -> ChatCompletionRequest { + ChatCompletionRequest { + logprobs, + top_logprobs: top, + stream, + ..Default::default() + } +} + +#[test] +fn requested_top_logprobs_case_table() { + // (logprobs, top_logprobs, stream) -> Ok(want) | Err(substring) + let table: &[( + Option, + Option, + bool, + Result, &str>, + )] = &[ + (None, None, false, Ok(None)), + (Some(false), None, false, Ok(None)), + (None, None, true, Ok(None)), + (Some(true), None, false, Ok(Some(0))), + (Some(true), Some(0), false, Ok(Some(0))), + (Some(true), Some(20), false, Ok(Some(20))), + (Some(true), Some(21), false, Err("at most 20")), + (None, Some(5), false, Err("requires `logprobs: true`")), + ( + Some(false), + Some(5), + false, + Err("requires `logprobs: true`"), + ), + (Some(true), Some(5), true, Err("stream")), + ]; + for (lp, top, stream, want) in table { + let got = requested_top_logprobs(&req(*lp, *top, *stream)); + match (want, &got) { + (Ok(w), Ok(g)) => assert_eq!(w, g, "{lp:?} {top:?} {stream}"), + (Err(s), Err(e)) => assert!(e.contains(s), "{lp:?} {top:?} {stream}: {e}"), + _ => panic!("{lp:?} {top:?} {stream}: want {want:?}, got {got:?}"), + } + } +} + +#[test] +fn refusal_names_the_architecture() { + assert!(unsupported_backend_reason(Some("llama")).contains("(llama)")); + assert!(unsupported_backend_reason(None).contains("unknown architecture")); +} + +fn step(chosen: u32, top: &[(u32, f32)], chosen_logprob: f32, lse: f32) -> StepLogprobs { + StepLogprobs { + step: 0, + chosen, + top: top + .iter() + .map(|&(token_id, logprob)| TopLogprob { + token_id, + logit: logprob + lse, + logprob, + }) + .collect(), + chosen_logprob, + logsumexp_full: lse, + } +} + +/// Every field a C11 row needs is present, `n_top` truncates the +/// alternatives only, and a sampled token outside the top-K keeps its own +/// logprob instead of borrowing the top entry's. +#[test] +fn chat_logprobs_json_carries_ids_logprobs_and_lse() { + let steps = [ + step(7, &[(7, -0.1), (3, -2.5), (9, -3.0)], -0.1, 12.5), + step(42, &[(1, -0.2), (2, -1.9), (4, -2.2)], -6.0, 11.0), + ]; + let v = chat_logprobs_json(&steps, 2, &|id| format!("t{id}")); + let c = v["content"].as_array().expect("content"); + assert_eq!(c.len(), 2); + assert_eq!(c[0]["token"], "t7"); + assert_eq!(c[0]["token_id"], 7); + assert_eq!(c[0]["logsumexp_full"], 12.5); + assert_eq!(c[0]["top_logprobs"].as_array().expect("top").len(), 2); + assert_eq!(c[0]["top_logprobs"][1]["token_id"], 3); + assert_eq!(c[0]["top_logprobs"][1]["token"], "t3"); + assert_eq!(c[1]["token_id"], 42); + assert_eq!(c[1]["logprob"], -6.0); + assert!(c[1]["top_logprobs"] + .as_array() + .expect("top") + .iter() + .all(|t| t["token_id"] != 42)); + let none = chat_logprobs_json(&steps, 0, &|id| id.to_string()); + assert!(none["content"][0]["top_logprobs"] + .as_array() + .expect("top") + .is_empty()); + assert_eq!( + none["content"][0]["logprob"].as_f64(), + Some(f64::from(-0.1_f32)) + ); +} diff --git a/crates/aprender-serve/src/api/cuda_chat_backend.rs b/crates/aprender-serve/src/api/cuda_chat_backend.rs index 62757c1c64..c689e1ed0a 100644 --- a/crates/aprender-serve/src/api/cuda_chat_backend.rs +++ b/crates/aprender-serve/src/api/cuda_chat_backend.rs @@ -969,46 +969,68 @@ async fn try_apr_q4k_chat_backend( )) } -/// OpenAI-compatible /v1/chat/completions endpoint (supports streaming) -pub async fn openai_chat_completions_handler( - State(state): State, - headers: HeaderMap, - Extension(cancel): Extension, - Json(request): Json, -) -> Response { - let start = Instant::now(); - // GH-152: Verbose request logging - if state.is_verbose() { - let msg_count = request.messages.len(); - let last_msg = request - .messages - .last() - .map(|m| m.content.chars().take(50).collect::()) - .unwrap_or_default(); - eprintln!( - "[VERBOSE] POST /v1/chat/completions model={} messages={} last={:?}", - request.model, msg_count, last_msg - ); +/// GH-152: verbose request logging for the chat-completions endpoint, a no-op +/// unless `state.is_verbose()`. +fn log_verbose_chat_request(state: &AppState, request: &ChatCompletionRequest) { + if !state.is_verbose() { + return; } + let msg_count = request.messages.len(); + let last_msg = request + .messages + .last() + .map(|m| m.content.chars().take(50).collect::()) + .unwrap_or_default(); + eprintln!( + "[VERBOSE] POST /v1/chat/completions model={} messages={} last={:?}", + request.model, msg_count, last_msg + ); +} - let trace_level = headers +/// The `X-Trace-Level` header, lower-cased, when present and valid UTF-8. +fn trace_level_header(headers: &HeaderMap) -> Option { + headers .get("X-Trace-Level") .and_then(|v| v.to_str().ok()) - .map(str::to_lowercase); + .map(str::to_lowercase) +} - let request_id = format!( +/// A fresh `chatcmpl-q4k-` id for one chat-completions request. +fn new_chat_request_id() -> String { + format!( "chatcmpl-q4k-{}", std::time::SystemTime::now() .duration_since(std::time::UNIX_EPOCH) .unwrap_or_default() .as_millis() - ); + ) +} + +/// OpenAI-compatible /v1/chat/completions endpoint (supports streaming) +pub async fn openai_chat_completions_handler( + State(state): State, + headers: HeaderMap, + Extension(cancel): Extension, + Json(request): Json, +) -> Response { + let start = Instant::now(); + log_verbose_chat_request(&state, &request); + + let trace_level = trace_level_header(&headers); + + let request_id = new_chat_request_id(); // #3723: two spellings of the thinking toggle that disagree are refused, never picked between. if let Some(reason) = request.thinking_conflict() { return fail_response(&state, StatusCode::BAD_REQUEST, reason); } + // #4026: a logprobs request that cannot be answered as asked is refused here. + let wants_logprobs = match super::chat_logprobs::requested_top_logprobs(&request) { + Ok(n) => n.is_some(), + Err(reason) => return fail_response(&state, StatusCode::BAD_REQUEST, reason), + }; + // #3571: a Qwen3.5 hybrid is answered from its resident session or not at all. if let Some(r) = try_qwen35_backend( &state, @@ -1023,61 +1045,99 @@ pub async fn openai_chat_completions_handler( return r; } + // #4026: no arm below records logprobs. + if wants_logprobs { + return fail_response( + &state, + StatusCode::BAD_REQUEST, + super::chat_logprobs::unsupported_backend_reason(state.model_architecture().as_deref()), + ); + } + if let Some(r) = try_qwen3_moe_backend(&state, &request, &request_id, start, &cancel) { return r; } - #[cfg(feature = "gpu")] - if let Some(r) = try_gpu_backend( + if let Some(r) = try_accelerated_backends( &state, &request, &request_id, trace_level.as_deref(), start, &cancel, - ) { + ) + .await + { return r; } - #[cfg(feature = "gpu")] - if let Some(r) = try_cached_backend( + cpu_chat_backends( &state, &request, &request_id, trace_level.as_deref(), start, &cancel, - ) { - return r; + ) +} + +/// The GPU/CUDA arm of the chat backend chain: (non-batched) GPU, then the +/// residency cache, then (cuda-only) the CUDA q4k backend, the APR Q4K GPU +/// backend, and the SafeTensors CUDA backend. `None` when nothing in this +/// arm is compiled in or resident, so the caller falls through to +/// [`cpu_chat_backends`]. +/// +/// Lifted out of [`openai_chat_completions_handler`] for the same reason as +/// `cpu_chat_backends`: each accelerated backend added to the dispatch table +/// cost the handler the same branch, unconditionally on every request. +/// `async` is only exercised when built `--features cuda`; without it every +/// `.await` point is `#[cfg]`-gated out, same as elsewhere in this file. +#[allow(clippy::unused_async)] +async fn try_accelerated_backends( + state: &AppState, + request: &ChatCompletionRequest, + request_id: &str, + trace_level: Option<&str>, + start: Instant, + cancel: &CancelToken, +) -> Option { + #[cfg(feature = "gpu")] + if let Some(r) = try_gpu_backend(state, request, request_id, trace_level, start, cancel) { + return Some(r); + } + + #[cfg(feature = "gpu")] + if let Some(r) = try_cached_backend(state, request, request_id, trace_level, start, cancel) { + return Some(r); } #[cfg(feature = "cuda")] if let Some(r) = - try_cuda_backend(&state, &request, &request_id, trace_level.as_deref(), start, &cancel).await + try_cuda_backend(state, request, request_id, trace_level, start, cancel).await { - return r; + return Some(r); } // ALB-110: APR Q4K GPU backend via dedicated inference thread #[cfg(feature = "cuda")] - if let Some(r) = try_apr_q4k_chat_backend(&state, &request, &request_id, trace_level.as_deref(), start, &cancel).await { - return r; + if let Some(r) = + try_apr_q4k_chat_backend(state, request, request_id, trace_level, start, cancel).await + { + return Some(r); } // #169: SafeTensors CUDA backend (format parity) #[cfg(feature = "cuda")] - if let Some(r) = try_safetensors_cuda_backend(&state, &request, &request_id, start, &cancel) { - return r; + if let Some(r) = try_safetensors_cuda_backend(state, request, request_id, start, cancel) { + return Some(r); } - cpu_chat_backends( - &state, - &request, - &request_id, - trace_level.as_deref(), - start, - &cancel, - ) + // Silences "unused" when built with neither `gpu` nor `cuda` — every + // branch above is `#[cfg]`-gated, so with both features off this + // function's parameters otherwise go unread. + let _ = (state, request, request_id, trace_level, start, cancel); + + None } /// The CPU tail of the chat backend chain: quantized, then the f32 diff --git a/crates/aprender-serve/src/api/mod.rs b/crates/aprender-serve/src/api/mod.rs index 79086d40d0..a294d86ee2 100644 --- a/crates/aprender-serve/src/api/mod.rs +++ b/crates/aprender-serve/src/api/mod.rs @@ -51,6 +51,7 @@ use crate::{ // aprender#2376(3): request-scoped cancellation for the generate handlers. mod cancel_scope; +mod chat_logprobs; pub(crate) use cancel_scope::cancel_on_disconnect; pub use cancel_scope::request_cancel_token; @@ -80,7 +81,8 @@ mod serve_trace; mod stream_tool_calls; pub(crate) use openai_handlers::LiveUtf8Deltas; pub(crate) use openai_handlers::{ - openai_chat_completions_handler, openai_chat_completions_stream_handler, openai_models_handler, + chat_prompt_ids_handler, openai_chat_completions_handler, + openai_chat_completions_stream_handler, openai_models_handler, }; // PMAT-923: Ollama HTTP compat (/api/chat, /api/generate) — delegates to the // OpenAI chat path so `apr serve` is a drop-in Ollama HTTP replacement. diff --git a/crates/aprender-serve/src/api/mod_create_demo.rs b/crates/aprender-serve/src/api/mod_create_demo.rs index 9687866775..8d1745bb17 100644 --- a/crates/aprender-serve/src/api/mod_create_demo.rs +++ b/crates/aprender-serve/src/api/mod_create_demo.rs @@ -122,6 +122,15 @@ pub struct ChatCompletionRequest { /// Stream responses #[serde(default)] pub stream: bool, + /// OpenAI `logprobs`: return per-token log-probabilities (#4026). + /// + /// Only the session engine records them. A backend that cannot refuses the + /// request by name rather than answering without them. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub logprobs: Option, + /// OpenAI `top_logprobs`: how many alternatives per token, 0..=20. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub top_logprobs: Option, /// OpenAI `stream_options`, accepted so a client that sends /// `{"include_usage": true}` is not rejected as malformed. /// diff --git a/crates/aprender-serve/src/api/openai_handlers.rs b/crates/aprender-serve/src/api/openai_handlers.rs index 470e6f1ce3..a4c64a8a0b 100644 --- a/crates/aprender-serve/src/api/openai_handlers.rs +++ b/crates/aprender-serve/src/api/openai_handlers.rs @@ -471,6 +471,8 @@ mod pmat821_chat_handler_threading_tests { ignore_eos: None, n: crate::api::ChoiceCount::ONE, stream: false, + logprobs: None, + top_logprobs: None, stop: None, user: None, tools: None, @@ -752,6 +754,40 @@ pub(crate) fn build_chat_response_traced( timings: Option, used_gpu: Option, ) -> Response { + Json(chat_response_body( + request_id, + model, + text, + prompt_tokens, + completion_tokens, + max_tokens, + stops, + traces, + tools, + tool_choice, + timings, + used_gpu, + )) + .into_response() +} + +/// The body [`build_chat_response_traced`] serializes, for an arm that adds +/// to it (the Qwen3.5 arm's `logprobs`, #4026). +#[allow(clippy::too_many_arguments)] +pub(crate) fn chat_response_body( + request_id: String, + model: String, + text: String, + prompt_tokens: usize, + completion_tokens: usize, + max_tokens: usize, + stops: Option<&[String]>, + traces: (Option, Option, Option), + tools: Option<&[super::OpenAiTool]>, + tool_choice: Option, + timings: Option, + used_gpu: Option, +) -> ChatCompletionResponse { let (brick_trace, step_trace, layer_trace) = traces; let (text, finish_reason) = finalize_chat_text(text, stops, completion_tokens, max_tokens); @@ -770,7 +806,7 @@ pub(crate) fn build_chat_response_traced( ), }; - Json(ChatCompletionResponse { + ChatCompletionResponse { used_gpu, id: request_id, object: "chat.completion".to_string(), @@ -790,8 +826,7 @@ pub(crate) fn build_chat_response_traced( step_trace, layer_trace, timings, - }) - .into_response() + } } /// Serialize a value to an SSE event, returning `None` if serialization fails. diff --git a/crates/aprender-serve/src/api/qwen35_chat_backend.rs b/crates/aprender-serve/src/api/qwen35_chat_backend.rs index 9d9685639f..8e3bce28b4 100644 --- a/crates/aprender-serve/src/api/qwen35_chat_backend.rs +++ b/crates/aprender-serve/src/api/qwen35_chat_backend.rs @@ -79,6 +79,107 @@ fn gen_config_from_request( } } +/// The Qwen3.5 hybrid's prompt for a chat request: the model's own template in the +/// request's thinking mode (#3723) and with the request's tools (#4650), encoded by +/// the model. +/// +/// The chat path and `POST /v1/chat/prompt-ids` both call this, so the ids that +/// endpoint reports are the ids this path prefills (PRM-S1 v2, #4354). +fn qwen35_prompt( + model: &crate::gguf::GGUFModel, + request: &ChatCompletionRequest, + architecture: Option<&str>, +) -> Result<(String, Vec), String> { + // #4650: the template's `# Tools` block is the only place the model learns a tool + // exists and the `` format to call it in. + let text = crate::api::realize_handlers::format_chat_messages_official_thinking_tools( + Some(model), + &request.messages, + architecture, + request.thinking(), + request.tools.as_deref(), + ) + .map_err(|e| e.to_string())?; + let ids = model.encode(&text).unwrap_or_default(); + Ok((text, ids)) +} + +/// `POST /v1/chat/prompt-ids`: the rendered prompt and its ids for a chat request +/// body, without generating (PRM-S1 v2, #4354). +/// +/// A cross-engine replay must show both engines prefill the same ids before it +/// compares their speed. llama.cpp answers that with `/apply-template` + `/tokenize`; +/// apr's `/tokenize` is not the chat path's encoder, so this answers from the chat +/// path itself. Only the Qwen3.5 hybrid path is covered; any other backend is 501, +/// never ids some other path might have built. +pub async fn chat_prompt_ids_handler( + State(state): State, + Json(request): Json, +) -> Response { + let (Some(_), Some(mapped)) = (state.qwen35_session(), state.mapped_gguf_model()) else { + return fail_response( + &state, + StatusCode::NOT_IMPLEMENTED, + "prompt ids are reported only for the Qwen3.5 hybrid chat path (#4354)", + ); + }; + let architecture = state.model_architecture(); + match qwen35_prompt(&mapped.model, &request, architecture.as_deref()) { + Ok((prompt, ids)) => Json(serde_json::json!({ + "path": "qwen35", + "prompt": prompt, + "num_tokens": ids.len(), + "prompt_ids": ids, + })) + .into_response(), + Err(e) => fail_response(&state, StatusCode::BAD_REQUEST, e), + } +} + +/// Encode the request's prompt for the resident Qwen3.5 session and derive +/// the context-bounded decode budget. +/// +/// Extracted from `try_qwen35_backend` (complexity ratchet, #4446): the +/// template render, the empty-prompt guard, and the context-length refusal +/// are all early-out validation on the way to one `(input_ids, +/// prompt_token_count, budget)` triple, and don't need the caller's own +/// nesting. Behaviour (including every error string) is unchanged. +#[allow(clippy::result_large_err)] +fn prepare_qwen35_prompt( + state: &AppState, + request: &ChatCompletionRequest, + mapped: &crate::gguf::MappedGGUFModel, + context_length: usize, +) -> Result<(Vec, usize, usize), Response> { + let architecture = state.model_architecture(); + // #3723 thinking mode + template, via the same helper `POST /v1/chat/prompt-ids` + // reports from (PRM-S1 v2), so the reported ids are the prefilled ids. + let (_, input_ids) = qwen35_prompt(&mapped.model, request, architecture.as_deref()) + .map_err(|e| fail_response(state, StatusCode::BAD_REQUEST, e))?; + if input_ids.is_empty() { + return Err(fail_response( + state, + StatusCode::BAD_REQUEST, + "Messages cannot be empty", + )); + } + let prompt_token_count = input_ids.len(); + if prompt_token_count >= context_length { + return Err(fail_response( + state, + StatusCode::BAD_REQUEST, + format!( + "the prompt is {prompt_token_count} tokens and this model declares a context of \ + {context_length}: it was refused whole rather than truncated (#3571)" + ), + )); + } + let max_tokens = request.max_tokens.unwrap_or(256); + // What the context leaves — the budget the session will actually decode. + let budget = max_tokens.min(context_length - prompt_token_count); + Ok((input_ids, prompt_token_count, budget)) +} + /// The Qwen3.5 arm of the chat backend chain (#3571). /// /// `None` when this state serves no hybrid, so the chain falls through @@ -116,50 +217,26 @@ async fn try_qwen35_backend( Err(r) => return Some(r), }; - let architecture = state.model_architecture(); - // #3723: the request's thinking mode, rendered by the model's own template. - // #4650: and the request's tools -- the template's `# Tools` block is the only place the - // model learns a tool exists and the `` format to call it in. - let prompt_text = - match crate::api::realize_handlers::format_chat_messages_official_thinking_tools( - Some(&mapped.model), - &request.messages, - architecture.as_deref(), - request.thinking(), - request.tools.as_deref(), - ) { - Ok(p) => p, - Err(e) => return Some(fail_response(state, StatusCode::BAD_REQUEST, e.to_string())), - }; - let input_ids = mapped.model.encode(&prompt_text).unwrap_or_default(); - if input_ids.is_empty() { - return Some(fail_response( - state, - StatusCode::BAD_REQUEST, - "Messages cannot be empty", - )); - } - let prompt_token_count = input_ids.len(); - let context_length = session.context_length; - if prompt_token_count >= context_length { - return Some(fail_response( - state, - StatusCode::BAD_REQUEST, - format!( - "the prompt is {prompt_token_count} tokens and this model declares a context of \ - {context_length}: it was refused whole rather than truncated (#3571)" - ), - )); - } - let max_tokens = request.max_tokens.unwrap_or(256); - // What the context leaves — the budget the session will actually decode. - let budget = max_tokens.min(context_length - prompt_token_count); + let (input_ids, prompt_token_count, budget) = + match prepare_qwen35_prompt(state, request, &mapped, context_length) { + Ok(v) => v, + Err(r) => return Some(r), + }; let stop_tokens = stop_tokens_unless_ignore_eos(request, state.model_eos_token_id()); // The context-bounded budget, not the request's number: what is decoded and what // `finish_reason` is judged against are the same count. - let gen_config = gen_config_from_request(request, budget, stop_tokens.clone(), cancel.clone()); + let mut gen_config = + gen_config_from_request(request, budget, stop_tokens.clone(), cancel.clone()); + // #4026: validated by the handler before any arm ran. + let top_logprobs = crate::api::chat_logprobs::requested_top_logprobs(request) + .ok() + .flatten(); + if let Some(n) = top_logprobs { + // At least one alternative is recorded: `logprobs_top_k` 0 records nothing. + gen_config.logprobs_top_k = n.max(1); + } if request.stream { let (tx, rx) = tokio::sync::mpsc::channel::>(64); @@ -269,6 +346,29 @@ async fn try_qwen35_backend( completion_tokens, num_layers: measured.num_layers, }); + if let Some(n) = top_logprobs { + // One record per generated id; the popped stop token's record goes too. + let steps = &turn.steps[..completion_tokens.min(turn.steps.len())]; + let decode = |id: u32| decode_mapped.model.decode(&[id]); + let mut body = serde_json::to_value(chat_response_body( + request_id.to_string(), + request.model.clone(), + response_text, + prompt_token_count, + completion_tokens, + budget, + request.stop.as_deref(), + traces, + request.tools.as_deref(), + request_tool_choice(request), + None, + None, + )) + .unwrap_or_default(); + body["choices"][0]["logprobs"] = + crate::api::chat_logprobs::chat_logprobs_json(steps, n, &decode); + return Some(Json(body).into_response()); + } Some(build_chat_response_traced( request_id.to_string(), request.model.clone(), diff --git a/crates/aprender-serve/src/api/qwen35_chat_backend_tests.rs b/crates/aprender-serve/src/api/qwen35_chat_backend_tests.rs index 0ea3ba83f8..4f5ed889d2 100644 --- a/crates/aprender-serve/src/api/qwen35_chat_backend_tests.rs +++ b/crates/aprender-serve/src/api/qwen35_chat_backend_tests.rs @@ -938,6 +938,50 @@ async fn a_streamed_completion_arrives_token_by_token_and_ends_with_usage() { assert_eq!(usage, plain["usage"], "the stream's usage is the body's"); } +/// PRM-S1 v2 (#4354): `POST /v1/chat/prompt-ids` reports the ids the chat path +/// prefills — the same count the chat reply bills as `prompt_tokens`, and the same +/// ids as rendering and encoding directly — in both thinking modes. +#[tokio::test(flavor = "multi_thread")] +async fn prompt_ids_are_the_ids_the_chat_path_prefills_4354() { + let Some((state, mapped)) = state_or_skip(true) else { + return; + }; + let msgs = [crate::chat_template::ChatMessage::new("user", QUESTION)]; + let app = create_router(state); + let mut seen = Vec::new(); + for thinking in [false, true] { + let mut body = chat_body(false, 1); + body["chat_template_kwargs"] = serde_json::json!({ "enable_thinking": thinking }); + let (status, reply) = post(app.clone(), "/v1/chat/prompt-ids", body.clone()).await; + assert_eq!(status, StatusCode::OK, "{reply}"); + let json: serde_json::Value = serde_json::from_str(&reply).expect("JSON"); + let ids: Vec = serde_json::from_value(json["prompt_ids"].clone()).expect("ids"); + let direct = crate::chat_template::render_official_for_model(&mapped.model, &msgs, Some(thinking)) + .expect("renders"); + assert_eq!(json["prompt"].as_str(), Some(direct.as_str()), "thinking={thinking}"); + assert_eq!(Some(ids.clone()), mapped.model.encode(&direct), "thinking={thinking}"); + + let (status, chat) = post(app.clone(), "/v1/chat/completions", body).await; + assert_eq!(status, StatusCode::OK, "{chat}"); + let chat: serde_json::Value = serde_json::from_str(&chat).expect("JSON"); + assert_eq!(chat["usage"]["prompt_tokens"].as_u64(), Some(ids.len() as u64)); + seen.push(ids); + } + assert_ne!(seen[0], seen[1], "the probe must distinguish the ON and OFF prompts"); +} + +/// #3991 applied to the new route: it is mounted and listed only where a Qwen3.5 +/// session can answer it. +#[test] +fn prompt_ids_route_is_listed_only_with_a_qwen35_session_4354() { + let config = crate::api::RouterConfig::default(); + let mut caps = crate::api::RouteCapabilities::all(); + let listed = |c| crate::api::advertised_routes_for(&config, c); + assert!(listed(caps).iter().any(|r| r == "POST /v1/chat/prompt-ids")); + caps.qwen35_prompt_ids = false; + assert!(!listed(caps).iter().any(|r| r.contains("/v1/chat/prompt-ids"))); +} + // #4274: a chat's second turn, re-rendered through the official template (which // drops turn 1's think block), resumed from the checkpoint turn 1 left, and // answered exactly what a full re-prefill answers. diff --git a/crates/aprender-serve/src/api/router.rs b/crates/aprender-serve/src/api/router.rs index 1d8b03baff..75b89b8f73 100644 --- a/crates/aprender-serve/src/api/router.rs +++ b/crates/aprender-serve/src/api/router.rs @@ -134,6 +134,8 @@ fn metrics_routes() -> Vec { pub struct RouteCapabilities { /// `cached_model` is present, so `/v1/batch/completions` can answer. pub gpu_batch: bool, + /// A Qwen3.5 hybrid session is resident, so `/v1/chat/prompt-ids` can answer (#4354). + pub qwen35_prompt_ids: bool, } impl RouteCapabilities { @@ -148,13 +150,19 @@ impl RouteCapabilities { let _ = state; false }; - Self { gpu_batch } + Self { + gpu_batch, + qwen35_prompt_ids: state.qwen35_session().is_some(), + } } /// Every capability present: the largest surface any state can mount. #[must_use] pub fn all() -> Self { - Self { gpu_batch: true } + Self { + gpu_batch: true, + qwen35_prompt_ids: true, + } } } @@ -212,6 +220,11 @@ fn openai_routes(caps: RouteCapabilities) -> Vec { post(gpu_batch_completions_handler), )); } + // PRM-S1 v2 (#4354): the chat path's own prompt ids, for cross-engine replay. + // Only the Qwen3.5 hybrid path reports them, so only there is the route mounted. + if caps.qwen35_prompt_ids { + routes.push(("POST", "/v1/chat/prompt-ids", post(chat_prompt_ids_handler))); + } routes } diff --git a/crates/aprender-serve/src/gguf/logprobs.rs b/crates/aprender-serve/src/gguf/logprobs.rs index 1359b34a2f..38b9b4fd98 100644 --- a/crates/aprender-serve/src/gguf/logprobs.rs +++ b/crates/aprender-serve/src/gguf/logprobs.rs @@ -34,3 +34,153 @@ pub fn logprob_of(logits: &[f32], token_id: u32) -> f32 { .ln(); logits[token_id as usize] - max_logit - log_sum_exp } + +/// One of the most likely next tokens at a step (#4026). +#[derive(Debug, Clone, PartialEq, serde::Serialize)] +pub struct TopLogprob { + /// Token ID + pub token_id: u32, + /// The raw logit the forward produced for it + pub logit: f32, + /// ln(softmax(logits)[token_id]) + pub logprob: f32, +} + +/// The distribution one generated step was chosen from (#4026): the `K` most +/// likely tokens, before any repetition penalty or sampling, and the token the +/// engine then chose. `step` 0 is the token after the prompt. +/// +/// On the wire (`apr run --json`, the HTTP `logprobs` object) a value that is +/// not a finite number serializes as JSON `null`: an `inf` logit makes the +/// log-sum-exp `inf` and the logprobs `NaN` or `-inf`, and JSON has neither. +/// Read `null` in `logit`, `logprob`, `chosen_logprob` or `logsumexp_full` as +/// "not finite", never as zero or as absent. +#[derive(Debug, Clone, PartialEq, serde::Serialize)] +pub struct StepLogprobs { + /// 0-based index of the generated token this step produced + pub step: usize, + /// The token the engine chose at this step + pub chosen: u32, + /// The `K` most likely tokens, most likely first + pub top: Vec, + /// ln(softmax(logits)[chosen]) on the same forward logits as `top`, so it + /// is defined when a sampled `chosen` is outside the top `K` (#4026, PRM C11) + pub chosen_logprob: f32, + /// Full-vocab log-sum-exp of those logits at T = 1.0: `top` plus this gives + /// the retained mass and the residual tail `sparse-logits-v1` stores + pub logsumexp_full: f32, +} + +/// Full-vocab log-sum-exp of `logits`, NaNs skipped, accumulated in f64 so a +/// 150k-entry vocab does not lose the tail to f32 rounding (#4026). +#[must_use] +pub fn logsumexp_full(logits: &[f32]) -> f32 { + let max = logits + .iter() + .copied() + .filter(|x| !x.is_nan()) + .fold(f32::NEG_INFINITY, f32::max); + if !max.is_finite() { + return max; + } + let sum: f64 = logits + .iter() + .filter(|x| !x.is_nan()) + .map(|&x| f64::from(x - max).exp()) + .sum(); + (f64::from(max) + sum.ln()) as f32 +} + +/// The `k` most likely tokens in `logits`, most likely first (#4026). +/// +/// Ties go to the lower token id, as [`crate::gguf::ops::argmax`] breaks them, +/// so with no penalty the first entry IS the greedy choice. NaN logits are +/// never ranked. `logprob` is `logit` minus [`logsumexp_full`]. +#[must_use] +pub fn top_k_logprobs(logits: &[f32], k: usize) -> Vec { + if k == 0 { + return Vec::new(); + } + // The step record's `logsumexp_full`, so Σexp(top) and the tail a reader + // derives from the two agree exactly (PRM C11). + let lse = logsumexp_full(logits); + let mut ranked: Vec<(u32, f32)> = logits + .iter() + .enumerate() + .filter(|(_, x)| !x.is_nan()) + .map(|(i, &x)| (i as u32, x)) + .collect(); + let k = k.min(ranked.len()); + let by_rank = |a: &(u32, f32), b: &(u32, f32)| b.1.total_cmp(&a.1).then(a.0.cmp(&b.0)); + if k < ranked.len() { + ranked.select_nth_unstable_by(k, by_rank); + ranked.truncate(k); + } + ranked.sort_by(by_rank); + ranked + .into_iter() + .map(|(token_id, logit)| TopLogprob { + token_id, + logit, + logprob: logit - lse, + }) + .collect() +} + +#[cfg(test)] +mod tests_4026 { + use super::*; + + fn ids(top: &[TopLogprob]) -> Vec { + top.iter().map(|t| t.token_id).collect() + } + + #[test] + fn top_k_logprobs_case_table() { + let logits = [0.5, 2.0, -1.0, 2.0, 1.0]; + // (k, expected ids): ties go to the lower id, as ops::argmax. + let table: &[(usize, &[u32])] = &[ + (0, &[]), + (1, &[1]), + (2, &[1, 3]), + (3, &[1, 3, 4]), + (5, &[1, 3, 4, 0, 2]), + (99, &[1, 3, 4, 0, 2]), + ]; + for &(k, want) in table { + assert_eq!(ids(&top_k_logprobs(&logits, k)), want, "k={k}"); + } + assert_eq!( + top_k_logprobs(&logits, 1)[0].token_id, + crate::gguf::ops::argmax(&logits), + "top-1 is the greedy choice" + ); + } + + #[test] + fn top_k_logprobs_agree_with_logprob_of_and_sum_to_one() { + let logits = [0.5_f32, 2.0, -1.0, 2.0, 1.0]; + let all = top_k_logprobs(&logits, logits.len()); + for t in &all { + assert!((t.logprob - logprob_of(&logits, t.token_id)).abs() < 1e-6); + assert_eq!(t.logit, logits[t.token_id as usize]); + } + let total: f32 = all.iter().map(|t| t.logprob.exp()).sum(); + assert!((total - 1.0).abs() < 1e-5, "sum={total}"); + } + + #[test] + fn top_k_logprobs_never_rank_nan() { + let logits = [f32::NAN, 1.0, f32::NAN, 0.0]; + let top = top_k_logprobs(&logits, 4); + assert_eq!(ids(&top), vec![1, 3]); + assert!(top.iter().all(|t| t.logprob.is_finite())); + } + + #[test] + fn top_k_logprobs_survive_huge_logits() { + let top = top_k_logprobs(&[1.0e30, 0.0], 2); + assert_eq!(top[0].logprob, 0.0); + assert!(top[1].logprob.is_finite() && top[1].logprob <= -1.0e29); + } +} diff --git a/crates/aprender-serve/src/gguf/runtime.rs b/crates/aprender-serve/src/gguf/runtime.rs index c6d29543c9..9cc230daef 100644 --- a/crates/aprender-serve/src/gguf/runtime.rs +++ b/crates/aprender-serve/src/gguf/runtime.rs @@ -41,6 +41,11 @@ pub struct QuantizedGenerateConfig { pub trace: bool, /// Return per-token log probabilities (realizr#191, F-QUALITY-01) pub logprobs: bool, + /// Record the `K` most likely next tokens at every generated step + /// (`apr run --logprobs K`, #4026). 0 records nothing. Only the one + /// engine ([`crate::session::Session`]) records; a path without it + /// refuses a non-zero `K` by name rather than returning an empty list. + pub logprobs_top_k: usize, /// Cooperative cancellation signal, polled once per decode step. /// /// aprender#2376(3): without this, an abandoned HTTP request kept decoding to @@ -63,6 +68,7 @@ impl Default for QuantizedGenerateConfig { stop_tokens: Vec::new(), trace: false, logprobs: false, + logprobs_top_k: 0, cancel: CancelToken::never(), } } diff --git a/crates/aprender-serve/src/infer/inference_result.rs b/crates/aprender-serve/src/infer/inference_result.rs index 66dc0fd9f0..d2cef0dbbb 100644 --- a/crates/aprender-serve/src/infer/inference_result.rs +++ b/crates/aprender-serve/src/infer/inference_result.rs @@ -209,6 +209,7 @@ pub fn run_inference_report( validate_model_path(&config.model_path)?; let format = ModelFormat::SafeTensors; + refuse_logprobs_off_gguf(config.logprobs_top_k, format)?; let prepared = prepare_tokens(config, &format)?; return run_sharded_safetensors_inference(config, &prepared) .map(|r| (r, run_report::RunReport::default())); @@ -243,6 +244,7 @@ pub fn run_inference_report( let format = detect_format(&magic).map_err(|e| RealizarError::FormatError { reason: format!("Format detection failed: {}", e), })?; + refuse_logprobs_off_gguf(config.logprobs_top_k, format)?; // PMAT-236: Prepare tokens with chat template BEFORE format dispatch. // This is compile-time enforced - format-specific functions accept @@ -275,6 +277,55 @@ fn mock_run_report(config: &InferenceConfig, result: &InferenceResult) -> run_re } } +/// #4026: only a GGUF reaches the session engine that records `--logprobs K`. +/// The format is known once the header is read, so APR and SafeTensors are +/// refused by name there, before the load, not after a whole generation. +fn refuse_logprobs_off_gguf(top_k: usize, format: ModelFormat) -> Result<()> { + if top_k == 0 { + return Ok(()); + } + let path = match format { + ModelFormat::Gguf => return Ok(()), + ModelFormat::Apr => "APR", + ModelFormat::SafeTensors => "SafeTensors", + }; + Err(RealizarError::UnsupportedOperation { + operation: "logprobs".to_string(), + reason: format!( + "--logprobs {top_k} is not recorded on the {path} path; only the GGUF session \ + engine records per-step logprobs (#4026) — refused before the load" + ), + }) +} + +/// #4026: `--logprobs K` is recorded by the one engine only. The two GGUF +/// routes known before the run not to go through it are refused here, by +/// name, before any forward runs; a route that turns out not to record (the +/// wgpu decode) is refused by the caller when no steps come back. +fn refuse_logprobs_off_engine( + gen_config: &crate::gguf::QuantizedGenerateConfig, + architecture: &str, +) -> Result<()> { + if gen_config.logprobs_top_k == 0 { + return Ok(()); + } + let route = if gen_config.trace { + "--trace (the instrumented decode loops, which keep no logits)" + } else if crate::gguf::moe_forward_handles(architecture) { + "the qwen3_moe decode (not on the session engine)" + } else { + return Ok(()); + }; + Err(RealizarError::UnsupportedOperation { + operation: "logprobs".to_string(), + reason: format!( + "--logprobs {} is not recorded by {route}; only the session engine records \ + per-step logprobs (#4026) — refused rather than reported empty", + gen_config.logprobs_top_k + ), + }) +} + /// Run GGUF model inference /// /// PMAT-236: Accepts `PreparedTokens` (compile-time enforced chat template). @@ -351,6 +402,8 @@ fn run_gguf_inference( // (M32c.2.2.2.1.2's run_qwen3_moe_generate). The dense path goes through // run_gguf_generate as before. This replaces M32c.2.1's // gguf_gpu_generate.rs short-circuit with an actual forward pass. + refuse_logprobs_off_engine(&gen_config, &model.config.architecture)?; + let _ = crate::session::take_last_turn_steps(); // #4026: never inherit an earlier turn's steps let infer_start = Instant::now(); let _ = take_generation_start(); // #3981: never inherit a mark from an earlier run let canonical_arch = crate::tensor_names::normalize_architecture(&model.config.architecture); @@ -2366,3 +2419,58 @@ mod throughput_3981_tests { #[cfg(test)] #[path = "generation_mark_4342_tests.rs"] mod generation_mark_4342_tests; + +#[cfg(test)] +mod tests_4026_refuse_off_engine { + use super::refuse_logprobs_off_engine; + use crate::gguf::QuantizedGenerateConfig; + + fn cfg(k: usize, trace: bool) -> QuantizedGenerateConfig { + QuantizedGenerateConfig { + logprobs_top_k: k, + trace, + ..Default::default() + } + } + + #[test] + fn logprobs_refusal_case_table() { + // (k, trace, arch, refused) + let table = [ + (0, true, "qwen3_moe", false), + (3, false, "qwen2", false), + (3, false, "qwen35", false), + (3, true, "qwen2", true), + (3, false, "qwen3_moe", true), + ]; + for (k, trace, arch, refused) in table { + let got = refuse_logprobs_off_engine(&cfg(k, trace), arch); + assert_eq!(got.is_err(), refused, "k={k} trace={trace} arch={arch}"); + if let Err(e) = got { + assert!(e.to_string().contains("--logprobs 3"), "{e}"); + } + } + } + + /// la-0.72 C: the format is refused before the load, not after a full run. + #[test] + fn logprobs_off_gguf_case_table() { + use super::{refuse_logprobs_off_gguf, ModelFormat}; + // (k, format, refused, named) + let table = [ + (0, ModelFormat::Apr, false, ""), + (0, ModelFormat::SafeTensors, false, ""), + (2, ModelFormat::Gguf, false, ""), + (2, ModelFormat::Apr, true, "APR path"), + (2, ModelFormat::SafeTensors, true, "SafeTensors path"), + ]; + for (k, format, refused, named) in table { + let got = refuse_logprobs_off_gguf(k, format); + assert_eq!(got.is_err(), refused, "k={k} format={format:?}"); + if let Err(e) = got { + let msg = e.to_string(); + assert!(msg.contains("--logprobs 2") && msg.contains(named), "{msg}"); + } + } + } +} diff --git a/crates/aprender-serve/src/infer/mod.rs b/crates/aprender-serve/src/infer/mod.rs index a419bf248a..dca5046297 100644 --- a/crates/aprender-serve/src/infer/mod.rs +++ b/crates/aprender-serve/src/infer/mod.rs @@ -234,6 +234,11 @@ pub struct InferenceConfig { /// removes the empty `` prefill so the model reasons, and is refused by name on a /// template with no thinking mode ([`crate::chat_template::apply_thinking_mode`]). pub thinking: Option, + /// #4026: `apr run --logprobs K` — record the `K` most likely next tokens at + /// every generated step. 0 records nothing. Only the one engine records + /// ([`crate::session::take_last_turn_steps`]); a run served by any other + /// path is refused by name by the caller, never reported with an empty list. + pub logprobs_top_k: usize, } /// The top-k a SAMPLED generation uses when the caller names none (#3754). @@ -288,9 +293,17 @@ impl InferenceConfig { use_mock_backend: false, force_chat_template: false, thinking: None, + logprobs_top_k: 0, } } + /// #4026: record the `k` most likely next tokens at every generated step. + #[must_use] + pub fn with_logprobs_top_k(mut self, k: usize) -> Self { + self.logprobs_top_k = k; + self + } + /// Set the text prompt #[must_use] pub fn with_prompt(mut self, prompt: impl Into) -> Self { @@ -425,6 +438,7 @@ impl InferenceConfig { gen_config.seed = self.seed; gen_config.repeat_penalty = self.repeat_penalty; gen_config.repeat_last_n = self.repeat_last_n; + gen_config.logprobs_top_k = self.logprobs_top_k; } } diff --git a/crates/aprender-serve/src/infer/tests_clean_model_03.rs b/crates/aprender-serve/src/infer/tests_clean_model_03.rs index d4e7834878..a1219cb5be 100644 --- a/crates/aprender-serve/src/infer/tests_clean_model_03.rs +++ b/crates/aprender-serve/src/infer/tests_clean_model_03.rs @@ -194,6 +194,7 @@ fn main() { use_mock_backend: false, force_chat_template: false, thinking: None, + logprobs_top_k: 0, }; let debug = format!("{:?}", config); diff --git a/crates/aprender-serve/src/infer/tests_inference_config_02.rs b/crates/aprender-serve/src/infer/tests_inference_config_02.rs index ec5df64d58..989a5f3b9b 100644 --- a/crates/aprender-serve/src/infer/tests_inference_config_02.rs +++ b/crates/aprender-serve/src/infer/tests_inference_config_02.rs @@ -324,6 +324,7 @@ use_mock_backend: false, force_chat_template: false, thinking: None, + logprobs_top_k: 0, }; let debug_str = format!("{:?}", config); assert!(debug_str.contains("trace_verbose")); @@ -355,6 +356,7 @@ use_mock_backend: false, force_chat_template: false, thinking: None, + logprobs_top_k: 0, }; let cloned = config.clone(); assert_eq!(cloned.trace_verbose, config.trace_verbose); diff --git a/crates/aprender-serve/src/infer/tests_run_inference.rs b/crates/aprender-serve/src/infer/tests_run_inference.rs index 487fc413ec..9c006371c5 100644 --- a/crates/aprender-serve/src/infer/tests_run_inference.rs +++ b/crates/aprender-serve/src/infer/tests_run_inference.rs @@ -366,6 +366,7 @@ use_mock_backend: false, force_chat_template: false, thinking: None, + logprobs_top_k: 0, }; let debug = format!("{:?}", config); @@ -401,6 +402,7 @@ use_mock_backend: false, force_chat_template: false, thinking: None, + logprobs_top_k: 0, }; let cloned = original.clone(); diff --git a/crates/aprender-serve/src/safetensors_infer_session_tests.rs b/crates/aprender-serve/src/safetensors_infer_session_tests.rs index 2eb5052b49..5dbf1dcc63 100644 --- a/crates/aprender-serve/src/safetensors_infer_session_tests.rs +++ b/crates/aprender-serve/src/safetensors_infer_session_tests.rs @@ -76,6 +76,7 @@ fn safetensors_cpu_session_generate_leaves_a_witness_entry() { stop_tokens: vec![0], trace: false, logprobs: false, + logprobs_top_k: 0, cancel: crate::generate::CancelToken::never(), }; let turn = session diff --git a/crates/aprender-serve/src/session.rs b/crates/aprender-serve/src/session.rs index e52e9ff035..52a0e7607f 100644 --- a/crates/aprender-serve/src/session.rs +++ b/crates/aprender-serve/src/session.rs @@ -39,7 +39,9 @@ use std::sync::Mutex; use std::time::Instant; use crate::error::{RealizarError, Result}; -use crate::gguf::{OwnedQuantizedModel, QuantizedGenerateConfig}; +use crate::gguf::{ + logsumexp_full, top_k_logprobs, OwnedQuantizedModel, QuantizedGenerateConfig, StepLogprobs, +}; /// One architecture's forward on one backend: the only per-arch code a verb /// reaches, and only through a [`Session`]. @@ -165,7 +167,7 @@ pub struct LayerTiming { } /// What one [`Session::generate`] call did. -#[derive(Debug, Clone, PartialEq, Eq)] +#[derive(Debug, Clone, PartialEq)] pub struct Turn { /// The prompt followed by the generated tokens. pub tokens: Vec, @@ -186,6 +188,24 @@ pub struct Turn { /// prefill and the start of decode (#4954). Earlier than the first /// `on_token` call, so time a caller spends there is never prefill. pub prefill_ended: Instant, + /// One record per generated token when `config.logprobs_top_k` > 0 + /// (#4026), else empty. + pub steps: Vec, +} + +// #4026: the steps of the last turn a session generated on this thread, for +// `apr run`, whose dispatch drops the `Turn` inside per-backend helpers. Same +// idiom as `infer`'s generation-start mark: cleared before a dispatch, taken +// after it. `None` after a dispatch means the engine did not serve it. +std::thread_local! { + static LAST_TURN_STEPS: std::cell::RefCell>> = + const { std::cell::RefCell::new(None) }; +} + +/// Take the steps [`Session::generate`] last recorded on this thread (#4026). +/// `None` when no turn with `logprobs_top_k` > 0 ran since the last take. +pub fn take_last_turn_steps() -> Option> { + LAST_TURN_STEPS.with(std::cell::RefCell::take) } /// A loaded model plus its decode state: the one engine (#4263). @@ -378,13 +398,21 @@ impl Session { /// return it and how many leading tokens were already held. A greedy /// choice with no repetition penalty goes through /// [`ArchForward::forward_greedy`] when the backend has it. + /// + /// With `config.logprobs_top_k` > 0 the choice always comes from host + /// logits (the device argmax never shows them), and one [`StepLogprobs`] + /// is pushed to `steps`, ranked on the logits the FORWARD produced — + /// before the repetition penalty — so it records what the model said, + /// not what the sampler was handed (#4026). fn advance_and_choose( &mut self, tokens: &[u32], config: &QuantizedGenerateConfig, rng: &mut rand::rngs::StdRng, + steps: &mut Vec, ) -> Result<(u32, usize)> { - if is_greedy(config) && !penalty_active(config) { + let recording = config.logprobs_top_k > 0; + if !recording && is_greedy(config) && !penalty_active(config) { let start = if self.extends(tokens) { self.processed.len() } else { @@ -408,13 +436,27 @@ impl Session { } } let (mut logits, reused) = self.advance_to(tokens)?; + // The forward's logits, before the penalty edits them in place: the + // chosen token's logprob is read from these, not the penalised row. + let raw = recording.then(|| logits.clone()); OwnedQuantizedModel::apply_repeat_penalty( &mut logits, tokens, config.repeat_penalty, config.repeat_last_n, ); - Ok((choose_token(&logits, config, rng), reused)) + let chosen = choose_token(&logits, config, rng); + if let Some(raw) = raw { + let lse = logsumexp_full(&raw); + steps.push(StepLogprobs { + step: steps.len(), + chosen, + top: top_k_logprobs(&raw, config.logprobs_top_k), + chosen_logprob: raw.get(chosen as usize).map_or(f32::NAN, |&x| x - lse), + logsumexp_full: lse, + }); + } + Ok((chosen, reused)) } fn reserve(&mut self, positions: usize) -> Result<()> { @@ -505,7 +547,8 @@ impl Session { let prefill_started = Instant::now(); let reused = self.prepare_prompt(prompt)?; let mut rng = rand::rngs::StdRng::seed_from_u64(config.seed); - let (mut next, _) = self.advance_and_choose(prompt, config, &mut rng)?; + let mut steps = Vec::new(); + let (mut next, _) = self.advance_and_choose(prompt, config, &mut rng, &mut steps)?; let prefill_ended = Instant::now(); record(tracer, TraceStep::TransformerBlock, 0, prefill_started); let mut tokens = prompt.to_vec(); @@ -519,7 +562,9 @@ impl Session { // never computes a token it throws away (#4325). if generated > 1 { let clock = std::time::Instant::now(); - next = self.advance_and_choose(&tokens, config, &mut rng)?.0; + next = self + .advance_and_choose(&tokens, config, &mut rng, &mut steps)? + .0; record(tracer, TraceStep::TransformerBlock, generated - 1, clock); } tokens.push(next); @@ -534,6 +579,12 @@ impl Session { break; } } + // The prefill's step when the turn was cancelled before its token was + // pushed is not part of the turn (#4026; #4325 computes no later one). + steps.truncate(tokens.len() - prompt.len()); + if config.logprobs_top_k > 0 { + LAST_TURN_STEPS.with(|c| c.replace(Some(steps.clone()))); + } Ok(Turn { tokens, reused, @@ -541,6 +592,7 @@ impl Session { context_capped, prefill_started, prefill_ended, + steps, }) } diff --git a/crates/aprender-serve/src/session_tests.rs b/crates/aprender-serve/src/session_tests.rs index 715c7cf76f..8193981edf 100644 --- a/crates/aprender-serve/src/session_tests.rs +++ b/crates/aprender-serve/src/session_tests.rs @@ -442,6 +442,119 @@ fn plain_greedy_takes_the_device_argmax_and_a_penalty_or_sampling_does_not() { "a repeat penalty needs the logits" ); } + +/// #4026: `logprobs_top_k` on a greedy config. +fn greedy_logprobs(max_tokens: usize, k: usize) -> QuantizedGenerateConfig { + QuantizedGenerateConfig { + logprobs_top_k: k, + ..greedy(max_tokens) + } +} + +#[test] +fn logprobs_record_one_step_per_generated_token_whose_top1_is_the_choice() { + let _ = take_last_turn_steps(); + let mut s = Session::new(Scripted::new(3, 100)); + let turn = s + .generate(&[7801, 7802], &greedy_logprobs(3, 2), &mut |_| true) + .expect("turn"); + assert_eq!(turn.tokens, vec![7801, 7802, 3, 3, 3]); + assert_eq!(turn.steps.len(), 3, "one record per generated token"); + for (i, step) in turn.steps.iter().enumerate() { + assert_eq!(step.step, i); + assert_eq!(step.chosen, turn.tokens[2 + i]); + assert_eq!(step.top.len(), 2); + assert_eq!( + step.top[0].token_id, step.chosen, + "greedy: top-1 is the choice" + ); + assert_eq!(step.top[0].logit, 1.0); + assert!(step.top[0].logprob > step.top[1].logprob); + } + assert_eq!(take_last_turn_steps(), Some(turn.steps.clone())); + assert_eq!(take_last_turn_steps(), None, "a take clears it"); +} + +#[test] +fn logprobs_zero_records_nothing_and_publishes_nothing() { + let _ = take_last_turn_steps(); + let mut s = Session::new(Scripted::new(3, 100)); + let turn = s + .generate(&[7811], &greedy(2), &mut |_| true) + .expect("turn"); + assert!(turn.steps.is_empty()); + assert_eq!(take_last_turn_steps(), None); +} + +#[test] +fn logprobs_bypass_the_device_argmax_because_it_never_shows_the_logits() { + let mut s = Session::new(DeviceGreedy { + inner: Scripted::new(3, 100), + greedy_answer: 5, + greedy_calls: 0, + }); + let turn = s + .generate(&[7821], &greedy_logprobs(2, 1), &mut |_| true) + .expect("turn"); + assert_eq!(s.engine().greedy_calls, 0); + assert_eq!(turn.tokens, vec![7821, 3, 3], "chosen from the host logits"); + assert_eq!(turn.steps.len(), 2); +} + +#[test] +fn logprobs_rank_what_the_model_said_not_what_the_penalty_left() { + // The prompt holds 3. A positive penalty only shrinks 3's logit toward + // the others' 0.0, never below it, so a negative one is used to flip its + // sign: 1.0 / -1.0 = -1.0, and the argmax falls to token 0. The record + // must still rank 3 first — it is taken before the penalty. + let mut cfg = greedy_logprobs(1, 2); + cfg.repeat_penalty = -1.0; + let mut s = Session::new(Scripted::new(3, 100)); + let turn = s.generate(&[3], &cfg, &mut |_| true).expect("turn"); + assert_eq!(turn.steps[0].chosen, 0, "the penalty moved the choice"); + assert_eq!(turn.steps[0].top[0].token_id, 3, "the record did not move"); + // PRM C11: the chosen token's logprob and the log-sum-exp come from the + // same unpenalised logits as `top` (3 → 1.0, the other 7 of the 8 → 0.0). + let lse = (1.0_f64.exp() + 7.0).ln(); + let s0 = &turn.steps[0]; + assert!( + (f64::from(s0.logsumexp_full) - lse).abs() < 1e-5, + "{}", + s0.logsumexp_full + ); + assert!( + (f64::from(s0.chosen_logprob) + lse).abs() < 1e-5, + "{}", + s0.chosen_logprob + ); + assert!((f64::from(s0.top[0].logprob) - (1.0 - lse)).abs() < 1e-5); +} + +#[test] +fn logprobs_follow_the_logits_a_planted_change_moves_the_record() { + // Negative control: the same prompt against a forward whose argmax was + // moved must record a different top-1, or the record is not reading them. + let run = |next| { + let mut s = Session::new(Scripted::new(next, 100)); + s.generate(&[7831], &greedy_logprobs(1, 1), &mut |_| true) + .expect("turn") + .steps[0] + .top[0] + .token_id + }; + assert_eq!(run(3), 3); + assert_eq!(run(6), 6); +} + +#[test] +fn logprobs_stop_on_a_stop_token_with_its_own_record() { + let mut cfg = greedy_logprobs(4, 1); + cfg.stop_tokens = vec![3]; + let mut s = Session::new(Scripted::new(3, 100)); + let turn = s.generate(&[7841], &cfg, &mut |_| true).expect("turn"); + assert_eq!(turn.tokens, vec![7841, 3]); + assert_eq!(turn.steps.len(), 1); +} // #4445: `apr bench` repeats one prompt and must time a whole prefill each time. // An extending prompt would resume the held state; after forget_prefix it may not. #[test] diff --git a/docs/audits/prm-001/analysis-plan.md b/docs/audits/prm-001/analysis-plan.md new file mode 100644 index 0000000000..c4bd0e97e7 --- /dev/null +++ b/docs/audits/prm-001/analysis-plan.md @@ -0,0 +1,57 @@ +# PRM-001 analysis plan v2 (pre-registered for spec PRM-001 v3, 2026-09-25) + +Frozen together with spec `docs/specifications/PRM-001-prometheus.md` §2–§5, +`crates/aprender-review-experiment/src/stats.rs` and prompt v1 by `docs/audits/prm-001/prereg.lock` +(`rex-prereg-v2`). Any byte change to one of those files is a new spec version (R-1); data +gathered under the old version is `exploratory`. Plan v1 (`docs/audits/rex-001/analysis-plan.md`) +and its lock are superseded, frozen and never edited. + +## Units + +- **Item**: one corpus diff (class P, R or G; stratum S/M/L; split dev/test). +- **Arm**: (lane, model) — `apr-4b` (primary), `apr-9b` (control), `haiku`, `agy`. +- **Receipt**: one `review-experiment-receipt-v1` row per (item, cell, arm). +- **Correct**: a parsed `FAIL` on a P/R item, a parsed `PASS` on a G item. + `Unparsed`, `NotRun{…}`, `Refused` and `ContextOverflow` are never correct and never a + FAIL (R-6); they are counted and reported per state. + +## Tests (code: `stats.rs`) + +| H | Statistic | Function | Rejection rule | +|---|---|---|---| +| H1 invariance | test items whose VERDICT differs from reference cell C2 (v2; byte divergence is descriptive) | `verdict_divergence`, `count_rule_p` | count > 0 | +| H2 determinism | divergent items on the 10 % same-cell re-run | `count_rule_p` | count > 0 | +| H3 size | McNemar exact, one-sided, `b` = 9B right & 4B wrong | `discordant`, `mcnemar_exact_one_sided` | Holm-adjusted p ≤ 0.05 | +| H4 usefulness | precision(4B) − min(precision(haiku), precision(agy)), min computed INSIDE each resample; paired bootstrap, 10 000 resamples, seed 4354 | `vs_voters_parsed`, `bootstrap_vs_min_voter(Rate::Precision)` | holds iff the Holm-adjusted one-sided p (share of resamples < 0) ≤ 0.05; the 95 % CI is report-only | +| H5 recall | recall(4B) − recall(Claude lane) on class-R items only, paired bootstrap as H4 | `paired_parsed_class(.., Class::R)`, `paired_bootstrap_diff(Rate::Recall)` | as H4. Class-P recall and precision are descriptive | +| H6 interference | mean(co-located) − mean(alone), 5 + 5 runs | `interference` | slowdown > 2 × sd(alone) | +| H7 independence (a gate, outside the family) | κ_err(apr, each voter) vs max κ_err over voter–voter pairs, same items | `error_pairs`, `error_kappa`, `h7_holds` | holds iff every apr–voter κ ≤ the max voter–voter κ; undefined κ is undecided, never holds | + +Family H1–H6, Holm step-down (`holm`), α = 0.05. H1, H2 and H6 are deterministic rules and +enter the family as p = 0 (rejected) or p = 1 (not rejected). + +For H4 (v2) there is no fixed comparison lane: each resample subtracts whichever voter has the +lower precision ON THAT RESAMPLE. Items where apr or either voter is not a parsed verdict are +dropped and counted. For H5 the comparison is the Claude lane (Haiku 4.5), pairwise-dropped. + +## Descriptive + +Per cell: p50/p95 review wall-clock (nearest rank, `percentile`) with a bootstrap CI of the +p95 (same seed), TTFT, prefill and decode tok/s, peak anon memory, parse rate, and the ratio +to llama.cpp `d1d3c3396` on the same cell. Recall/precision/false-refute/localization with +Wilson 95 % intervals (`wilson`). + + +Sensitivity: every per-cell number is reported twice — all runs (primary) and runs with +loadavg1 < 0.5 × physical cores (secondary). Decisions read the primary column only. + +## Sample-size rule (§2.2) + +After PRM-05 (was REX-05), project the test-split Wilson half-width on recall at the pilot's point estimate +(`wilson_half_width`). If > 0.12, the corpus grows; hypotheses do not change. + +## Seeds + +- Run order per cell: SplitMix64 seed 4354 (the epic number), Fisher–Yates. +- Determinism re-run subset: the first 10 % of the seed-4354 order of test items. +- Bootstrap: seed 4354. diff --git a/docs/audits/prm-001/prereg.lock b/docs/audits/prm-001/prereg.lock new file mode 100644 index 0000000000..417025d834 --- /dev/null +++ b/docs/audits/prm-001/prereg.lock @@ -0,0 +1,7 @@ +# PRM-001 pre-registration lock (rex-prereg-v2) over spec PRM-001 v3. Written by PRM-00b; v1 had 0 data rows. +# A mismatch against the tree is a new spec version (R-1), never an edit here. +spec_s2_s5=f53db26dee2d5f66ab597f2da012fa13a87a0a5002532b267a28091f0e5626f7 +stats_rs=abb51113c5971f9460f8c990fdb58094ca04943cbdb863f00a916472d9b088cc +analysis_plan=2b84b3b0026300364bdcf09e66967a25c43dc47749abf790f7c238380e9f2969 +prompt_v1=dd9d67c2df458d92d66aed65fd7f3f6f2e5602499c05e6b0967465b9bd2662e6 +prereg_sha=f0b4e4ac07e605e313e61ce5efa9e43f7339ca5a5e5275b0d5f06b0277011041 diff --git a/docs/audits/prm-001/prm-09/RECEIPT.md b/docs/audits/prm-001/prm-09/RECEIPT.md new file mode 100644 index 0000000000..ca3c995cfe --- /dev/null +++ b/docs/audits/prm-001/prm-09/RECEIPT.md @@ -0,0 +1,31 @@ +# PRM-09 — ladder extension: the three planted reports (rex-promotion-ladder-v2 1.1.0) + +**Outcome: acceptance met.** The PRM-001 §9 row reads "H4 only → tripwire; H4+H5+H7 → tie-breaker; lambda primary → refused". All three run through the real binary, `rex ladder`, and not only through unit tests. + +The reports below are `prm-001-report-v1` documents (spec PRM-001, version 3). They carry the **live** prereg lock, `docs/audits/prm-001/prereg.lock` `prereg_sha` `f0b4e4ac…`, so they exercise the gate and not the stale-lock refusal. Every run is recorded verbatim in `runs.txt`. + +| Planted report | sha256 (prefix) | `--current` | mode | why | +|---|---|---|---|---| +| `planted-h4-only.json`: H4 holds (Holm p 0.01), within budget, gx10-cpu primary | `3922acbb9040d0f3` | shadow | **tripwire** | H5, H7, H1 and H2 are not in the report | +| `planted-h4-h5-h7.json`: adds H5 (p 0.04), H7, H1, H2; availability 0.97 | `b4e225242cd0f0eb` | tripwire | **tie-breaker** | vote needs ≥ 20 decided tie-breaks, and it has 0 | +| `planted-lambda-primary.json`: the same report with primary `lambda-cuda` | `e9e875dc457aa891` | tripwire | **tripwire (refused)** | `S-4: primary cell lambda-cuda is lambda or unnamed; lambda is shadow-only (R-9)` | +| the same lambda report | ″ | tie-breaker | **tripwire** | S-4 caps at once. It is the one exception to the one-rung step | +| the H4+H5+H7 report | ″ | shadow | tripwire | the positive control for the step rule: one rung per report, Shadow → Tripwire, never straight to TieBreaker | + +## Defect fixed on the way + +The S-4 reason printed the Rust Debug rendering of the cell, `primary cell Some("lambda-cuda")`. That string would have reached the quorum gate and the arbiter's receipts. It now names the cell as written, or `(unnamed)`. The fix is FALSIFY-RXG-006 in contract 1.1.0: planted RED in `cf7bdbfbb`, then GREEN. + +## Named follow-ups (not in this change) + +- **pi#428**: the paiml-implement quorum gate reads `rex ladder … | .mode`. +- **infra#1095**: the arbiter reads the same `.mode`. + +Both consume the `.mode` field and must never re-derive it (contract description). + +## Gates + +- `cargo test -p aprender-review-experiment --lib`: 167 passed. +- `cargo clippy -p aprender-review-experiment --all-targets -- -D warnings`: clean. +- `cargo fmt`: clean. +- `pv validate contracts/rex-promotion-ladder-v2.yaml`: valid. diff --git a/docs/audits/prm-001/prm-09/planted-h4-h5-h7.json b/docs/audits/prm-001/prm-09/planted-h4-h5-h7.json new file mode 100644 index 0000000000..0a303671c1 --- /dev/null +++ b/docs/audits/prm-001/prm-09/planted-h4-h5-h7.json @@ -0,0 +1,50 @@ +{ + "spec": "PRM-001", + "version": 3, + "prereg_sha": "f0b4e4ac07e605e313e61ce5efa9e43f7339ca5a5e5275b0d5f06b0277011041", + "lane": { + "rung": "shadow", + "availability_7d": 0.97 + }, + "speed": { + "within_budget": true + }, + "results": { + "hypotheses": [ + { + "id": "H4", + "ci": [ + 0.01, + 0.2 + ], + "p_holm": 0.01, + "verdict": "holds" + }, + { + "id": "H5", + "ci": [ + 0.01, + 0.2 + ], + "p_holm": 0.04, + "verdict": "holds" + }, + { + "id": "H7", + "verdict": "holds" + }, + { + "id": "H1", + "verdict": "holds" + }, + { + "id": "H2", + "verdict": "holds" + } + ] + }, + "hardware_ruling": { + "primary": "gx10-cpu", + "rung_eligible": "vote" + } +} diff --git a/docs/audits/prm-001/prm-09/planted-h4-only.json b/docs/audits/prm-001/prm-09/planted-h4-only.json new file mode 100644 index 0000000000..4a63a57df3 --- /dev/null +++ b/docs/audits/prm-001/prm-09/planted-h4-only.json @@ -0,0 +1,29 @@ +{ + "spec": "PRM-001", + "version": 3, + "prereg_sha": "f0b4e4ac07e605e313e61ce5efa9e43f7339ca5a5e5275b0d5f06b0277011041", + "lane": { + "rung": "shadow", + "availability_7d": 0.97 + }, + "speed": { + "within_budget": true + }, + "results": { + "hypotheses": [ + { + "id": "H4", + "ci": [ + 0.01, + 0.2 + ], + "p_holm": 0.01, + "verdict": "holds" + } + ] + }, + "hardware_ruling": { + "primary": "gx10-cpu", + "rung_eligible": "vote" + } +} diff --git a/docs/audits/prm-001/prm-09/planted-lambda-primary.json b/docs/audits/prm-001/prm-09/planted-lambda-primary.json new file mode 100644 index 0000000000..b1405e0394 --- /dev/null +++ b/docs/audits/prm-001/prm-09/planted-lambda-primary.json @@ -0,0 +1,50 @@ +{ + "spec": "PRM-001", + "version": 3, + "prereg_sha": "f0b4e4ac07e605e313e61ce5efa9e43f7339ca5a5e5275b0d5f06b0277011041", + "lane": { + "rung": "shadow", + "availability_7d": 0.97 + }, + "speed": { + "within_budget": true + }, + "results": { + "hypotheses": [ + { + "id": "H4", + "ci": [ + 0.01, + 0.2 + ], + "p_holm": 0.01, + "verdict": "holds" + }, + { + "id": "H5", + "ci": [ + 0.01, + 0.2 + ], + "p_holm": 0.04, + "verdict": "holds" + }, + { + "id": "H7", + "verdict": "holds" + }, + { + "id": "H1", + "verdict": "holds" + }, + { + "id": "H2", + "verdict": "holds" + } + ] + }, + "hardware_ruling": { + "primary": "lambda-cuda", + "rung_eligible": "vote" + } +} diff --git a/docs/audits/prm-001/prm-09/runs.txt b/docs/audits/prm-001/prm-09/runs.txt new file mode 100644 index 0000000000..16648ce615 --- /dev/null +++ b/docs/audits/prm-001/prm-09/runs.txt @@ -0,0 +1,5 @@ +rex ladder planted-h4-only.json --current shadow → rc=0 {"mode":"tripwire","reasons":["H5: not in the report","H7: not in the report","H1: not in the report","H2: not in the report"]} +rex ladder planted-h4-h5-h7.json --current tripwire → rc=0 {"mode":"tie-breaker","reasons":["tie_breaks_decided 0 is not ≥ 20"]} +rex ladder planted-lambda-primary.json --current tripwire → rc=0 {"mode":"tripwire","reasons":["tie_breaks_decided 0 is not ≥ 20","S-4: primary cell lambda-cuda is lambda or unnamed; lambda is shadow-only (R-9)"]} +rex ladder planted-lambda-primary.json --current tie-breaker → rc=0 {"mode":"tripwire","reasons":["tie_breaks_decided 0 is not ≥ 20","S-4: primary cell lambda-cuda is lambda or unnamed; lambda is shadow-only (R-9)"]} +rex ladder planted-h4-h5-h7.json --current shadow → rc=0 {"mode":"tripwire","reasons":["tie_breaks_decided 0 is not ≥ 20","one rung per report: Shadow → Tripwire, not TieBreaker"]} diff --git a/docs/audits/prm-001/s1-v2/RECEIPT.md b/docs/audits/prm-001/s1-v2/RECEIPT.md new file mode 100644 index 0000000000..54ca071e0e --- /dev/null +++ b/docs/audits/prm-001/s1-v2/RECEIPT.md @@ -0,0 +1,28 @@ +# PRM-S1 v2 — set a4910a65, per-tag receipts (#4447, #4354) + +Set: `set-a4910a65.jsonl` (review-replay-v2, seed 4354, 50 items per stratum × 2k/8k/16k/32k = 200). +`sha256sum set-a4910a65.jsonl` = `a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632`. +Every row below carries that `set_sha`. The diffs themselves are not here: they are the corpus +items, addressed by `diff_sha256`. + +Cell `gx10-cuda` (GB10), GGUF sha256 `00fe7986…ef11a4`. Competitor: llama.cpp `d1d3c3396`, run in +the same session on the same GGUF with identical prompt ids (`prompt_ids_sha` `f3c20136…`). +The rows hold timings, token counts, peak RSS and the parsed verdict state only. There are no +prompts or completions; transcripts stay private. + +| apr build | rows (apr/llama) | wall p50 s | wall p95 s [95% CI] | ttft ratio | decode tps ratio | +|---|---|---|---|---|---| +| `96eb79f51` (0.69.3) | 200/200 | 26.61 | 69.36 [59.49, 82.56] | n/a: apr untimed before SRV-TIM-001 | n/a | +| `33b5e5b24` (0.70.0 dev, SRV-TIM-001; apr sha256 `38313994…68ce`) | 200/200 | 26.98 | 68.83 [60.91, 85.64] | 4.40× | 0.216 (`evidence/prm-001/s1-v2/33b5e5b24/speed-block.json`) | + +`33b5e5b24` is the build of record. The `speed-block.json` in each `evidence/prm-001/s1-v2//` directory is the §9 block, +produced by `replay summary --rows rows-both --voter LANE=FILE`. + +## Queue budget (voter p95) + +`evidence/prm-001/s1-v2/33b5e5b24/voters.txt` is `replay voters` over `evidence/pr-review/` at main +`8cf336c60a2b5071b855af99c99def618fb1f41b`. Of the 48 quorum receipts there, 23 carry a +`duration_seconds`, and all 23 are `antigravity` lanes; the Claude lanes record none. +Budget = the largest lane p95 = **601.6 s (antigravity)**. apr's wall p95 is 68.8 s, so it +is **within budget** at both builds. A Claude-lane latency, once receipts carry one, can only +change which lane sets the budget. diff --git a/docs/audits/prm-001/s1-v2/set-a4910a65.jsonl b/docs/audits/prm-001/s1-v2/set-a4910a65.jsonl new file mode 100644 index 0000000000..bc4642af69 --- /dev/null +++ b/docs/audits/prm-001/s1-v2/set-a4910a65.jsonl @@ -0,0 +1,201 @@ +{"per_stratum":50,"schema":"review-replay-v1","seed":4354,"version":"review-replay-v2"} +{"group":"paiml/aprender#3409","diff_sha256":"f1ca076d355345060d79de6ab291b725d78bfd3596f2cfc01444d40d11e2ea79","input_tokens":980,"stratum":"2k"} +{"group":"paiml/aprender#2192","diff_sha256":"7788e95d4d3ecd6288d7d51fa0ed8459158b9b7ff7b94bbbffd244bbb7e1dad1","input_tokens":689,"stratum":"2k"} +{"group":"paiml/aprender#2823","diff_sha256":"366eed3dd5e0055ebfdd837af58bd800debdd62e4c0a7f7da2dffa84d06ada9f","input_tokens":1396,"stratum":"2k"} +{"group":"paiml/aprender#2342","diff_sha256":"f9d7c31e37b0ed40fbfd87af8984c8860ec6c047dc68a8b55bcb88ee67ee6452","input_tokens":711,"stratum":"2k"} +{"group":"paiml/aprender#2351","diff_sha256":"f8879c54b81ffd2aaeaf6398391a818b5f56d1c37ffd4a71100bf19864c1b902","input_tokens":948,"stratum":"2k"} +{"group":"paiml/aprender#1942","diff_sha256":"296c5544001c3a5e623d3bc13a3b59728bad8d086d6f5baaa8ec44409d3e5cea","input_tokens":320,"stratum":"2k"} +{"group":"paiml/aprender#2345","diff_sha256":"275cbe4c206c2eb59aca3ca92f0bbd7f171bc949a205afe8e44882e4e56fdc5c","input_tokens":1902,"stratum":"2k"} +{"group":"paiml/aprender#2321","diff_sha256":"2d012d3f8e232bfdbed32db1d29426b65b9c7725d2a001308ac4fb59f8cdafa9","input_tokens":1825,"stratum":"2k"} +{"group":"paiml/aprender#2341","diff_sha256":"e777a873cb9229ece5b23335a8dd1bb421f1b5e298f76024a7af842e2cfcadfb","input_tokens":1887,"stratum":"2k"} +{"group":"paiml/aprender#3066","diff_sha256":"a7a62e12ad053746008e769e95d9c06f39e8b741a52ad4f650d05d1ca720c754","input_tokens":1824,"stratum":"2k"} +{"group":"paiml/aprender#2329","diff_sha256":"66ec2c8042874218dcf97952a6a557fe1223cb59db4a28bf7615cd2a6a4f269b","input_tokens":361,"stratum":"2k"} +{"group":"paiml/aprender#3192","diff_sha256":"5f449a13b53a4663c67c7b0cea0588720e0a2e63e79561ba6adcd58d9bd87d77","input_tokens":1742,"stratum":"2k"} +{"group":"paiml/aprender#2805","diff_sha256":"3b86db746737d9338f2ac5c3c0aa7184acd7f05314b46fb8ae1a8bbdd83ac4ad","input_tokens":655,"stratum":"2k"} +{"group":"paiml/aprender#2485","diff_sha256":"a5f04841ad4bdf65c809d00511d1b4669ed6f52b6ec0be2eac5ee0417b29297f","input_tokens":1628,"stratum":"2k"} +{"group":"paiml/aprender#1988","diff_sha256":"f1206fea7a5282d2607d846f5d4f0a9fe7b6982f8d0ab75a42632d7a8ca4e487","input_tokens":1003,"stratum":"2k"} +{"group":"paiml/aprender#2312","diff_sha256":"993b00de597f00fefd6e145197fc5659223b89690cc0a671c636d955b4c961c5","input_tokens":1584,"stratum":"2k"} +{"group":"paiml/aprender#3188","diff_sha256":"47015b6bbe7f1a4b0b27cd8396c3430142487ab219124adf8c7f331e365c7ec3","input_tokens":1658,"stratum":"2k"} +{"group":"paiml/aprender#2124","diff_sha256":"9af203f89adb6dc0348f46161c0da6bdf7ebea3752a638220571ad3a65854634","input_tokens":1709,"stratum":"2k"} +{"group":"paiml/aprender#3326","diff_sha256":"34d4c5a7a0e46bbd1b00935ef3000cc8ddba8dd64cbdffa9e01f1207863f945c","input_tokens":1451,"stratum":"2k"} +{"group":"paiml/aprender#3414","diff_sha256":"167d286e0f8a0bae32d26b9d8abc056df02e030188cf395ff255acc3cb1d640d","input_tokens":942,"stratum":"2k"} +{"group":"paiml/aprender#2460","diff_sha256":"fab75467c60ebd40cc94c13296129957a26a7eb9865a58175dd09db9610f1631","input_tokens":1051,"stratum":"2k"} +{"group":"paiml/aprender#2333","diff_sha256":"c274c3708f67fd97e467895e6c780665af6a13a28c8a727d25ea1863e3328dc8","input_tokens":1542,"stratum":"2k"} +{"group":"paiml/aprender#2219","diff_sha256":"11ca5b1e87d2bbf217c1fd81b2c2ea63a96390dea93e9d9258aa7dff541f8de8","input_tokens":1529,"stratum":"2k"} +{"group":"paiml/aprender#2718","diff_sha256":"7ba9cfea210f85f585654545dc13dc487fe37b56924c0996577c87a463100bac","input_tokens":272,"stratum":"2k"} +{"group":"paiml/aprender#2564","diff_sha256":"b1c9fb8c5a26aa63dcdb57e6ec6ce35a2c5a73287516f57082f77944c007add1","input_tokens":790,"stratum":"2k"} +{"group":"paiml/aprender#3407","diff_sha256":"9d82e15d5df69ff6b6a2f1d4dd81ba144dca2978ee4f352de1b134cf86a4bfdd","input_tokens":1613,"stratum":"2k"} +{"group":"paiml/aprender#1916","diff_sha256":"4dca9a17db342792f25605430639922e342855e3341653071266173c29d9a859","input_tokens":1180,"stratum":"2k"} +{"group":"paiml/aprender#3299","diff_sha256":"e8e5551522f3e72418df3e0fa2227dded865de9949b65a4b85becc6cd3acbc35","input_tokens":382,"stratum":"2k"} +{"group":"paiml/aprender#3198","diff_sha256":"e0cf54bdaa7fbbbd164db0ebf846b77b0b78b65b8bfd4a392fcc3171c64a2504","input_tokens":603,"stratum":"2k"} +{"group":"paiml/aprender#2316","diff_sha256":"7492d83b3314d90dae86dcb3530baaa502e8d425427daa63a8e9fc03de96b069","input_tokens":1293,"stratum":"2k"} +{"group":"paiml/aprender#2259","diff_sha256":"0d9bcac1ca95dc9fb8293713b22add78150aaaacfa6a269e8065ea5605feee17","input_tokens":962,"stratum":"2k"} +{"group":"paiml/aprender#2538","diff_sha256":"3f51712000a76356352961b38d5b137cdf5179679655ed3fbfef2acad4eeb0e8","input_tokens":686,"stratum":"2k"} +{"group":"paiml/aprender#1937","diff_sha256":"b7e5b398d98a7f3dc848eee0e437e07104d372e9365e6f3ce0c101e6667ed890","input_tokens":306,"stratum":"2k"} +{"group":"paiml/aprender#3417","diff_sha256":"903ef35fb09bf1cb357547ce61b9b7828b90c24845aa5f9fecd8923e022be75f","input_tokens":957,"stratum":"2k"} +{"group":"paiml/aprender#2253","diff_sha256":"34186bc542a9bc6191d88846aadf532b79f67209e4b17474d1d4b0a1cf87cc8c","input_tokens":524,"stratum":"2k"} +{"group":"paiml/aprender#3426","diff_sha256":"c9a02d51ce74d1c65af0fa596da511b8d2a571aa8f65f8d380ed9838b4c5243e","input_tokens":1021,"stratum":"2k"} +{"group":"paiml/aprender#2263","diff_sha256":"d09cb9713ee02abec5dce5dd4cd0b58237e449a0032fdaf96dcc6ac95a0a6101","input_tokens":482,"stratum":"2k"} +{"group":"paiml/aprender#2659","diff_sha256":"a889b6ddfa18809d32510029c8f832ddf1dc03f3209feee377d47043c757104e","input_tokens":602,"stratum":"2k"} +{"group":"paiml/aprender#2298","diff_sha256":"c1d14d2cd0dbbdd90638158ee715c79fc76baa8ff2c6322160dac0bb5bd27771","input_tokens":330,"stratum":"2k"} +{"group":"paiml/aprender#3309","diff_sha256":"955705510e7439ac4b8e27505f87bcb7d3c71b677cb11bddc4a91d5c44e16a04","input_tokens":1790,"stratum":"2k"} +{"group":"paiml/aprender#3074","diff_sha256":"bf94290e1f98fc502cb4f962bce2883c3eaa9ca2fe6cb67f6b87c7083d3ec0e1","input_tokens":1272,"stratum":"2k"} +{"group":"paiml/aprender#1992","diff_sha256":"258be4b71904b9f67bdda4ea10d76b9be53c374b04a9d486facd87a97fab0756","input_tokens":1211,"stratum":"2k"} +{"group":"paiml/aprender#2720","diff_sha256":"1bf59f2a1774a0efe188bbfa1ac4a7af14b2059017de009fc98df2a6b8382017","input_tokens":1789,"stratum":"2k"} +{"group":"paiml/aprender#2126","diff_sha256":"305cdbe50000c485ff10ec6127fbe0744109a126be589ccbb6d243834c3c13ef","input_tokens":1036,"stratum":"2k"} +{"group":"paiml/aprender#2066","diff_sha256":"7680438d94dac7e6c61561a71575435fcd22d1543b508ec9434d00f686511896","input_tokens":917,"stratum":"2k"} +{"group":"paiml/aprender#2120","diff_sha256":"d928b82fd55ca6c509650e822de0f3946d5ccb3d5fbbabba30784e0f6483e2ff","input_tokens":1426,"stratum":"2k"} +{"group":"paiml/aprender#2468","diff_sha256":"64cce1452c38a191c5f249f069acb21eb4098bd2620cddec04b346c3075849dc","input_tokens":611,"stratum":"2k"} +{"group":"paiml/aprender#2060","diff_sha256":"69b831cc2059acf0f511c8d788337f36e8ef890f6b86b91f6ffa159b8369d1e5","input_tokens":1937,"stratum":"2k"} +{"group":"paiml/aprender#2057","diff_sha256":"14d125f8a319774d2ddc89c330d806c3371695a96ec1cc8757cb63f20c9afe37","input_tokens":577,"stratum":"2k"} +{"group":"paiml/aprender#2059","diff_sha256":"0b54a849b9e7541ec803e57fab1147aee8d209935102b9230c0a024229ccdac7","input_tokens":563,"stratum":"2k"} +{"group":"paiml/aprender#2123","diff_sha256":"ae0f31c8b5b43715e0f5f31727de5648fe149c06faef682adb8898f4dddc0d59","input_tokens":2157,"stratum":"8k"} +{"group":"paiml/aprender#2019","diff_sha256":"6324ab9b418859326df39301f2fc9ca30815280479d95ad375d91e6252a4b82b","input_tokens":3571,"stratum":"8k"} +{"group":"paiml/aprender#2848","diff_sha256":"187723585ce31ae0a9dfb2bd1c94ee372724ba456ade6c3ef28816fb1e369e4d","input_tokens":2264,"stratum":"8k"} +{"group":"paiml/aprender#3136","diff_sha256":"dd16771d607cda7d13b2528bec93be7261e9d739a6d40c855b0794500da92522","input_tokens":4956,"stratum":"8k"} +{"group":"paiml/aprender#2225","diff_sha256":"40ce4b133ac399054d67393d53e91be97d3cafaa33622094f90a35fdea53c703","input_tokens":6939,"stratum":"8k"} +{"group":"paiml/aprender#2258","diff_sha256":"bc9a159757fe4720bfdc9073b5c2d8879c8adec829ab9308b1b1031768b5c3f0","input_tokens":6518,"stratum":"8k"} +{"group":"paiml/aprender#2271","diff_sha256":"33b63a5283685481a92f5226d573249f59330892462fc75a3c5ff14e9c61ba82","input_tokens":6789,"stratum":"8k"} +{"group":"paiml/aprender#2261","diff_sha256":"df48ffe5f40ed18ea677fa829b51ee4eaaee5c8caa5a7462bbe212d83396c540","input_tokens":3441,"stratum":"8k"} +{"group":"paiml/aprender#2037","diff_sha256":"cbc1450d830a7bda671630905c4b86039bf483e305a0d8fecf6971de02ad0406","input_tokens":6777,"stratum":"8k"} +{"group":"paiml/aprender#1991","diff_sha256":"62b4bd91655d82632f2ad6754eac98e25881dcdfeb3eff15172402b9da179800","input_tokens":3912,"stratum":"8k"} +{"group":"paiml/aprender#1972","diff_sha256":"a6c9bb4b1ea341e343c764aed2b694ca7f4c6ab69461291cf6e20a2d1fe53a47","input_tokens":3039,"stratum":"8k"} +{"group":"paiml/aprender#2079","diff_sha256":"1239ceb996f617d8cda28ed536075072f460a875c5a4a25a2ef422a9d923043e","input_tokens":4298,"stratum":"8k"} +{"group":"paiml/aprender#2273","diff_sha256":"90134b8a8a941ee22156f752e986d85a39c92f11186fff94174ce25e7fc2c65d","input_tokens":3140,"stratum":"8k"} +{"group":"paiml/aprender#2364","diff_sha256":"b3c917322e600c6a75b1fdabd254bbb864628227545b9f51c05fedcb33835e55","input_tokens":2743,"stratum":"8k"} +{"group":"paiml/aprender#2740","diff_sha256":"38f0a202313d9e2120e55b92747ced425cac793e09a4db44c212449b1cd8566e","input_tokens":4586,"stratum":"8k"} +{"group":"paiml/aprender#2114","diff_sha256":"e12a747f914c805b9dcbc4349d5ed7fe706e6d0334848a677837cef3d8b95400","input_tokens":2465,"stratum":"8k"} +{"group":"paiml/aprender#2242","diff_sha256":"a74db9b8ff37392fa217b05b5e70a2564d3c8a6960f02eebe3db8016184e114a","input_tokens":5959,"stratum":"8k"} +{"group":"paiml/aprender#3280","diff_sha256":"27565a444276a49af4557cade6d92398cbccd183c6366e78a5453683923bfe6b","input_tokens":7873,"stratum":"8k"} +{"group":"paiml/aprender#3163","diff_sha256":"e0d0c14ea617d00ecec166c8e598898b3d1b757d639df1729ddf1e3af37872cd","input_tokens":2825,"stratum":"8k"} +{"group":"paiml/aprender#2133","diff_sha256":"7269a7b0e6abf53a633d344677435b768b4342ad507d8833da2aa1b5b90747ee","input_tokens":4637,"stratum":"8k"} +{"group":"paiml/aprender#2031","diff_sha256":"f87c2ffe3aafeec5f8308352f5190b3c8ec03ed0eafd6d94b000ef39bb3d922f","input_tokens":4973,"stratum":"8k"} +{"group":"paiml/aprender#2239","diff_sha256":"c7323e9dc8cc06122998a5fb83ccb487d2538b7444f9852de80f63983e93dc44","input_tokens":3806,"stratum":"8k"} +{"group":"paiml/aprender#2425","diff_sha256":"3bed1317707a01f00daf828a5434d3681278c66100d991520153baa4dc7ebd61","input_tokens":2063,"stratum":"8k"} +{"group":"paiml/aprender#1974","diff_sha256":"a974a820da6ee897219e5353ad22358cd5cec02a3534baa60388a3f554850d96","input_tokens":4339,"stratum":"8k"} +{"group":"paiml/aprender#2176","diff_sha256":"1c50664db77045b861287a6f03ded90d514b8d0a32f04336caa5c9c90777b18c","input_tokens":2215,"stratum":"8k"} +{"group":"paiml/aprender#2250","diff_sha256":"7e09255545a53208daf6239daa699089bbd30c3ec111d44a59cbe007a8004a57","input_tokens":3917,"stratum":"8k"} +{"group":"paiml/aprender#2365","diff_sha256":"1aee819c753ccc3dd6976511c879d33fd99478a25bfe8c5639d006aebe6d00ad","input_tokens":4804,"stratum":"8k"} +{"group":"paiml/aprender#2046","diff_sha256":"e92967237c1b1abeb4a548c343928a0afda2c3df6c49e94cec87b7146276b045","input_tokens":4024,"stratum":"8k"} +{"group":"paiml/aprender#2791","diff_sha256":"5ef3cfe455ce3ddedb5fd853a62ccbda685b16f06e7ec8818c46f639474fd302","input_tokens":2813,"stratum":"8k"} +{"group":"paiml/aprender#2061","diff_sha256":"08db5d324448f2faa678eca074285b6275f42db2c667f594ede9fa4adfc9c566","input_tokens":5696,"stratum":"8k"} +{"group":"paiml/aprender#2367","diff_sha256":"bc995711883be2b01bb079a4433bb34b0eb50a9cda6440856b8c0665d7a4afb5","input_tokens":2793,"stratum":"8k"} +{"group":"paiml/aprender#2220","diff_sha256":"9f29c99424409b984aeea5524ec870e016e2e16c617351a938ed700a057ce9ba","input_tokens":7481,"stratum":"8k"} +{"group":"paiml/aprender#2197","diff_sha256":"3d765c54312ef5c279e895fb042a9c08ef9bcd6b44e1098b9db17f68d379f845","input_tokens":3512,"stratum":"8k"} +{"group":"paiml/aprender#2001","diff_sha256":"b66cef24141dfcf0cd4776c23e13ac2a297f86fb966ea2f474d3ba2b22ac4689","input_tokens":2211,"stratum":"8k"} +{"group":"paiml/aprender#2103","diff_sha256":"bf69f8045076d4b114aef30013614158379bdcec1dddb96497b724fd4bd16df2","input_tokens":4755,"stratum":"8k"} +{"group":"paiml/aprender#2154","diff_sha256":"0ba74445847c37901a423c32a811b0f15026d75d3dd2225fd67ab1a474f97ae2","input_tokens":5531,"stratum":"8k"} +{"group":"paiml/aprender#3507","diff_sha256":"a08ed5ccc3331e0a7179c7d49737607bccb9a9cfbc16c6533afacfa6072b133f","input_tokens":2328,"stratum":"8k"} +{"group":"paiml/aprender#2102","diff_sha256":"b0bdb087e1d95680866dad2b47061fa38e925929fe937f39b834992f44838399","input_tokens":6144,"stratum":"8k"} +{"group":"paiml/aprender#2048","diff_sha256":"6e56cee326693c4915a7feef43eee6a597f75ff2e54f970e8f3197135869aed1","input_tokens":7040,"stratum":"8k"} +{"group":"paiml/aprender#2249","diff_sha256":"c81288783ae98e2b5a143cdfc4738ce6c817ae09e2b01a0979e18a0de5f90bb6","input_tokens":7405,"stratum":"8k"} +{"group":"paiml/aprender#2173","diff_sha256":"0354fce0d010d6d69a1539589a7d6046495172ee6865f418b0a39c852edae583","input_tokens":4041,"stratum":"8k"} +{"group":"paiml/aprender#3214","diff_sha256":"b45b72bc1a202cb0504dfd27dd16f60af1ec09570fe31e7ecd927168837a02cf","input_tokens":5995,"stratum":"8k"} +{"group":"paiml/aprender#2155","diff_sha256":"b6dabc525c8486ab7bab78b2968c241327371b5c234ca1b8a87adb93bb8768ab","input_tokens":4835,"stratum":"8k"} +{"group":"paiml/aprender#2118","diff_sha256":"fd684a94b71725b926c4fb5be284b7c83518a537c466c552d7dbc8c9c1f64bec","input_tokens":2284,"stratum":"8k"} +{"group":"paiml/aprender#2007","diff_sha256":"c5a7b5a62a110bae58ca259b02ea467d3dcdbde7ad3308a657e8028e63322e39","input_tokens":4001,"stratum":"8k"} +{"group":"paiml/aprender#2368","diff_sha256":"b71b016e3bf4e4aa82e2c46d8b1c2f137099cf683949b4c65e5900bad8de95ce","input_tokens":2053,"stratum":"8k"} +{"group":"paiml/aprender#2188","diff_sha256":"611ff958e756f16b6143ffe425e3351e94a71034d0306f0e00ebd25696e66a31","input_tokens":3563,"stratum":"8k"} +{"group":"paiml/aprender#2069","diff_sha256":"1f97a6ca5af77396cc4febaacda124d2f993c18d3e3b96e3e0b28ff92f20610d","input_tokens":7875,"stratum":"8k"} +{"group":"paiml/aprender#2092","diff_sha256":"3811dbd8d38716269bb87eca6a6039f05923f1c129d3d2b09659f4cff15dfc2e","input_tokens":4568,"stratum":"8k"} +{"group":"paiml/aprender#3503","diff_sha256":"f65d88532cd6ca202557458343fb035dbe140810881382b7ce5d8c668afda27d","input_tokens":3958,"stratum":"8k"} +{"group":"paiml/aprender#2140","diff_sha256":"6bcf0a0fb95dfe5e90aa366595a006bc443f7a3d9de3c86ba176541bdbbc01cc","input_tokens":15754,"stratum":"16k"} +{"group":"paiml/aprender#3334","diff_sha256":"c70e109e3c965254163267ebeee325ed9dfebe4479eb874c38a592a3488af34e","input_tokens":11111,"stratum":"16k"} +{"group":"paiml/aprender#3538","diff_sha256":"9ec99261cd227d8f5e528bab7391b88aaafa4483233ccb0731a46b4f64bee8dc","input_tokens":13572,"stratum":"16k"} +{"group":"paiml/aprender#2203","diff_sha256":"9444f47e49f1779a103b611560b50ba89a8f20df19b976c9d4e06c2b01e103a5","input_tokens":8880,"stratum":"16k"} +{"group":"paiml/aprender#2185","diff_sha256":"5e5b0935bd95c58dc95d3caebb22a4f41d0573018dff4ce1633bb7668afdd1a2","input_tokens":14528,"stratum":"16k"} +{"group":"paiml/aprender#3352","diff_sha256":"df963cc72999db8bd18665c41caca7b3cd058ec66923e7f5f3c161415d0cd4a2","input_tokens":11623,"stratum":"16k"} +{"group":"paiml/aprender#2857","diff_sha256":"675cb632f8d8a2bb1d48f14a4d927d717d45d10dd3c0cacbe19f6e2cf94455a7","input_tokens":8923,"stratum":"16k"} +{"group":"paiml/aprender#3404","diff_sha256":"c4cf0db8bffc8083c5691e577702ca73a932cb4cde56dd1c9dac360353ee38b1","input_tokens":11185,"stratum":"16k"} +{"group":"paiml/aprender#3244","diff_sha256":"aa3e63ab8090ebf9be4f8d697c9f96d692a2763af91a2ad7bf96d1a26c384025","input_tokens":9784,"stratum":"16k"} +{"group":"paiml/aprender#2040","diff_sha256":"ea0520ed2859350cd1372043b3a06ff52b3e2f75fb0195b3d0521d1f23b3e7d6","input_tokens":8746,"stratum":"16k"} +{"group":"paiml/aprender#3000","diff_sha256":"54246f96c1844cc9afe81e68e68bd564d60308d58851d0af96c22aeb7059b962","input_tokens":10998,"stratum":"16k"} +{"group":"paiml/aprender#1984","diff_sha256":"014e7869f90798986ecf651ca3a959bdf86f6369b6ce3d5ff71ecf0fd7cf7417","input_tokens":12008,"stratum":"16k"} +{"group":"paiml/aprender#2054","diff_sha256":"2cdf5aadfe826822cbeef1cfad68e8d2d21d6bb99bc84ba02b50ad8224102206","input_tokens":9585,"stratum":"16k"} +{"group":"paiml/aprender#3488","diff_sha256":"6aa76be5cbd8ad7332c2acf93046bc20e866b9847dd276b88d4b66abe11949c8","input_tokens":9163,"stratum":"16k"} +{"group":"paiml/aprender#2226","diff_sha256":"720dc2dbf89b8f4f37b09af101e95d2b2c11697ff7a5d59d50f131286980e9ab","input_tokens":12964,"stratum":"16k"} +{"group":"paiml/aprender#2409","diff_sha256":"ebb26bc757178bfd19484adb04721a053df335e8059aed67cf8e11eb4d53cca1","input_tokens":13684,"stratum":"16k"} +{"group":"paiml/aprender#2244","diff_sha256":"1ee0546fe5e55ded3962edb4c5d8eb0632d49a0d3fcbcb6de9ec2d9936a567f2","input_tokens":9151,"stratum":"16k"} +{"group":"paiml/aprender#2416","diff_sha256":"1c2d4d2a2013dc86953f03eddb0ce4b67ac53e05931b7b18ff6d527d2c880640","input_tokens":11162,"stratum":"16k"} +{"group":"paiml/aprender#3030","diff_sha256":"f51557a52249a78f34a126d0646c5108cec49aa8bf9829fbfa487f75f82c7984","input_tokens":8438,"stratum":"16k"} +{"group":"paiml/aprender#3053","diff_sha256":"69a9d1096c603ec6d3706769831302fbf6a198a02c189f771a730704506c2941","input_tokens":14382,"stratum":"16k"} +{"group":"paiml/aprender#3631","diff_sha256":"36528c9baa7523120f3683b655e9bd996e7ec070c288fa94b92aa17f033e18c4","input_tokens":8565,"stratum":"16k"} +{"group":"paiml/aprender#2075","diff_sha256":"7d6b1ea94587f39adc61be730808d638df69dc46fb23c0292d839974de14c4aa","input_tokens":8950,"stratum":"16k"} +{"group":"paiml/aprender#2096","diff_sha256":"25c07e0a68726a522357ec60ea143c91cea4227fe4139c3db6f0c52ac274bb8d","input_tokens":14492,"stratum":"16k"} +{"group":"paiml/aprender#3425","diff_sha256":"cc6f12cb2a5dc9e2cd0cff0eda9338245b06a7b29be77caf7c7ad415dd948bb4","input_tokens":11324,"stratum":"16k"} +{"group":"paiml/aprender#2200","diff_sha256":"88ef36ab5267cf425c39c87c20a1e697a4b612b9d36b17c39ca3b51cd24f9a9e","input_tokens":11833,"stratum":"16k"} +{"group":"paiml/aprender#2186","diff_sha256":"43d42945260ed2fee02c5b9fda7974cf8f66fd65dff63de4e17ea8faf547e878","input_tokens":9726,"stratum":"16k"} +{"group":"paiml/aprender#2680","diff_sha256":"059c0fd951a748e2ae74924c143d7c8e8d3c0c277645ba032d7f3fe28458b002","input_tokens":15304,"stratum":"16k"} +{"group":"paiml/aprender#2420","diff_sha256":"0882349afbe288f0c5a3e2112311c022c01875743aabcc1d12afe9ce23093012","input_tokens":13765,"stratum":"16k"} +{"group":"paiml/aprender#2251","diff_sha256":"bcef4d3206e948203a70d22840d2a5bbbd067648e038d4939ac7913c2d0d977d","input_tokens":12074,"stratum":"16k"} +{"group":"paiml/aprender#2234","diff_sha256":"f44f0554c5d06d6d9bd8a87e7c9adef7b5acc16f8dd84ee8990ad95cdb0d7a86","input_tokens":11181,"stratum":"16k"} +{"group":"paiml/aprender#3006","diff_sha256":"d49cd0c3fad3e8f5c978cbcb184e89f2cd360cdf37c1a97d192e6ca1e88aa974","input_tokens":13762,"stratum":"16k"} +{"group":"paiml/aprender#3031","diff_sha256":"6472e6c3bf01349b337af3cf2ddd2fc277b3af86eeb21bf5ca250923f3760dbb","input_tokens":8810,"stratum":"16k"} +{"group":"paiml/aprender#3339","diff_sha256":"5a466ceed2939ee8936626418aadce90a670d794ce8d1289af467fe01430718d","input_tokens":14537,"stratum":"16k"} +{"group":"paiml/aprender#3348","diff_sha256":"c8cd063621db62fd4dde1e4e2fe07d67562b8d073e89e6bd35c00b88b2c6d546","input_tokens":13189,"stratum":"16k"} +{"group":"paiml/aprender#3530","diff_sha256":"1c7345312237ba7892931e444d0b663807feae59083cbc849d13f6e4748d2bd2","input_tokens":14620,"stratum":"16k"} +{"group":"paiml/aprender#2175","diff_sha256":"0dc43a5ae0a19646ed73c5f0aa0495eda3a3b460efd42e35ab5a0d1fe2002192","input_tokens":8740,"stratum":"16k"} +{"group":"paiml/aprender#3211","diff_sha256":"5189d6142784fae84f7d1634633cf93f49b3d2528950f68906f1c5ca9e8dfc5a","input_tokens":16196,"stratum":"16k"} +{"group":"paiml/aprender#2227","diff_sha256":"28c93295ce9295f699d1ab17eecef2d2ef643f6ec8434694ee70fa20ad70c0b9","input_tokens":13580,"stratum":"16k"} +{"group":"paiml/aprender#2292","diff_sha256":"0f1bc13a198826247cef2fe67851805b3a7046ac9d291b02b47581866c6d896c","input_tokens":10000,"stratum":"16k"} +{"group":"paiml/aprender#3327","diff_sha256":"4e7c51716440d0055d37c6d118eca15cfb84381cb2984b437ddbf7920cee4651","input_tokens":10233,"stratum":"16k"} +{"group":"paiml/aprender#3405","diff_sha256":"d2107848512e9694b1e838662ca844b67f962cdc8d3826265b75f6b002d2982d","input_tokens":8600,"stratum":"16k"} +{"group":"paiml/aprender#3089","diff_sha256":"f0466d3c53c99fb39ebfe065673d3b9bdd70fdf30ee02898eb6e886201e8d4a3","input_tokens":14167,"stratum":"16k"} +{"group":"paiml/aprender#2479","diff_sha256":"f570aee8b67fbb453024af27fe758441e49aa16bc6b09147772a0d43a92115ec","input_tokens":12300,"stratum":"16k"} +{"group":"paiml/aprender#2547","diff_sha256":"409826d3610e79d93d92ba45ad55d943307796f6b4218ad327e872390681ef26","input_tokens":16265,"stratum":"16k"} +{"group":"paiml/aprender#3265","diff_sha256":"b53f7c2d4a456e341e0f1e013eede2c53dbe5230bb2ef59c8437a9dbd9b1126f","input_tokens":13353,"stratum":"16k"} +{"group":"paiml/aprender#2781","diff_sha256":"fd326fe7d4bacc97fcf64369c18bd23e1a718b755ec9d720b6010b7bea96f970","input_tokens":8402,"stratum":"16k"} +{"group":"paiml/aprender#2840","diff_sha256":"fa7720559e61059c92e06f6156fc0da7ae0a6367cd814f36465e85f7d04800e7","input_tokens":15831,"stratum":"16k"} +{"group":"paiml/aprender#2159","diff_sha256":"f8f4e9f89c28ef4f440267322b74bbf68eed96ea240d6a883bec0d50f2dbe576","input_tokens":9547,"stratum":"16k"} +{"group":"paiml/aprender#2089","diff_sha256":"ca612af4633a006928c0400dcf7a90a769c47290edc85241d3ba156583b2f6b7","input_tokens":9254,"stratum":"16k"} +{"group":"paiml/aprender#3411","diff_sha256":"f80dddb74b021666f2f30078ee85f6a231fb529e1ed67f051a9b662ffb17b4e6","input_tokens":9252,"stratum":"16k"} +{"group":"paiml/aprender#3145","diff_sha256":"fa49d36e7eb9eed429d0d549464654396d8fe8fc5a0be7b4e667bddb37922c36","input_tokens":24681,"stratum":"32k"} +{"group":"paiml/aprender#3295","diff_sha256":"3b51ad3f76c868adaa04840a77eed635ade302cec8c85bf0d3795526ab237fa7","input_tokens":20207,"stratum":"32k"} +{"group":"paiml/aprender#1950","diff_sha256":"f8815530d0c7dc219ea3341c95432e757c3ab7fb0308bd2146f2d90b840f726b","input_tokens":16912,"stratum":"32k"} +{"group":"paiml/aprender#1963","diff_sha256":"717524cb0e8f36fac7a56af480fe9ecfe30ed9e797cefb9908a1c663d247b0c7","input_tokens":19112,"stratum":"32k"} +{"group":"paiml/aprender#1928","diff_sha256":"03c15ba04bbacf6b92b3ab9bd18cdca26071fbb4a09dc4de9bef2ad078ba580f","input_tokens":22095,"stratum":"32k"} +{"group":"paiml/aprender#1955","diff_sha256":"351ae6cc9897a62bc51dc629ead6161e8242c49a7cfbeabdd8f05e9c0c3cdaea","input_tokens":19558,"stratum":"32k"} +{"group":"paiml/aprender#1927","diff_sha256":"7bb9e2edd88d122d0553e7776b5ac605cc7da369d4410dba027a27df9290f060","input_tokens":21282,"stratum":"32k"} +{"group":"paiml/aprender#1968","diff_sha256":"d667fffd336a3dd90c6245ba9527ce0fb0448c554d576a089e347230a6ecb2c0","input_tokens":19184,"stratum":"32k"} +{"group":"paiml/aprender#1961","diff_sha256":"38a1dbfd07591ad9a6e4b27e7916e661614e6cd7231082e0e4e2601297e5fe7f","input_tokens":19527,"stratum":"32k"} +{"group":"paiml/aprender#1945","diff_sha256":"3d8a6474dcfe17a53ec35178358b968c69405032d23e581b7a700fe39982960a","input_tokens":19036,"stratum":"32k"} +{"group":"paiml/aprender#1922","diff_sha256":"bf2811a3edb86d54d69166c93ec73ebcf6ff64b5556d9d7d9f090726e8fee9a2","input_tokens":21743,"stratum":"32k"} +{"group":"paiml/aprender#3281","diff_sha256":"b515045eff1517e48974319c1c62e87bc8ef658b739d6e365b934ee04f6667f0","input_tokens":29881,"stratum":"32k"} +{"group":"paiml/aprender#2429","diff_sha256":"8f82c371a972bb3e1d51e43968dccb02cfdaa9e94eaabb04b2b6202addef14fa","input_tokens":24089,"stratum":"32k"} +{"group":"paiml/aprender#1952","diff_sha256":"304c3c29ecc274958e1042ea4f3e6fb4108bdc048767ad229c669ae8025e23f7","input_tokens":19206,"stratum":"32k"} +{"group":"paiml/aprender#1951","diff_sha256":"c9d4c24c8671432ccf62fc9a693dd62fc3430da536f4b20f53abb64ba65e010b","input_tokens":18499,"stratum":"32k"} +{"group":"paiml/aprender#3557","diff_sha256":"7173ebdf8174e7e3ac987fc47c136b866b83cdf52d03a1c1fce2fb4ed8192ded","input_tokens":31442,"stratum":"32k"} +{"group":"paiml/aprender#2657","diff_sha256":"0f963ad6b87e0d3342e5334a4115c16eaf148f32c1e0adc968706c77b5191039","input_tokens":28218,"stratum":"32k"} +{"group":"paiml/aprender#3450","diff_sha256":"74f801a07d6f9f7366f719b4b2302c99b8c74f8eb9f099cb0088470f7ed180e1","input_tokens":22762,"stratum":"32k"} +{"group":"paiml/aprender#3065","diff_sha256":"430c7c0f69397873ade236dd3f5db7663f2f1fb4f40eb879109629b59e0b3ce3","input_tokens":21851,"stratum":"32k"} +{"group":"paiml/aprender#3698","diff_sha256":"8003dd98e1763cde69c838d89b0bab41aaa2622c580b9e3de39b8b3c1e182f85","input_tokens":24957,"stratum":"32k"} +{"group":"paiml/aprender#1971","diff_sha256":"b931d437202ef5adb659042b0e7255fd1754602c9d52039ef36d831ad232674b","input_tokens":19706,"stratum":"32k"} +{"group":"paiml/aprender#2277","diff_sha256":"084e31008f5a675e5153246d96a5142e0adc8215e66f598266bbb76a10be6db2","input_tokens":30181,"stratum":"32k"} +{"group":"paiml/aprender#2215","diff_sha256":"14b3251ddf800d7bdfdb7a775c2281eba042c909e8a46700689825014ded27de","input_tokens":17965,"stratum":"32k"} +{"group":"paiml/aprender#2256","diff_sha256":"ca55757f48d1d3f049d77fd86f25cc77c581af66809d307a89d2a5b7bb14d278","input_tokens":20900,"stratum":"32k"} +{"group":"paiml/aprender#1953","diff_sha256":"5bc59b96abf0b69ee35dd697a3d38ea685e54eb0cd1544a880e08b0f64c3905f","input_tokens":18671,"stratum":"32k"} +{"group":"paiml/aprender#3044","diff_sha256":"4608b8ddaa9de754634659da66dee0e3d59c7097dfbebe7ef8ff664f7a8e2a3c","input_tokens":20129,"stratum":"32k"} +{"group":"paiml/aprender#3737","diff_sha256":"dc10ee3c8ec1879df8935451288b060dcba94ff3db9fb2d0e714e938b65c15ca","input_tokens":26356,"stratum":"32k"} +{"group":"paiml/aprender#2141","diff_sha256":"841dd09f216b308d2acb091c53e79c3c22c6dc9c70d99ddd5aaa007e65fb8f0f","input_tokens":18317,"stratum":"32k"} +{"group":"paiml/aprender#2264","diff_sha256":"24bc258f12dfbb5600afabdff2efa2edeaf9efb4c7e13dae0bb627e6c0add649","input_tokens":22827,"stratum":"32k"} +{"group":"paiml/aprender#1954","diff_sha256":"78a121fb8ef6ffe62c572f12c3113443d88b19491cd0db0981294e8ed29d38bb","input_tokens":18638,"stratum":"32k"} +{"group":"paiml/aprender#1944","diff_sha256":"c0b75e2f0a6bcd1a3341d0cbb4e96aa16449934b09e961f790c377d972340d08","input_tokens":21345,"stratum":"32k"} +{"group":"paiml/aprender#1924","diff_sha256":"21a87ab49fd3587febc4c89bb82872bb6967bf0a9492abc590112d5f097f1bd5","input_tokens":23478,"stratum":"32k"} +{"group":"paiml/aprender#3139","diff_sha256":"0534621eb0e2215a3c13b22ebf197a2a472f4306bf24f0e380154a527dbc1463","input_tokens":22860,"stratum":"32k"} +{"group":"paiml/aprender#1917","diff_sha256":"add564d1303cf5966124647b302705301dcd4528a1caad315f4d86aa3485e61a","input_tokens":23434,"stratum":"32k"} +{"group":"paiml/aprender#2561","diff_sha256":"756c7e34ba781b2a93fb5ba02710dfea9f422786e59c39eb6e2b38e11ad4bf4a","input_tokens":18827,"stratum":"32k"} +{"group":"paiml/aprender#2871","diff_sha256":"40d8b0a4203d03441d9dcc41d1b76704f65ec7ea5ebd165cac2457a566b70162","input_tokens":17831,"stratum":"32k"} +{"group":"paiml/aprender#1946","diff_sha256":"3db9e737af60c9d2537ac8d8b93aeaffc7b6cc68294a49978e5c870ffb36eb4f","input_tokens":19723,"stratum":"32k"} +{"group":"paiml/aprender#1956","diff_sha256":"c354c595f4a8ea1b09659b55a5ce312a495e4eb6400c77183cd34cca3cb8327b","input_tokens":18714,"stratum":"32k"} +{"group":"paiml/aprender#1965","diff_sha256":"c42dfb333a42fbec1745318326627fa05d41440da37c5d2d57304daf1e4bb3b6","input_tokens":20071,"stratum":"32k"} +{"group":"paiml/aprender#2218","diff_sha256":"8136fdc9902571f25cbf93bf14de71435aea774d50ab0fb50ba2196f422389af","input_tokens":20632,"stratum":"32k"} +{"group":"paiml/aprender#1915","diff_sha256":"a267f9989c8533bce8be2d300c85a8509c45a80b1fe16836591ae91341138b2a","input_tokens":29268,"stratum":"32k"} +{"group":"paiml/aprender#1960","diff_sha256":"60d2f019ddec1435b5b1230d70553a3833405b30dae55c8dbc8f6bcc1f0f6487","input_tokens":19053,"stratum":"32k"} +{"group":"paiml/aprender#2282","diff_sha256":"4be2763e70a83f3b538045b385bf920c45f0ab971c6641e4573f745b32673998","input_tokens":17094,"stratum":"32k"} +{"group":"paiml/aprender#1918","diff_sha256":"4a66cbc27f5a915eaa9fed54adca1621f3d27e77b95dee234328d0114d722209","input_tokens":27133,"stratum":"32k"} +{"group":"paiml/aprender#1973","diff_sha256":"f552e516d8a086ee988e3c5488a31467a2bb6b35b277aecfb98ad15c96d6d3da","input_tokens":19101,"stratum":"32k"} +{"group":"paiml/aprender#2985","diff_sha256":"5c2a849b4a443f183000bbd6996f84b95d3df274e267f4a0b9be650a53080e75","input_tokens":23690,"stratum":"32k"} +{"group":"paiml/aprender#1969","diff_sha256":"a2b4707bfaf04f8599408ccfb45a44703a000f71a094a68ba1759061b2a43672","input_tokens":19540,"stratum":"32k"} +{"group":"paiml/aprender#3011","diff_sha256":"3ab4ac140f83056d2501ee334f17e3ee310e217094d568b4fec0c75a1a30dce9","input_tokens":22474,"stratum":"32k"} +{"group":"paiml/aprender#2313","diff_sha256":"bf6a52843bdf826313ab90adf25c9d7a253496a9a3c5a46c08ef62ef5249470d","input_tokens":21252,"stratum":"32k"} +{"group":"paiml/aprender#1970","diff_sha256":"4c7ad86de1bf01701febc9297c4271016e389826ac06ac0fdc766c598c4ba789","input_tokens":19596,"stratum":"32k"} diff --git a/docs/audits/review-corpus/test-sketch-v1.txt b/docs/audits/review-corpus/test-sketch-v1.txt new file mode 100644 index 0000000000..4c03872cfd --- /dev/null +++ b/docs/audits/review-corpus/test-sketch-v1.txt @@ -0,0 +1,106 @@ +review-corpus-v1 cluster-sketch shingle=12 k=64 +G-pr1026 00233819e5ca79b4,0040f41cf2cdc2ee,00554c64909e4866,006329be4d4c98c5,00be04ec02c1226f,00e2a93f397e2729,0190cac0351b9dff,01aecfb4210e9da2,01d0ab0d1b113b9e,0218ebb86cd00043,02df0e4d6fe70042,0342b01c90b5a17c,038b3f29309c3b22,03c45a12eb539e04,043fff7dd9b7d697,052b639c9a21ce14,05b6b7f9275ecaa0,05c6494194e06068,061654c568cdddb6,061fa71528790b12,0674db9fe020f342,0675a76d7a4b618d,06fe16a711e5b28f,0719c4fe36119ec8,078dbb4190fd934f,08326766b3e2cb2a,087e05b3d1c1f331,09110c6684c9802a,098a408bfb550327,098e1c123480c6e7,0998d2b6ac385d12,0a29b65bd2d1310b,0a78e8bca2c143b2,0af25da2217b9ee5,0afd466664cf3c1b,0b1e3a52c7f13a65,0b33cb29f4f15659,0b3cd171fd855279,0b6273c21ddbdd77,0b75a8b2070be85c,0ba3bc0e2cdc82c9,0bbfb35dcc7898b1,0bd8a95f457e0bc4,0c8ac93ff8777da3,0d1b544648b24077,0d60cebd27d6cad0,0d922bad253744ca,0d9fbbb8c3c542ae,0dbe83e70f32366e,0e69782af6cdaa0b,0ef3276f2925e500,0f2761d52527e0e3,0f281cb5ff3b5917,0f5f54ee223c57f8,0f8c149ffaa16b65,0fb1af42aa25e7b2,0fdda4e524e295a8,100af4fbd1906010,1045837adf1648dd,1067f998d31fd186,10850e8bfa0c87b5,11d46e91f215e6f8,11f0982f0b8afbdd,123326b137eb39cc +G-pr1052 00028a86a45efe1b,000e88b89fcb5fc5,0015a74b3c42f0d2,001d0c38afa5bc84,00314de3d279dac6,0033884767c35407,0036fa5d8449e767,0038d2cdf1f74474,004c6c6b7a2dea81,005fc33f4c1402a1,006e1b5f21edb932,0072fe6ee3073849,0086e4ebbf9aa654,009f002ac3a1bf7c,00b47a59627b9f14,00bf03f1cb2e8ee6,00c44768722bec38,00c45311217a8b44,00d54e7631bc3c29,00e4710d385839e1,00e591c16ec0536e,00e8646e82db8ba0,00e91632e5cfdd88,00edf5236882b733,00f96e6b069874a9,01047326b84eff29,010772ca66a4724f,0110ce9a255e0d1e,0111ba2c3ec29dc4,0113c03ab3790783,0126fb1dc4b7879b,012b9fa95e8ce942,0131a029074b0428,0133daccdc238472,013bb31783e39ed0,014745a66ac9f817,015e11dd0ca808bd,016541480b4258c2,016b6756e98d22a5,017a6969e4c7bdb7,01823132922c8d9e,018a2b1f97dd03b2,018a6d95db36ed5a,018a7d2b1f1f3913,018e44ae791f137b,01ad2f8b89106fe5,01bf48b827a3d1de,01d24e5c273c496d,01d6e508dedeadce,01e5f2279905bf37,01e9751d2e26080c,01eaa6010f805f6a,01eaf32b11b5ebee,01f06722d0822893,01f11c3a6c45eaec,01f19f5b32045544,0202305d76e5ce4c,0214eeee34f83fc0,0216ce6b688ce6ac,0216f5dadefdbfab,0217b3e1e145abe5,0230d812e887953c,023d9b081a98cff5,02485bd9f2f816c9 +G-pr1058 00b78d644411808a,00e7f34adebf7a62,00f1431b15f4c34a,01044badced52faf,014ae6d36cfe4e3b,02b65fca275f988e,0342626cda0e4313,049463b6133a233c,060ade7c8491f860,0628368f88a5a686,0633889301b941c3,077ab802988df174,090c87a79943daa7,09166ea1d22fa7e7,096cb40708cd6d5b,09d04dd37be03104,09d0c98ae57fd281,0a9253c9faccded1,0b5ec2756e480510,0b94aeca2b5bc0d4,0bad80b951ae8f52,0bcf3405e9e03056,0cbfab2e27645c43,0d91c001f4f58da3,0e18df1778f92cde,0e707a38d1d2362d,0ff57abb9941949e,101fc5645ff53b8c,11a1a3b434b83617,12264a807e333363,12d225d5faf4b968,135fcef17516acb3,14e90bc6e15d3903,1637097538976c42,163d46be4ac92449,16eed46804c93b17,172d0e05b7400a78,178625e8eb397e99,187750c97b69dd5b,1890138da69a8c98,1a8bb8ec7bad879e,1b22a49773a8ca48,1b6dcbf3bad4bb5f,1c5b5b597c64cd33,1ccb1c31ca0df9c6,1dff23ea5334d5e8,1e30a5004b7dd1c6,1ef70f871b5142b2,1f4f71123fab1168,2152b12ca1898054,219dbeff73f27503,2297bdeab053fefe,254d7732b914f266,2647933a558d1acb,26f764ee85bf6b2c,27b5e313ab733ef1,28636d420fbcea19,28e19848c277ca3d,296a2d253b6e897e,29bb3094cb559d34,2aa13fd2de07e7f1,2bb9dfabf49b696c,2bceb9834b712ff4,2e08510c0ca9001c +G-pr1081 000d182a44389507,001f208236e3475f,0025745cf1369a37,005c6a0e8ad99670,009a6ebd3b1f51f3,00c0a76404afc744,00d4852ba07efde7,00f14f6f079afdf2,01354ca1bdaeb150,013848ca9612168b,014749d3d96ced25,015fbcbddfa8dc57,01993073d52804ba,01d5d6182b830936,020ddf0412a70536,022e009974aa5d9b,027267e0e7d1f579,02761bda87188aa3,027a75826483131e,029fd8cd7dc99b0a,02c5b4aaa03e7b0f,02d634aacfe70021,02e1ccdceffe033e,03020dba55f4dddc,03459fe00877ae66,035d0fd3ace7a0e1,0361a2b6a0c3a27d,03cde114be3420dc,03ed08a7659cb1be,03fb6c9c648eb2c5,0403f23496784e1b,040ee92b9cb1f93b,0422a5f20897aa7b,0432299ed78aa1ef,046453cadd31a1b2,0472a2387164e197,047926a59a5ab70b,048df5b8c7d8a986,04ba2483ba3c0e28,04c23b22300babc4,04ede716265fb981,04f05422bca60543,04f75e6dda982789,04f892fa5fd17bfe,0508ed67da9e1858,05138b2c1196b9ad,051822e198de6253,053e7d4e0d3e2544,05439c6052e58e5d,05706b9bd39c1b26,05a7efcc852834cc,05ec84838354696a,05f25489d49b99c8,05f2cb65a7da3f06,0608370b41eacf1a,060f53bc11d24aee,061705989a6505a3,0620043fb58ff9b3,0638731bb4319ee0,0667deaf1ed878a7,067dee325bf4dfcb,06887c6b4328b438,0698c0c09303abf4,06be5558980751eb +G-pr1136 005689c497c435a1,0071795423fab65d,007c977a0a045202,008aa870a26db23d,00bd37ebc1d84957,00bd91174ecae264,00c814318b7f6752,00fd2d2c0ee8abe7,011124969f83e4d5,0124fa1e7334ea33,0145e083220bd50a,0153c1d1a79aedeb,015c906cab9d2fea,016ba0018a93e34a,0179cc7ec67ea97c,019794097bb36615,0199f2eed587fc3b,01a56168aa152733,01c71e365dbdc6f0,01d2640e6dd9877e,01deaa70ddb1ca03,01e095fdb8927acd,01e7f792a2e0319f,022b0241be39cca7,027eb4785351a927,028d784dd27ad105,028f038c26850436,02b011aceac523ac,02b16bcf24df62ae,02e22f814ee353c9,02e8012c248a986a,02e91ace44015d59,0318c785045c3902,035dec81b0896d39,03729abd0d8faee1,03788081e7719aba,03876b6cc5e13c30,038fe127d540984b,0392b5b4faeef0d8,03aa0a89cfabd656,03c1283cf609a782,03cf0f37722e2a6a,03e2d79943739998,03fcebbe278c88a6,040d81e7d8ebf164,042e57abf7d5f143,0437f5e18d7cd7a9,043af7e11a990df2,044a84f9b7fbbe07,045d9eb2ef7d9de4,047b2768a90adc87,047e62c6756a28d9,048cf834ae178858,049e5b5ada218fbc,04ba2483ba3c0e28,04d084b009bde1aa,04d700764f8b02e0,04e05c81b3d11a3f,04e99a653501ec94,04ed9b41b8610a2e,04ff0d2a780ab1c8,051ebac026970342,0540c0fd34fa8499,05834f0be2f32699 +G-pr1408 000aa4a005860d5c,0010ae590eb71dfe,007e724d0f39c9c2,007fe6fde9f37c3d,009433a0a3305e29,01009b70b8616c9b,014bc4e98820cde6,017edb43fea2405c,01c71e365dbdc6f0,01c9157c9b595338,01d3cc65f63e8fcb,01fe63c5cf38a819,021afab2d3fe3910,022bf34b6078b9de,026c5078e14e5ee0,027a4a09f5be90df,02866cdd7bed4d73,02b1e24294659055,02d3528a82469c3b,030a68128ad1dcf5,031f14317da411cc,03e58f5af10b7592,0420bbedfeb5720a,04383e4e199302bc,0488c38b173c2924,0494bb6145d6befa,04b8a2ebeced6e81,04bf91403725060d,04d39988554a678a,050b930a96c77226,053e1f24cc0c2195,053e7d4e0d3e2544,054153435320e585,05778ddde2927e03,05d7397bbeb171bd,0608f0aa1c710fba,0642ca6a78b305fa,06af808f3928207b,06ef1a9f6a449ba2,06f98cc3cf372817,07cf1486e9cd2c93,07d4cd1e0e8c56f7,07f39e44b0f980d4,07ff7eb09d8a4284,08b4dc6277218e8c,08b798f0c37a4a43,08c0a46ef42d195e,090206419579ccdc,0930c2e9f7454962,0980e3ebce771019,098cb3eab3a22ea2,09b86c6ee6daebd5,09c9c2ac16c9f42d,0a6e68462edd50a7,0a726fb5015e2179,0ac12ecfdf926374,0b122ef32ec1a77c,0b2d89e3369f1908,0b4f270d24a517ef,0b6a0da8e9540068,0bad80b951ae8f52,0bc9be0b2b6f2578,0be70060cf34c23e,0c3793b46a6cbc81 +G-pr1516 000b6d621ca4b044,0039a85ef9ef4065,00577fd5e1bb105e,006db43a7814a475,00bd89fd330b6bac,012f58afbc2033bd,014749d3d96ced25,018bdb46c2e96584,01c71e365dbdc6f0,028f038c26850436,0305bf1378ea9e02,032e0f280d491a8b,03459fe00877ae66,0377c7bab9538a0b,03ed224ad7c73525,040bb5cffd4dcafe,043d4b5c3aa5540d,044393dc4325f145,0489465256e55e94,04b8a2ebeced6e81,04e72554b735f1da,04f25ecb13e83fb8,04f5686ba46cdbe7,0502c4c595c3b59b,05143ce9a51bdae9,051c12f02efbc168,054983c2db90b8d3,058d9e205029aa2d,05c7fdbfb61100fd,05fa68663e6bdce1,064f0ef9f2ad8024,0651a44de2a62499,066c0926a2b09ca7,069a364112eacdc3,06ff49c95dda0202,0718238bad800dd3,071b114079f6e059,077327079d241d27,077aef6d9cb315f0,08260d44e5e99cd5,0846df09c205d70d,085d48957fc71291,08607c2ab718be57,0876474ec1eb0876,087a667ad33cd360,089ff42f4c4bb345,08a31a4851171c8c,08a4e44a71c621c9,08cc603196f93ea7,08d5eac97110075f,08e01dd674f3d094,09293e3e1c11914b,0930c2e9f7454962,09cf7494ed0beadf,09de1ac97674de6f,09de758819416615,09ec58a0e8f65ad5,0a10bdae6e5e8501,0a148db7ee5e3fe0,0a310818a8e00aaf,0a36599c7f0a3ac4,0a6e68462edd50a7,0a8458edf2a7ed0e,0ae5525446ecf897 +G-pr1536 0038efc275bea373,003cc64d201a42f8,005a6f5b1334b0ec,00e5e929c1667f4b,013f55e93981f78d,0167e3e57ea30353,01a5c8616db57b61,01c71e365dbdc6f0,01deaa70ddb1ca03,027b444eaf0e04de,028f038c26850436,02c0d73048319272,034e26fa6e201b10,0365c702e28d4e6f,036fd4ae021605c8,040f9e403a8af948,04779826fa4d5d84,0484896aba6dcc56,04918711b2c7d6ca,04dce9cef0f08fd6,04e05c81b3d11a3f,0523c58f3e931656,05882436a627aca8,05b2d3a521e65746,05bece7f0b41583b,05ce744c119c2d77,05e883143e18ee04,05e9f5a71dfccf92,05f0e96dcf720a66,060b240a414307e2,060f6d0f4fae78fa,062557b8a81993cf,062ef2f49e5a783c,0646026ea149fdfc,065faafe5e9013f9,069029288a243db8,0695003b84acbed7,06a215e1247f7175,06a2b02b8bfd9a3a,06b318ad2c61eeee,06e29c8a3b91db3e,06edb692a55214b7,07154d6e29aa526c,076591a9a7afa7c0,07826a615fea4a9d,07adb731a9157060,07b86164170e21af,07e9003c64520986,08555b130ad0f083,08604e8f65332b70,089a14adef6f9a2d,08ab0dcc1104a015,091944209d4424a8,091c7679ccf8e709,09c44ffae0da684c,09f8fa999e9d0cbb,0a2c7b021423f139,0b2006bd6e97f83a,0b46ab60bff9e006,0b4f7d28799e6838,0b7097c308b243a8,0be8f78f37d47a59,0c20ae8634d7b6e1,0c220863b890cda8 +G-pr1543 00120e7f77b55f99,001509c574b5d6a7,0038efc275bea373,0069a903194dcee9,0096a622d351afac,009b58b2b7f5ce84,009c24f6d7c272c9,00b0f93bbfbfb27c,00bd0b391a960767,00bf6e2bd6135426,00c3f6b82a724842,00ccdbd0bda8c393,00dbbf34d7274cf1,00e22c2718e37591,014981f5e1e2aad6,01a4becd760c6030,01c71e365dbdc6f0,01c9346e6f6cc351,01d029677ea9a848,01d773d9e2763ec4,01d836af8ca6d37c,01ffe1540dbf0135,020012e881cbd925,0238a5152e0de2e1,02429626dc3b7e9a,02473d62f0cb8ecf,02686641e6d41c84,027ccac95414143e,028c8e99df38614f,02d8962fc6bf3266,03043b3d26f3e726,03193ff41547b02d,03395cf27ce0e4a8,03459fe00877ae66,03469bf0f7cc9e07,0352ee2db84b986e,036233d6c5517094,0395690ea4f44dbf,03e50f7e4b36afa1,04017144cc8d3a3b,0414ac16fa510b14,041fb5313c2d6ac9,04207a794d5117f9,042144c3b75883fe,045a3c15ab46fe8c,047f80a8f3a9378d,048f0b92824f1e28,04a07af60c055074,04a56c594d37f4af,04f00b716fbdae96,05303357f83c7222,05322f31eee255fe,0548f574a159b7df,05875b1fc2bae231,0596cd480cf1204c,060f6d0f4fae78fa,0641a52371ea5509,0652cc40141f0f2f,067013368704f2af,0680cf46ddcd8c31,06924ed51218f214,06a7272c1dd71d85,06ab8efa1d270433,06b6fec4ce7e2817 +G-pr1555 00031a36c5322969,003038efb5fa5117,0039ce8a596e070e,004710719118884e,004dbdd25d799004,0052fa67e95ceeb3,00582eb2754cd938,0060acea5dfd2e45,007b193f2b7c2e10,008138da6b1979a0,008b3d9866ad92cf,0095bcc5a8069c8d,009929aed284a068,00a227e909c453a5,00a6e8193e166331,00b096adf3c85689,00bbc99ab262de19,00be6b09bc963cfb,00c612aec618b404,00c647fc48d23ed2,00d06e1fc4cf74c4,00d28091caba1460,00d66a50d2440376,00dbb361bd8fe7da,00dbd0d9e79af202,00e3c396f7f5a847,00ed7224598d900e,00ef52aced202243,00f95f2ca18f4d51,01082bac13239108,0126d38b005ec4a7,01270ae13b789900,0128545b8dbd009b,012d9da750fecdbd,01331fa128b482c0,013f8dce8c0f1f6e,01457dee8ab62215,014f968670479812,0156d4d34f0e0dae,0159e45cf0077c2e,0161ebade7b2561e,01699b59aaec6e21,018139f9fc77d4b2,018d574d863ea61b,01b6d42160eff137,01b6e4ad2e2930ea,01b9e029b32f0f73,01c80fc15aa5e599,01ef4fcaf078b0cd,01fa6846e93d27ab,0201561401f6b487,0209d23fd088ea19,020beadf8b14bf40,021232ea7a63328d,02229a8dbc63ad7a,0225921475f97c61,022d9f096de1e080,0232cf8e6e285c24,02330a6546e8182b,0240d453f1482e3d,02506a4ad0ddd01e,0264433c491b6230,026f3d3e458e9c6a,028f213586e13646 +G-pr1568 0012fc94caf1ba41,0017b905b6551794,001b1cad8722edcd,002b892427475a78,00340da0e8441a98,005421948510a8fa,007093e6fb67f5b7,007aad658d421f11,007c977a0a045202,009ce54f9d530624,009f4c02fd47dcd8,00abcd2fb3eca351,00d1fb844d35225c,00f2ce90d760f791,00f66511add4e2e6,00f697fbd8697a44,00f811236734f304,00fa6d2bfd6f6148,00fd2d2c0ee8abe7,010a8cefbdc34a9e,010c2665cf9d1d30,010def1185af35c1,010fa931fe4d1d90,011f271e52e8ef50,0121cf9865f7bda0,012a979b0a5defbd,016103c9e554bcbf,017042731c36b6e7,01790124e2d7cde5,017aa514b8eb4e1a,01c3250327de4240,01d2640e6dd9877e,02113408973947ea,021149e15e2d21ea,021584daac8364c8,022d06312d70c7eb,0244b54d40b9f8e3,0266f06702b88290,026735e4628dc7b9,02680c2a3661b06b,026ef85fbad60f4b,028016725620fb29,028d6baeec59069b,028f038c26850436,02a52a972754e64d,02b6b66f3af57cdf,02cec62cc9b4af4c,02e22f814ee353c9,02fec701318c00c7,030eacedcaf93ce1,031c75f1a1bc8198,03394c79a85bd236,033a1c0dee7b0fc3,034515b68419a22b,0346f4aa2084422c,034b98d441f8c87a,03678674dab35ce1,0382aed8803ac19a,0392b5b4faeef0d8,03a4d4569b972b70,03bab09f8e4e7c86,03e2e9fc4e3f452a,03f81e1df76aa3a2,040d81e7d8ebf164 +G-pr1602 0022c5ab9613d4ce,004e0ec2d02622ce,00576c6242680043,005e063913f1997e,00737712c96d00aa,007c977a0a045202,0081dcdec9bbc371,00db0c1874d16120,00fd2d2c0ee8abe7,016185a811786722,018c3979c481bc44,023027e82750764f,026b50bd5918be07,028a2f6437404b1b,0294d3b990819b0f,029905295dc5fd19,02b0fadf1a0883aa,02cbbaf816d9cae4,02dfc2bd2c983d61,02f48b1c4f2c2f65,03459fe00877ae66,036d831b23fc8ba2,038110a80e1c7c0f,0381baaea9b106cf,0384247c010f3227,03995ec8e309b70e,03b2a50e01a95c25,03e34cb7c396b7ea,03e5a4ee4b5a548b,03eeb732a28fbadb,03ff3dfb94dc52a5,0435b01dd9a6f87d,04ae1e6f40c47635,04b018225ec0b737,04baf742043c9814,04d87e64f1d6af45,0504a5ce51391a8d,050e8278fa58da72,051bff2f645b263b,0523fa23ca5613f4,052ae7bf182fbabb,053c3532b0910b7c,053f1c46162e3199,058e386f4011baee,05aed39f51b63b65,05d073434557932e,05f0d941c920f1b7,05fcf3a9f7afdee9,061ab31965bfb740,06366fc160ca9ee6,065c85ca8aa69d17,066a0f2664e22e3d,06931ed4e89daf99,069a5f776a737a6e,06e78f1cfff0ca0d,07154a758dd48759,072edb3eb6c3be6d,073c4474df3e50b0,0740908ea11c51b7,0744787108beec3e,07520cedc6c147b9,075464192425bc49,075e6783d3ba0cb8,07661bfd4b3b4f07 +G-pr1664 004d577ba8cad082,006a5a18a0218a69,007d64b346e798f8,008d48e93485236c,00c85d83f536e585,00e58e8aff8597a8,01073de34b41f5c9,017e570242ecaaa6,01ddc867553a059b,0229adca26e8e983,02b436c991fd1491,02f547429a5a4578,02fd04257b4c979e,03eca9e99b727fe9,03ee92a31170dc4f,042eb322117c615e,04476dea909a1ae5,04a4de436ec0fd64,052dc7a0e4b546fb,053dce5e86cabe4d,05ab6de47ac965d4,05c1ffa9d20536ea,060dae0a0ce56f06,0629b024a618f656,0664b185fe1fd016,072edb3eb6c3be6d,07adb731a9157060,07cf1486e9cd2c93,0868d8fc234ad26b,08a3a96a8b3c8dc7,08aa10a4982ab2f3,08abfce1ae9cc08c,08f298b9c37ca8b3,091cc40833926fe5,093ee019bc711c31,096c7a5844104c79,0977e19e62c988fb,0978767d960973b8,09f0b01d8b046828,0a4679e2e5506924,0a577c095022be7a,0aff280e2ebff339,0b60443870e860f7,0b7a0ee03da36a74,0b8d20b2c15a8dbd,0bc05a4cca7ea946,0cddf17d49a92fc7,0d0f9af18f71de3c,0f2d5d6196677e66,0f68e9397ee395d2,10614cd6363aeea0,10cf460b712c93a8,10dd00b90afa7955,114f23ca51053f57,11816387f0ce869d,11d3dfe5b4aa7df6,120d99500de509d0,1288b3c0263abc60,137ae2526d8cc20a,13f5ad637dc25d56,1414dfcf029f7a06,1419896d7e3375e3,145c49f64faedba3,145e307fec2e8225 +G-pr1694 002511c291509840,003c465251da2515,008b9e78aff09d22,00f7a9723fec351d,012f14b54750835a,01e2dc6edcf4e7ae,028f038c26850436,02ef0d07008e0248,035fc048871d8b92,0387250ad8a4a527,038a7dd4599b6cc8,038e60c24eeecbb0,0419dc75f4af857e,042c4c5da16acec4,0432ad52909d1f9b,046490e6d19db7fc,04aa3c75c85d3e80,04b891af86a96366,04ee34ca390df516,05d027d4334f798a,05e18d1a0f8eb572,065ae2ac7869e0a6,06edb692a55214b7,074e80a7b20a9b2f,076a7586d53b2c37,07a6db684e1e5b7c,07ecbbcf6ae54f14,08be8969d0b48abe,09e6bb3621010f91,09ee911ec5270582,0a0c77d4391520bb,0a14fa1c9ff6f53f,0a1b92cce8ccce0c,0a60df6782b541c2,0a8860d0e34e79ec,0a8cebb02a764e83,0acbf0b831b7423c,0afcd86399e11245,0b16a8e08b903da2,0bbcf27c925a83ee,0bf656f229edd7fd,0c150631747d9d37,0c9bd8622e9798d0,0cca789b04594f64,0db4bf140e1c7142,0e33103f1c73cb39,0e85adaf8580480a,0ee45cd2fab0ef16,0f5bc79453be6df5,0f7d2a3713f97165,0ff6ce816479b840,1030a842c9fc1eed,10478e6c8be1c072,10a41032d344b8b8,10cf960731ebc0b3,10d0bb328ba85338,10f14aeec5bccc57,11710f8fc0902151,11768d30c0efaab4,11a8a6db2cf59f7c,12293dbecf2b215a,123205af413eedc4,12941c5a59710c52,12bf919e5b7f7e78 +G-pr1779 00084a942091c317,000dc20867ee4e97,001cbc2dc29ff58f,002c0d18e19abd51,002c7be35b44a035,00307cb5486d4eeb,003676f4a989763e,00381a00a1f3c92c,003f3e420f19d3f7,00520247347ef185,00562aac85a4e3b8,005ef8a938d4727f,0060867fdb69bee6,006420b2bcbca8d0,0065efeed51a9d46,00754b0c26e8783c,0076cee69d0f6a5c,0078af6fa3e631c3,007c977a0a045202,008294a0d3c1513e,00853a75e60d2b61,0097ff15e6f35de4,00b0a162fd94b60f,00b2deb94444f4c7,00b950963b758ffe,00bf1f31bdd965e1,00c1d1b2135a6eb3,00d076d4b2fc343c,00d9db6f034b2cf6,00e9ae8675956c2b,00f0b17735c5f927,00fd2d2c0ee8abe7,01014943ce86aab1,01110e30adb05619,01169d16b78074ba,01169dc863e214ad,012980041ca43c42,0133a6a453379ec9,0138712a3a865af6,014981f5e1e2aad6,0155c8e1f74ce257,015b9b000a39dfbd,015f9cf07d3d4c13,016786873eeecf12,017ba5497fcd08ab,017bd77b04d66595,0180387e183a01b6,019b04da875d6a01,019b10bb59d85f49,01a62ae3cb5d2361,01af0ac96e73fdb1,01c30897667039d3,01c32c732085107a,01e012fb73d7bb20,01e22c083277a21d,01eaeeb820b59c22,01f137dfbbe230be,01f5070671774c66,020bcf8cc343f394,02215eab6c9cc470,022d06312d70c7eb,023b14b8be6f7c3c,025633a60e1af3b8,0256dab7fe3784b7 +G-pr1804 0203b65ecde2213c,02aa4b15166c3ddf,02e8b6b810a72089,03e44aa70dff0928,0480e50343561379,04dae1f3adc100b5,05707ded3387a447,058ff29c074ee314,0600bbe3b0c8efd9,0619e14e9b3df655,06b38dd945149aaa,07adb731a9157060,084ddcffd0aa8efc,096f03435b625862,099ab18c555bee66,0a08e2a4ddf94ba8,0a5797efc157c783,0a735e30a7d36319,0ad2c14c8866c3a6,0afd69a97ad2242b,0b4cae7fd62dca54,0d5d1d4703a59123,0d60e8f522247d23,0e0cee7ecec483be,0e345ec17abe0b51,0ecd330ce085e749,0ed9ef56629dd1ca,0f1a32f31e00b04e,0f308c8c5afd4b26,0f313d0942e7dbfd,0fac1f9544217298,0fae693170ebe3ed,10222f521c2594ce,1056d66c6f34362b,1142b4f4336a71cf,1150f30361dd3af1,11a030cb511c6768,11e58b5ccc10335d,1230f4335335c556,126b248f0616beee,128291fe30fdcf9e,1288275810223eac,12ada63318f03ae1,12e46e8aa0953e84,13a0fbc55b7a3059,144f522ce3dd3d48,17337c8010637d7a,174de3baf230dcf2,1953f25712bf6e59,1a7e44107f423640,1b7e57ad23659ced,1bad41bed888c450,1bcf34e4252f8f7c,1bd7f3483efe3663,1bf852a62e96af3a,1c39e5a63f1eea7f,1c78db9a60c7ae8b,1cc64be12d7b4e0f,1d6ef3b7af972d2e,1df5c8203e348cb5,1e37e2583288849d,1ec49286afa96639,1ef81b9d5ab61ad7,1f015eb855b18d38 +G-pr1919 0024f4c8a63b1e80,003cc64d201a42f8,00483fb9454721ee,006d674399b1172a,00835629964dc347,00dd22a77e0dea55,012a5051e19f36b3,015c1c4226ff6303,020b1ea785303898,0226d44616160370,023947e9c05a216e,028790f2b6d1b9b2,028f038c26850436,029ed319f011b300,02a86652bd02b3b3,02afee7ab6040672,02e8a7806d15dc50,02fa6485b55233d6,030cad1fbfb4246c,03729abd0d8faee1,0392b5b4faeef0d8,03aa0a89cfabd656,03d3c0eacfd9c3d9,03df6c9407c6e32b,03f8eac6fa3913c6,0422a5f20897aa7b,04ba2483ba3c0e28,04ffe61c5622e7ab,0541a8564782cfb2,0582a885ea125bbb,059cb5ae695cb8e2,060ade7c8491f860,060afaced9b7f16a,065bce901361cd34,066f533d34852a90,06a2db74d13a2db4,073576de13076883,07a81a3a36a1820c,07ac6922857464e6,07e6b48a557e55db,0828f5d428a9560a,08389b17a0bf7394,088211ee61b1e1a6,0894864934716847,08b84f628f72dd0c,08d7af4713e545dd,08f298b9c37ca8b3,098992b93526a700,09dab7ac4becd38e,09f9e19eeb0958d0,0a54ba29d0f82226,0a5ea1cb05e3eff8,0ac32f1b999beb74,0af24b7f4e8dee25,0b566d9b2b6b09dd,0b79ab7c16a84a31,0bfde798c8e36fcb,0c1165f35ab747c7,0c2e99d66857568d,0c71d74a2ad5e2fd,0c9327de24f365c4,0c948e35bcacffb8,0cb2749a09e5911e,0cb808f2fc4d9507 +G-pr1962 001b1cad8722edcd,007e1f4e73550908,01269eb559d74356,0141c35274918376,0186ba5acf713fce,01c88b62b29712e9,01fd7c30b7306570,023752eb37297cb6,0255a1eb706d6bc6,025a48e715ca47ea,025dabc7fd5b3450,02dc128d4f837dc1,02e22f814ee353c9,02e2d07818d844c3,031f8cf2ca65f5c1,0343d0685842a7ec,0370dfc1582da228,03bee2ae5d82c736,03c7f4a85f4f8d29,03e96b1f2e902156,03eb396dce16d552,03fb57d4384063f4,041a6e667005c4bd,043558b0805f70c2,04821f95485a9745,04b8a2ebeced6e81,0522a768acfc5501,05b6111db5db2275,06319b31b727be02,0648708cb4cc3ad7,0658493abcebb8fe,065bce901361cd34,0669e9de334876f7,06829ec471e18d0f,06858a6958d4013a,069e9344c0cfbe23,0745e61ccfdd8e32,077ddd942184c063,077f7830ae4c8b14,07bb9a9b0d1e1d82,07cc0f1c252c6ada,07cf1486e9cd2c93,07e8d50e199ab301,07f680e20d3cc396,0801385db35decd9,0818f24544f8e840,087276c0d98ad312,0881e8121d312ff9,0888d358ed272aa8,08c6dd12eb139d69,09250496abc1e1be,092de0c595c4c7c6,09317c3b59c885bc,097aac8e2eae4e5c,09898ef0701caeb4,09aedb454eb24d8e,09c915993912f525,09f1aec1c49eb0ae,0a1e42ef967f73af,0a8300fe1a550cf4,0a9ac2cbc17b8699,0ad468bf0485d43b,0ae301833c3cb50f,0b170c893aa8c68e +G-pr2064 00013b4e44910cd8,000667c2eedd9859,000ac3d339349b03,000b837e8f96a2a5,000cd8bce2138792,001da66a5fb8d2a1,00214f1d6699cbcf,002aa8ccf311f300,004c97fd172a07f9,005ce696c83e007c,005de808b7ba480c,00634b9c9159a097,007023d465f3b6a4,00753af7a051b236,00819ef6a545bffd,00888e31642953ce,00a473538d470f0e,00baf636064d678e,00bffca0dc2b41bd,00cdf5d1b720c0ad,00ee00d6f43c9099,01191d8358f75846,012980041ca43c42,012a57c531d7130d,013253322c9b97ed,0134bcf7eb52a703,01583fdf0b6ae383,016032862255c5c8,01689d5fc4e34d54,0178f3835185aeb1,017c0002bef6cf34,01b82229e971b32f,01c8234bc475bae8,01d0041fe6c53c1b,01d8fa95db4d5f6c,01f358bd0a465907,0207c91a451854c8,020bbed0824cf2b8,024ac765fd131c21,026cf2dbe815b030,02849960c45b3415,0285136e36de0dfc,028a6160f4214ea2,028f038c26850436,02c4a79a7100db06,02d08efdc64af1ec,02e5f15c09a80c59,030022d19f4fddc9,0304c3c8bc0d015b,030be60c666fbfd9,033e8e35c2daae74,033f147a02b1cc19,0350430c60179dce,0361e5abf41cb397,037057dc1ad04a47,03729abd0d8faee1,0379c6911e51b01c,038462950d4c13af,038983fca03eb623,03a25d9347950599,03be3526a7fa23bf,03d378dd3ae179a9,03d7d3d03dc0dc8c,03d80ae5f7bfc1fe +G-pr2169 000ba541defec402,000bde90c5116f3f,000d0dac6005a62d,0012e0732cc7c899,0014e1ec04527fe6,001b5651d32a01d7,002781c27e8d8921,0029abb24c3307b9,002d73d625eef08e,004ac58debccce17,00563b17798b588f,005fff7e66bd280b,0067a879de941523,006a036ea9c9e973,00712fb574084e55,0088092ddb4da342,0093f1c0e2bd5811,00a7455da8276266,00be6b09bc963cfb,00c850204ce27fa9,00db041210a59e41,00fd2d2c0ee8abe7,00fe25b144a7262a,0102d89bb89f7284,01079da2a8026edf,011a5c1eff5b35bc,011b85b785921199,011d3936a9b52595,0123f84845bbdb94,012f38b443b765d8,0146f45ae7a6d110,014a6f15accb85f8,014b270e9c339047,015920b6f83fac50,01622dd2ad1faf45,016552f37e1e5de0,0174fca5b67567f5,017781f06054ddc0,017cf6627b91e02d,018078fc86b92397,01884236b8215fd6,019be1298d9a2287,019dbe647a3bd234,01a2700721566110,01ab017b27569c7f,01af6c8586439f1b,01b9962fc8996481,01bc7c2bdce62935,01c08a7dbabf3f89,01e08b50b71b8039,01ea8e8d68a0aa87,01f861483590de7f,021343a6ea87d6e1,02309dc273e40d54,023900e6ba88842b,0243206518b093f8,026423f9f578b995,026866e35415999a,026c7ed468b914e7,0286cac466189999,0287e62c9afd951c,02967f54581e5c9f,0297ab4734835086,0298b544762aa3af +G-pr2184 000ba541defec402,0012e0732cc7c899,001710344c0ad3e9,001eff24456bc60a,001f208236e3475f,0034d7b104f1a7ef,004dda6cdf1dfed4,00563b17798b588f,005e124cf4a85cb6,00645bb63cfc5511,006598090d2a406e,0067a879de941523,00712fb574084e55,0088aabdabf3fd5c,008c20f13553855b,0093f1c0e2bd5811,009f83cfa9f77dd1,00a7455da8276266,00a771a73e6dbe2b,00bbc3436313767f,00cd3eef318b787f,00d14874e49fa85d,00d9db6f034b2cf6,00fd2d2c0ee8abe7,00fe25b144a7262a,011cc1c5e9ce3027,011d3936a9b52595,01311b598d6d1ced,0146f45ae7a6d110,014b270e9c339047,015007dd0efe3630,0160de9142854c11,01730d682c0e7254,019446d5d098db32,01ab017b27569c7f,01cdabe27d2b5625,01dbb9699da50c89,0201197bb41da981,02151f7c5a17e4d6,021c1de2db340c65,0238c5e62c185413,02593a6150f41ac1,027ffc4bb0263c0f,0283d81f7de6f4ef,02967f54581e5c9f,0297ab4734835086,02b7becfe5ffaee2,02b998b1022cca6d,02c4120edbdf48af,02ca433e8dd36f4d,02db450ec489aa76,02dfdf482cb9ed76,02ef7b34b81b39a8,02fac50789eb00a2,03020ae1af3e1995,030fab506d3d364c,03190ff404720f7a,0334cd5db5470521,033528993300105f,033c2c30c5041273,0346f4aa2084422c,034b8fc97456d53b,03598670df4b9e1f,035d0fd3ace7a0e1 +G-pr2216 00137a0cb34ca2fc,002ac1df835ea040,002eb6a9d78c99c5,0036e87ea62217ac,0044c6138f70cb5e,0048a4458d784ac7,004be879c75215d5,0058282761144a02,006c01be90c0a0b4,008760c133e12b51,0097e49a2a9d92ca,009c5cd229521f2d,00a0b128d9c971a4,00a35cc451a2033d,00cc7fdb12b5fada,00d61ef159b8dd96,00e9d0aa786ac533,00ecdd8a6860e958,00f0c897dbf38a36,00f18945a2ace430,010def1185af35c1,010df9c6ac449ee3,010f6d0a3f70b0c3,0119596fcba04b60,012448a941121aa9,0132f52fd2d11d5a,013c0d73a090bcb5,014fb4691f288bfc,015b9b000a39dfbd,016a63753dc963b5,01771148f3c58db6,017876013cae3157,01955a2cbbaf3bf8,01982bed00359a39,01aa1eadfc85f55b,01b6d42160eff137,01c71e365dbdc6f0,01d6290b2372a609,01d8e54f03203584,01e653e7445efae5,01e8fa38fe4bf38f,01ebaa14c11dd664,020119978c5a0735,021ee7ff27ab7086,022255869814c7c8,0249aa32842310a2,0251d48dd86b71b6,0252548392aa90ed,025d5cbb284baae3,0278516366937f42,0287ead6d430991b,028f038c26850436,029480eb12b5aada,02a6dc444a899b9e,02a77862f5e0408e,02ac0ebb78fe890f,02b3218270f65ec4,02b7a2f3c550d6db,02c186c029dd9c1c,02c2bfc2bdc56340,02c6858b41d79a98,02cd923af4488a2f,02e22f814ee353c9,02e68d69e8fd1565 +G-pr2229 00002540eb9026ce,000c297cf92e71d6,00106bd3236a2fe5,00120e7f77b55f99,001a1fbc9a138a4c,001b1cad8722edcd,001e78b2df26067d,0020d421157cfe54,00280c342c986e7f,003cc64d201a42f8,003cd8d8c3803a36,0043009facef2ee8,00469cdc71fe911a,0055e6e16c5eb884,007e0a1b208aab3a,008513cbae00863b,009068a8c62e3c68,00990f87f3885ad0,00a6995db76ab253,00a9b1cbe56acf43,00bf42c56dcfb872,00c850204ce27fa9,00ccaa8fa146dbc1,00d076d4b2fc343c,00d16e819a4041c9,00d2d7d320278965,00d8f644f121e849,00dc88a61f153335,00dd37b26ac9d5f3,00fd2d2c0ee8abe7,0103446f4bb540a1,0103ce90b666fa04,0106996fc591283e,010d93da867eca23,0111a9b734cfa632,0117482d7f71ea3a,011c070896a81563,01230c2329ab3f6c,012c5d9c797b11e6,0141c35274918376,0141d0a0e5905591,014749d3d96ced25,0148d3301eccabb8,014e782aaf4c9b4e,0152daf17bcb26f0,01592eba81c7a218,015f2e210de4114c,016084b58ef7e7d0,016f9dba1a14b617,017feee7078701ff,019aa16285987722,01a026c230d6550e,01bc5d843c619982,01c9ce885c6bdfbc,01dcf425f86f8756,01e1d1cc3f905541,01e34a9d9da32cc6,01e50b1b1c791448,01eab326388a8e4f,01fcffab1e6b47bc,0204562a2f91c614,0206c4dd45356bd7,02113408973947ea,021504bc5fb38ce2 +G-pr2230 00222ba12ab827bc,0036a0180295e93c,004be879c75215d5,005cc3115ec180b1,005d465379d0410f,00942b452733813d,00af97e85b9d2c83,00b4b82d423bdba6,00e9d0aa786ac533,00ebbff2268b748c,0121cf9865f7bda0,0126344a284ce20f,012d0dcb4abebff9,014340b2a68401b7,0151a2f1f623dd3a,0153c1d1a79aedeb,01615d50709c1518,016a823a5b0a1fba,01760e64a910cbc7,018c1112f41a0be5,0193ae115fc71e19,0199343db7e94a3c,01bc0479b140e48d,01c5ac76b7c45dd1,01ceb9f2c86b3e41,01d2022cfbaa68fa,01d2640e6dd9877e,01d32e8786dabb73,01e0aeda4445a156,020e1011da480b76,020f7b5585b1a50c,022289edb698978c,0232eb80cf3aa7ed,025c5611cc468def,027b5909f377f463,02aa68649f187912,02ac0ebb78fe890f,02b011aceac523ac,02b02584c245e68d,02b690662283ed01,02cb053f0c0b820d,02d90c8075429a60,02e22f814ee353c9,02f3d6664c7aa287,0312794e4396d1ad,0347e2deb6e19a68,0348f177ce00e18c,034f8d35cbe1c1db,03544b36fb96ede2,035a1df2490dad97,0360d13f104a1d0a,03729abd0d8faee1,0382aed8803ac19a,038be5cf294d279a,0392be233461382a,0393505ce249d4c5,039e7c52ee17dfa2,03c1283cf609a782,03c4e0712215665b,03c9c0855ff7622e,03e3b9540529a90d,040af0caac37f52d,040dc91427ae4f15,044403d1e1afb172 +G-pr2233 0001baed340e07cb,0009e4d225b13f65,0016e9e4e3696466,0019584c84a2a0f6,001f5178beb8e9eb,0041f5d717ff3815,0052121c4829b9fa,005b60524bb16f89,009a2b3eaea3ad48,009b8f55b25b5a88,009ea1f47028bd94,00e66043e2b77ec0,00f1f13f9ca78868,00f2541239e0f9b2,00f8a37068318db1,010be78c3b459285,0123d0e53996545b,012c5db9a0be9bf4,013d9cf3dc6a13a5,01445c8e915ea2e1,015986870bf6154f,016bf30bee17259a,016cfe3b95b68dd5,017aa514b8eb4e1a,01c2da8f829a37d1,01c9851008b4cd8c,01d2640e6dd9877e,01e94e2fd8da745a,01e9d29c79604209,01ff1153ee854037,021ae074f64d5f81,02336e90dd1d1e31,023fb57ec6bb2136,0254bd88d3647945,027363fdf12300aa,02a341a784f8274b,02ce05ad90ac1834,02dfff0c5eaf7890,02e22f814ee353c9,02e32b00cfacafd0,02f53bb3ea2b42a2,03179417a0b94226,032f73f21896d145,0333192eacf61970,039646af23e3d262,03a428fecd401e5c,03bd8f50be48cccc,03ea6aee04c5de78,03f699ebc6a66462,03fa5ffcf8bf4503,0409a6429fc10920,040b21f59e562eca,042cae861a281c56,04559eab9d24270a,047209239ff45986,0472662c2b6d7836,048f0dfd8a89b49c,0492fb1c6f20b53d,04c508a1a98042d8,04ccc3c234fe1083,04d0feedb4f06cae,04dea125341f1423,04fd43bccc021e03,0505908bd90accc9 +G-pr2266 000bde90c5116f3f,0032633d2121687b,00a5b05e4cc8995d,00c56df8c736f46c,00c850204ce27fa9,00cd4e1d7a4afe39,00cee2218a9238ed,00d2d7d320278965,011b5ee38e2a88a4,014690739d396c3b,01e085b79a600cd8,0249e9e93fc1c184,0279e3d3f5b36235,0284b3abfa88373e,028f038c26850436,02b4116d9c88d527,0301a94c49833305,0392b5b4faeef0d8,03ba61c02a255ad5,045ea274266a502b,047d061807442836,047ddfcc14224df1,04ba53cae59527c1,04d2d4650cdafb59,04e11ec79afcf66d,0544de53b7540a91,062e30f39614efbe,06361e145ce10cc3,06403f0e9d2bd232,06b6f1aafa55a5a9,06dbd074a202dfbe,06f947e4bcb5ccf4,070054dd7c218acd,071fcf0d3bc681f9,07378ae964789983,074410e2ea8e0aa6,075ce8d0a3cb2b50,0794e54f04751775,07d75b8a295aff4d,082348667cb37708,082625093298255f,0866207f2df62a84,0869307d8863ab4b,0899cd553316b94b,08af42cea7c5f1d1,08f2ca387d7babb5,0983b5532aea39cb,09ea041dfe75ef5a,0a5119c57b6a6962,0ae8d59d3d1f57dc,0aedfa23c0fa4ba9,0afe1e55eb03c575,0b74352f943477e1,0b8c6f9e8c97e37c,0baa788d8504ecaf,0bb0bd04c02f3877,0bb672bfc623a98f,0c806d5087be0dd9,0c8851cd1a8a8a9f,0c8856872fe5be6c,0c936914bbc8ee99,0caf9057a7c84aee,0cb97649f03fd55e,0cd7dbcea237ec54 +G-pr2337 001cbc2dc29ff58f,0025bf09107153d8,002ef88da2545c9e,003cc64d201a42f8,004443b123690909,004d9973da3ea075,00518e938c424fe1,0070108479e4ae8f,00be6b09bc963cfb,00d2d0392c797041,012980041ca43c42,013e53c77dbf2701,013eb888c4ad9c95,0153df77a85698ae,01595b5e47f10ab7,017b994545cc2e60,01895e98aaea35cd,01a9a2f08b826f73,01bdce457b6b0bd2,01c039dc63fed42e,01c226e901a1652c,01c30897667039d3,01cbfc983f569a15,01e1d1cc3f905541,020a7603bc1ea4fd,020cd658bfc72255,02100a774a720114,024715fbeedfad2c,026bc82521673df1,027037f5c6f8c9b1,028016725620fb29,0283254cc5482a0f,028608519b5b25fd,028e0dd1fbf80c3f,028f038c26850436,029fc68d5ee51e54,02c1218310e897fe,03215f316508048f,032b90b24991ccbc,0337a2af55915d6d,0350945a42c12c54,035367303d24a148,035b18793c273d20,0365c702e28d4e6f,03bd7bda77e608d9,03cec8ec4431279c,03d49fa6fe1234c7,03df7c96cdb78dd5,03e4b9b892d3739a,03e9ec9eeedc1513,040422aaf68e8241,04145be31315c10e,0416f2bbc663f7f3,0417929bf539263b,04292874c57287e0,043618b3e94d27bc,0441d354f02365bc,04604b245080022e,04ba2483ba3c0e28,04c04b58e3ecde53,04c2a665ab4dbf91,04e7435d87155799,04eae727516db25a,04f68d9e26e2d822 +G-pr2710 00041988c0b4b24b,0007148a30030a92,000ac0aee6be755d,000c6649b962c1f2,001564df1a35f2bf,001696acee569b2e,0017f5e37ed77e06,001a6c66b59c3b5e,001e6c6b597176e5,0020782948785814,00218454f9f6c323,0025165887734175,0029d154339b9107,0031b98ed22c7f6f,0043f324ae09ffa5,00465b78e7dad1b8,004b17e1abca2a9b,004e4ecaba95c53e,00505c2ba1117165,0051deb76f1bfd91,0052920435c3cf44,00543ce8cf33f9db,00547b07916e7c4b,00585b341d6817f5,005c6d3fa4d3a830,0065f8a6c350448a,0068cd0dd1cbd2e0,007301d2a701efaa,00746d8bdc013cf6,0078b3629ff26755,007c01cded3a705e,00801fab5d2b8871,0081eea6bfb2fd20,00822ad14378ec83,00828c7775aa4294,0088034c5aa7e56a,008bd4acf4f5a54c,0093b0ca0c8283b1,009b68aa2d6ce75f,009fa1c0b46b1955,00a14d7b98c805c5,00a1c9aa2028768c,00ac64f8bf0b92a4,00b38cf7ee351d16,00b3d61f695c9f7a,00bd57dd6707ce36,00be6b09bc963cfb,00bf55de30c8593e,00c27e988639d0ea,00c70fe766e60959,00d2d7d320278965,00dcc0d325f76577,00ded932ff15323c,00f367eeffc0d2c4,010122230de41085,01079c8197ca5e33,010b6cba87341ad3,011100e616887b9b,011753ec6d99124b,0118cc07f06594b8,0122bb50e2fbd521,01264b36afe056f6,01280cdf523129f9,0129fc970156d97c +G-pr2746 00023b4501bcf42b,000541eed2b4398b,00075deb13f920a1,000761f47ef724ce,00253941832cbd70,0025cd4c4ee16ace,0030587a8c599acb,003a826d3cd45172,003bad9cd1d594ba,003d281b65dc423a,004c58687d828161,00521971f6976945,0055312928108b89,0056851f62f6bb8f,005743edea4ca1b3,00576c6242680043,00579cab1d0e0d81,0059005fc9b23a88,005e7f7252720ac7,005fb3cd47c70471,0060f1d18612cf28,0066e3e5f7a5f7d0,0069946d2e3ec521,006e7b195df30210,0072478bbad813c8,0074de9f5f9d1330,0074f1af57f8b225,0077b0aea0456785,007b1b3cb86f9336,007c12e2b0bb7acb,007c977a0a045202,007cac525760b5e4,0081a1fe30a2cba6,0081ee3b80265e19,00916b03b841a78e,009206d9b7e054ed,0093baf8ef5cffe2,009554612ab203f8,009c5cd229521f2d,00a31c2904470bf7,00a568114fc79a26,00a7011f69d4a6a4,00af76cf89ef46b4,00b1ad1c4b2293ba,00b2a40644fd6e62,00b38cf7ee351d16,00b4d7cc2b27b2e1,00bb83a1fcff9de8,00bd75f3e885163f,00be6b09bc963cfb,00c35e884ab7e7d0,00c87da7fc2bbad6,00ce3fde08e7e18b,00ce410e679150c2,00cf23d9aa23abc6,00cf60ab4d49e929,00cfa3312b1f1619,00d28091caba1460,00d46d2b69ce2e2c,00db121df8906d20,00e18607bb68d019,00ef116ad7e3e8c6,00efe469022cf718,00f46f22ae42df3b +G-pr741 00d1cb0d38ae0946,0126afdc2edeefbf,02998b50fcc4d9ba,02a1418ef3983f3d,02c8f61259001af5,0553fab83b909ce0,05adcc76dbf7859a,0605af6237d67466,0649ba8f3de22d0b,07577f2f82762d70,075ead371f26531c,0915908871d4a9c3,092ad6fbbb0e1888,094e4de4de240646,0a283691f5f61fcc,0b52f2ea6413e9a5,0b6a83a0f8b54669,0b6ab74bc544dd6b,0b8719d7ab3a834d,0c77802d11f12e5f,0e020d160d0e4a79,0e0bfb1d597a0e00,0fb5d596b96b1f6a,0fec78707059b593,1255784be7eeb3aa,12e18bf84db3be1d,130ab5f5f8c309ed,131ab39b3ee058c1,132ede06eeb05ba3,140fa89d70416009,15d01591eb6a3c16,16ffdccc67e07fb1,17a27a3bde2360ef,18018fa9e8083e70,1808dc496b212cbc,18aac18579cdb03c,1ac8af11e42bd01a,1d81b451d5e7c114,1dae06a87cee41ae,1e808b1a430616d8,1ffdefda3a5b2fab,208f6fd430a20c64,209102eaf05cab7b,2133661df4ef408f,2158d5bed3628352,21b405a15cd9a9bf,224c759397865b33,22a702fbb47dd705,22f01aef734b0e63,236c9ca185932844,23d8dd868541c3e2,25876048fbc31b05,264a6b3a5f6b65e0,270c2f66a63432bb,272b3cfdd3ef096c,2785423398cde2b4,2789bfa206053df4,27d7d93ecded5486,2915eac63c02dff0,29df133656efab6c,2a3237866b6e911c,2a5283c4eb32d91b,2a59f5603264cb63,2b6cc2b4dfd7e51b +G-pr753 000c199894f2bb7a,002642836c691b80,002ccdb4ee1778c8,004719f268d443b6,00737712c96d00aa,00a81d57ce9b63a6,00af6c049db4b78f,00bc1f43b06d325a,01416acb47d066db,0147371cb5d82828,014751bd5ddb3bbe,0147d03ec048d48f,023023138970751f,023b82e4eef40757,0244b728716b3d21,0256fcfa28d5f54c,028cf8f92d4a8068,02a9d95e243223f9,02c898b7f682fe9e,02e783c47157d025,02f85eb09273dd50,033314e36c19d66a,0343e5bdcb55d760,0377d464514d4053,03910300a486be29,03d0ea5e74d728af,03e62f695db84d61,03eb49689c8983ed,0422a5f20897aa7b,043ffc2e60852e49,044ab519cb16f1b1,044d1fa3bb2a465e,045a162a4e227e3c,045d0f65a9d5f94f,048babc1cb67a1bf,04b8a2ebeced6e81,04b90735343eb84d,04ba2483ba3c0e28,04e4843a8b0863d3,04e6d464d5e30df3,04e7435d87155799,04f68d9e26e2d822,0554e9fede8f2da5,0564967bc112ab84,05677a292633ae65,06065cb036f13c7d,060ade7c8491f860,0619a5e88e5f7202,06391152f51d3b7a,0646105bfd5d7b38,06972c297c42a2db,06feaed4375082cf,0710f0863c847a19,074d1847336b7105,075cf52aa5e64161,079674fc372a6c8b,07972bf9fcaec847,07c36a1d578bc2cf,07e3d4ff49507eb0,07fdfbfaa4c9b661,083ae7ab13403e0c,0891d17d3bafad72,08f1b41ba719601a,091d254637b66e5c +G-pr888 0003aa867f5710dd,0012411dabcfbff5,0015e4101fd88ffa,0016fed4ea0da30a,001cbc2dc29ff58f,001e0cf4f12b6d27,00261e77b36f1ed3,00261ec95457925a,0033e13503eca165,00367609e43d03be,0038b25a36f3c1ef,003948b4d5404640,003b1f9422a92467,003c465251da2515,0041d32e9a6e0cde,00422d623d43119f,00462cb3241138db,004942d59f2880b9,004be879c75215d5,00508f2e29a35612,00530f29012b1b66,0053afb4bc8304b0,00598b9b208b2ab2,005d1c65a5532f06,0063e375a3f37880,00653dfdb5f2723b,0076730a753b06c3,0076c206644b288c,007ae69852c04d36,007dc40f21ac57f5,0084789340ef6784,008604b34f547983,008c1e138c7391b7,008d8e26d43cad23,0092a4d63375407f,0095a65cebe8f7a3,0099f60f595fc1c2,00a10e2ea466a7de,00a82b6e69075c00,00b5b00c9155e509,00bfa328a824ca0e,00c5f9d0c21a53f5,00c7469658cfc327,00c814318b7f6752,00ca1dda497c6b5a,00cb5c4cf8291048,00cb60562390fdaf,00cfa3c068954758,00d20d94369ee46d,00d5aabbced06d1d,00d839d5ae71b7e9,00d8d62cebbc31b0,00d9459692c16c61,00dc025b929b1bef,00e2ed709bd764bf,00e438ae92ec178c,00e49dcd1e383cf4,00e6aba249c8ef7e,00e749795d1c3d59,00e83cdb7888bec9,00ea421764c9b82c,00eb743a47cea0d2,00eebab7ef8039c8,00ef116ad7e3e8c6 +G-pr940 000dd25e8edfc41c,003adaec4ca9b6c7,005529b2c454e164,005640bb5a7ee93d,0061ab4b59607fd9,006b9d8b9ad6e438,0076c206644b288c,008c460f80d3c629,009dc2203d1ab24d,009ea1f47028bd94,00a0808c813e2189,00a97115cec7a6da,00cd24e9ebfc7060,0121c81cc8286a65,012673ff104a4cbd,014db5f21c696ab2,0155a98f6164d6cc,0183592621babd69,0190b8bcd3684808,019a3ffdac5699d4,01d29a5f04436797,020db6d2f1d9afa4,0248350d9fcbd989,024dea284afac223,0250a1b5fc853224,026cd936c7d7b610,027058cf542af58f,028cf8f92d4a8068,02ac0ebb78fe890f,02e22f814ee353c9,031a6d7ff8f6b1b7,0397f731b495ad7f,03dbbb82df412825,0428838de537a67b,043d6d9611f92e61,04821d5a3ad6cc34,04dd7c0c836318d6,04ddba57eab257ad,04e175a288a2cc33,04e7435d87155799,04e938981f04a7a3,051fb18a1e7c2028,053551963e90e92e,053a45f6f81127d3,053e5b0ff13c2cb5,055e819657f3b804,058f0a14aef6170d,0630cc760b6cf6d3,06376eaae0d0c5d9,06522a7348af72b5,065eb444624dd4c3,0661259c93bf2262,067f07ecf063d261,06af808f3928207b,06ca837bf1fb9709,06e2529c71938804,070de833fe2ac895,0748ba87a3d912b8,075a9b3fb9dd68f5,076556b12da04ea6,07848cdf3720b4b7,07c7639dc7752811,08297316c9ee2146,0871675e72c173e4 +G-pr941 0018e7e8b46f516d,002291121c0254b5,003adaec4ca9b6c7,006b9d8b9ad6e438,007da1c25b11a274,0084e9ba72f99698,008ed1356d99c790,008f0bf20e57e4b3,009dc2203d1ab24d,00a97115cec7a6da,012673ff104a4cbd,0126c2f22f38fea6,016f05a1b67ad3c0,019fceddd4d93f84,01a71b06415382b8,01ab67b1b6efac3e,01b9b8c03954852c,01ea010f57512e5c,021f739cbea02f4f,023ad2f369c84f93,024e7a772f02dba8,024f026c490bd924,02564b8cee44008c,026cd936c7d7b610,027058cf542af58f,028cf8f92d4a8068,028f038c26850436,02ac0ebb78fe890f,02e22f814ee353c9,0310e279a66c88ba,03162f118ea5e0b2,034b8fc97456d53b,036e4b87e4bc4b84,0387287728fe812d,03c1283cf609a782,03c36e5a47031abd,03d880ff9376d033,03ec2a28e07ee84b,0407e4ccf8f129c0,0413e9eb7910e1cc,0439453ad335734d,043d6d9611f92e61,0456e1961ee030e8,04821d5a3ad6cc34,0484357646190e64,04b34275d653eccb,04ce1456a095a07e,04d9358dc067c536,04dbf534aba494d4,04dd7c0c836318d6,051fb18a1e7c2028,052b6c71ceb41eb9,05440d4e8a78408e,055c17f433e5cab3,05785c6dedd898eb,05f01203e7adc331,05f796e031f0b8fb,0630cc760b6cf6d3,06376eaae0d0c5d9,0641a56dd48435be,06676011a595a43f,06e2529c71938804,0738b61c6114bfd4,075a9b3fb9dd68f5 +G-pr953 0003ac4e2092ae65,002b1ab86cdbd954,003284d45b7be55a,0033e13503eca165,0056859d6e8f315b,005a019bb881f943,0076c206644b288c,00772a1a7334c28b,0085af65153836e6,008f6e514c37947b,0091198df1d9466a,00adc124bfc080ed,00c7a6f5c43e6f5c,00d1fb34e56dd0e2,00df91b80fa8466a,00f2bce2d5c2da5b,00fd2d2c0ee8abe7,010eecf92eebf90f,011480f318a05ba8,0132ad9ff8a4dc15,014981f5e1e2aad6,01501762400f99cb,0176db30afa3b866,01860b84e713aa12,019d1cd84e8bb774,01aebd3e1255909a,01c40cef383efba8,01d22f8ca3cf76ec,01d2640e6dd9877e,020b8a1215db48d3,022380920f651a25,022bf1afa98b180a,02448bf3c95ea095,024bf04e8a3e6074,025047920c1f5751,025633a60e1af3b8,025b6052b9f3229f,029f8dae82b74443,02a3d1a2c0400e97,02ac0ebb78fe890f,02b3218270f65ec4,02b3b46766f0acef,02cef6d2e75b04a3,02db7f3682248eb6,02e22f814ee353c9,030607e01536a247,031158dc91f8d3eb,03269ba0bc5f1ed5,033941c08ccc5d8f,034b8fc97456d53b,034d6541c51de8ed,03573c38d9939ef7,0371540852a6c0ac,037705c75d886a2e,03b40d17950c39df,03c33da6a4fe4f0e,03d8ad0fc2c1cfaa,03e6ac5e9531696e,041326b6a98f578d,0421bafb247d45c3,0422a5f20897aa7b,0431e532fbf494e7,04524f10e017664a,045293822d12fcb0 +P-033caf9c30ca 0031360a9a9d04f6,0144c9bf49d75bbf,01c14e1b6b2ca7f9,02497922355453cb,0324bbb6a45e9b2b,043661bbd931bed6,04e678afeba2b625,065ef25de1187f68,08fee363b4f95e91,09a42091df735cc1,09d9e872684b6e71,09ec91c7cf5d52b4,0baaec80cb1c2388,0be80348821f2a0a,1034cc58df3848b6,126549f9d83af951,153a12c4953f24ed,184c01f31e7bb76a,1853967d1168436c,1970bd75e8ad06ab,1a5a31d97d45d6e4,1a82f2181027ab2d,1b5d31865a070afb,1b6b250b0bc15300,1cdb12f7b00b59ae,1e9640a8c1a49f5e,1f2aabe8118f3c37,239f1c8c27febaa9,23fe0c6f0f430246,26c43089efb46e7f,27cbad30b145967d,29975e6fbc407ae3,29ad2c618ef2a6cd,29fb953d7beca8b1,2a3b58565d7e43ea,2af827d6ed05e133,2b194f34e96561fb,2f6c3cd8ef6dce4f,30ac3f5b76fcc717,30c9746c66046836,31c6519255130336,330a16d52f793f56,33431db26a57a68e,334b25ac7f1738c8,34003d36ab81da93,34acf8ddc6c8a8de,34fc62d6b79ec247,3590235ff40b3abe,35df1d441929111c,361f3c9181628f57,3714b3485056532e,372e544b03438e93,373fbd8f1ff47a9b,37a114ca03d8fb5f,37d4742f19e72de5,38898bb10cb6bc2a,38ab0be78545b1ed,3a7574b67b5121ab,3ab9dc1cbc565d3c,3b206b8d7dfb3070,3b7a504fb9d41baf,3d7181d14a8ccf7e,3e1daed23f39f0ff,3e548669dcaa6fb9 +P-06a15fbc27d1 0219a4b8744bc9ec,02d44b0ebc1b16de,05197755806b3c03,09c80db4f33a8a81,0a8300fe1a550cf4,0efd00ddf0a37ec4,0f16320d7849fc7a,0fcb2cc4036cc1e0,119ec154a0f674cb,124f405783787e3e,142d9bf0e52a3734,17fb8a73afa6cf51,18a94efacc4762a4,1dbbe4c2e7ec40d9,1ebfc26a6b6d10a7,218b4029a4a1292f,2240719ecf195028,256cdce2494185c2,2678cd85fd08ffe1,28b2357c6d2bcec8,2ac2c02b62eaadd7,2c4a915d70f933dd,2d67e1cf1e1161eb,2e7a5c9414c6cf06,30121b3a3be97a18,318d84985251d47e,31c183568e70f8f4,33c6e0efbc21520d,375c8cb343fcabe1,393ec2f3a2adf146,3aeea34d73235e0d,3bcb22cce6d97435,3ca10af6d2a4fa22,3cd0628ee1f2efe2,3eb3399bc4ebe96b,48936fa135bf1acc,49672849d06e2915,4e9033b73abab65e,5211658a4ab4fad6,54f905e8795e74ff,5698ce18749d56f1,5804e5f55815c3d4,5cadd6ba65dc89f3,63406dc67cb9c311,6567c386acfc4a45,657594f31be42aa1,6b9bb7855e7aa4e4,6c2f07b806495e8a,6d10708bc1ec15dd,6d436cb439019b74,6e6b27052e65bf05,6f3fb02f02333d64,6f7ca05ab8107092,734e44a0dc1d444a,744b0b684ac268bd,7a32ae52cb1c18b4,7abb9a493c8c753e,7e323aea4f00c428,8139b5e8e473e851,836f95f2282f615d,85fafaa7235439b2,8d44cfc5bbe8e1ba,8e598ed3577b6045,8e5d87cdbe1ef11e +P-0afb0412cc4c 002a3e446a06c6a0,00a639ebe59c8534,00b366f2f06ad5b4,0260830850ce2bad,0272c42ddc3f279d,061de0caea99ed6c,0742c588b37f8e1e,07cf1486e9cd2c93,0c614cec5fb20d06,0c8a94cebddb3faa,0cf12a583410311a,0f00dc50d5526ba7,161a904c672262d6,166fd9bad49f7bb6,173dce8f726d9735,19b58c88fc0fb627,1a38e4b44d433a39,1a75432312137ea6,1bdde652a2cd8bba,1e63a1e0586edb7b,1edda565dfefbd64,1edea0db3b60822a,1ff97ae5a27ec16b,21b38f1ae2fb1be2,21bad5ef69cb04b8,22eac16b4423f8eb,23567820cd06ac2e,249d65af2d59747e,271add331bcb5d22,27699c6105e0a5df,2c4b410aac6031a5,2d4e67b569f5741e,2dc49b02028aa9e9,2f3f120f3dae3ef4,30986557857fca9d,3128a1bc086a4594,33d49a69e07163b7,3478f7a2db6bd187,35fdabfd43b988e9,37928884236e4719,37a8b4e7b91090cd,38676a07bf9015e5,3983a9a056912c51,39dc3a57d7fd7b47,3a2d98a5c8769334,40b5c9bb11b05b04,4169823207c479df,431703a855858e32,45bbe4f84bc44f82,476861b6f9ad6115,48c9fee98149edb9,4b3eafcc9b885158,4b3f1e094d38c7b8,4b469c481529ec8e,4d10196b97a1205e,4e20a533ee867fa3,4f61a3d8802f4c34,503d6201ed19f681,52bf692f007af397,52d57210e7239018,53146453c100986a,54e72e24a136354c,59cbd451c7e3be22,5dafab4889d8db13 +P-0cfe3b858c03 01714d45f84edae7,028c8d3f3f7df3d5,0726d577357449f7,0846c7c2751b1876,086ac1154cbb5f7c,08be6eb51dda90b5,08d8f00b788ce0ef,0a6d9ffb1d047315,1192ff8a895f94c8,17e667a5bed482c7,1861b4c893ca24d8,18665c98aac9c04f,18f732d41d7d9aa9,1990780d7f3210e6,1a23811aa495eb0b,1b5597720d45ef0a,1c3b433289d9028b,1c7545ab6bc682fd,1d9d5b4a2ac0f04c,21374ad14b9cd922,219b6945621b8dc2,227b0dd7030c4592,23cd1540336678b5,23de1194e959c2d6,26688c2f8f6f13ed,26ae1467e2bed124,26bef196f28359dc,2890ea48d0faade7,2954dc8ca2753b09,2d298d19ae812ebd,2e50d302ede61715,2fa6300993acee87,3073dc8dcc19a5e1,3245b1aac6eba79a,3359a9420142f0ff,3593c13275673cee,38085f93de45b59b,3a0ab5bb55f20aa7,3d03f27bc9261d45,3efe7b822febe969,3f71709065c3ba42,3fa966904100c21d,42c1a1a221724422,4324e31e3e3f900e,43807f23e0b9eb08,445fa6a8608c29bd,459c1b83c95245be,49084c87c1dbcb93,49b2db4ce2ea085c,4b151a2b9995e03f,4c37532e8e362396,4cf13db73871dbb1,4d2f36b548f9fa48,4df68b08a93a4e50,4f11022c15fd9371,4f3b9e0f91440d61,5248a9b9d46b9ce9,54710ee8b620b006,580ae607064300dc,5850981f94312847,59201dba352961c1,5a1673378a860795,5f5c34653df91f29,5f7b68e8c74d1353 +P-101aa47005a0 03e5caf585baf7e5,0497bf133efe84ee,0628895c43d59c2e,07e20dedef07b0dd,08ea7555e5a85e02,0d12facfe54afd79,0d6d6770891d894e,0dbe522f4006efad,0f034b7d5489a6ab,0fe7f7b15e44de2c,11b863068e7770bc,14863487db206c6e,15cb808fea8e967d,166cb7bbf1dc3ced,1728bb0dedae5efc,1770187554eeb95d,17b719760ad123a5,1bd6486e3dc3357f,1e2508a706332521,20061711ba219367,203f643db1ea374a,2047de7349b972ef,2064355e9d4ea205,22c2d214007aa2b7,23cbaa5ae23982a2,24c169b8a3096ac8,25d43d72f0336953,25f2afec3f9fb186,261669f5651a9315,27b32f8d5d71c393,27d8b5cf448e1017,27f73f18949a8ed1,2802ddfd14b49319,2905c9bd05baa134,2bc099ec6c573fca,2ec3f234568018c5,2f2412df19e08387,3001ab4db497865e,30b5681595dce44c,3152ffde1c8232b6,3233b72b4c8aabd7,33a63fa786a5449e,380ba02bfd8b565b,397f8b94c0a6d6de,3b46babc1e3da07a,3c02e2ddba29e3a2,445317f50bbfa594,4656c5ced2ea5693,4a048c58b36d1e52,4bbc9fc7973603c9,4c9894ca2e4799e3,4dfbb29cee537120,4f5e9078f23f7e1a,52673d72c66bf53e,527490824704ca96,52cd2f7c8e10672d,53f205b6a1c8584a,54808939abaf54bf,563f05c98ab78826,5b58cb0d7e6b982f,60e05b34b9e3f2cc,612b01d3aeb11665,6507d05eb1d4f3ef,653362afa38f7a56 +P-1486e2af4c6e 00d766503d58137a,037cd5dabe7d5470,07cd45b638fa3481,0c0105ca8e529d7f,0e2be6d4ba650484,10a78eade99fdfc8,19b27632d2def6a9,1bcda4a40d4f29f3,1d6fcc2bb869cd7b,1e2b532ac0b897ad,22eb1739bcdfe4c1,24c927fd30ddee54,25fa771d8ed8e802,26c11c4eeaa4bc28,29cf4908b74a8904,2b091acdce7e12e4,2f702bef6b4c1914,30ab0ae196fe7663,313f205d5a334e7a,321d78aa4cc54b52,3724bfc8dbee6374,39725db619ba581b,3c76429526cf16a0,3df706b21b75479b,42e675a4d64dc798,438fe407cafda3e6,43e60d64e2d37e3f,4770998e21c72620,498dbd8f7b236e04,4bc6a0a476d57d17,4c4eef32bd59c85f,5018a765a98300c6,5348b5c21cc0120e,5b8440eee0885a1f,61535ec1312aa670,62fee4e0aac45071,6c63122547efde01,6cff3e6389e1c5f9,6d992994ed16ca33,6dd1950385ae4928,6e1d85ff5c486ab0,6f871e8c39f6b465,75feff0874f4598f,7691296cd8709507,780ef27e29745000,7c74a94f9328e566,7cbc7a0db6ea02f4,7e4d3e2614122cd2,7eed5d395969ed16,837bce00884209c4,84d264380b7b9815,8ae8c6213d935f95,8b6f9b5584b42e84,8e114e1350aff427,958e038d5d73ae6e,98f430dd786b524f,9a0edf86dcbdaf21,9cafb343c3f942bb,a10ea1492cb3592d,a4c837bfafe20ac7,a5e89ee19d3d0a98,aabbccc85cad32b1,ab38e95f0e66acc3,ade63539141f7003 +P-16a3b1d9bab2 00cc2b8626be8094,01d5b77a2e9f8890,01fc03fcc3f56703,0260830850ce2bad,05a1826a62e87fe2,0742c588b37f8e1e,075269c7b4f0b592,07dd0ee9e1d02e5a,0c497fd7b0c83fc8,0e40f7fb519917f7,1133d2035f4c4564,134922756e9c6a23,13bbc369520bd4f5,167fdd471b37b4a4,1d0079a8b8a1148f,1e138da228a4522f,1ee03840eb41681f,206e65ff95404dba,249d65af2d59747e,257cda98d46b9c69,25df20678093632b,26c7d5e609c1e5f6,27a24b2ab2981bcb,28ae323c71fa7541,2a172c61ae9ddf0a,2a59f5603264cb63,2e942542c3f26424,2f8dec400daa1b21,32d81ff1f104eafb,33f6129fb00e025f,35c942b0e5f3ff1e,370729f1cff5782e,3752232e4ecc216a,37fdc29d51e0a1ee,38c9652dbd4964d1,3a1d8a4d52d9b1f0,3c9135b9233632e2,3ce3083753cf0cd3,3e3fdb912f8b9c63,40d275fb3449cbae,46d4582054848e0f,4ad25fbdf974aa99,4b3f1e094d38c7b8,4ec32b29519fdff0,520b1571278f5c32,52b464104e269b65,5336a0102628e08c,5372624cf0b645f8,5474d9c739bc8bf8,54e72e24a136354c,55095af65ad67f97,5536ce1c8cdfeb3b,59f92ba2f3e0961e,5acf3cce7b9830ba,5e94b0acb7b93911,60e4669578cc05a3,618475896e255495,6234ccbea1a7a596,627fbc03abd0d5be,65098b49ec8fefa0,67ac2e1b970b44b0,6c417b7020cb71ee,6d386708a7b8ff3b,6f85de94ff13a5fa +P-19fde4fa1f1f 002cbb634d382921,0046623124216c9c,00bd39e7d09a74c5,0186b2dfd5a1840b,0297bc4c3afa6a65,02dad0ad7d082b92,036827d07b58a3b2,04c05dc8f0939cfb,05443ce1a0c403db,0625c4eae186ffc9,078dfe16313bb74f,07c4269ee8bf4ed7,0892abd706f00e3b,08c448b835f14c7a,09b6f2d6113925ec,0a0ba0bfc1ccfe27,0a90704628f6c859,0aa649006e2bb6e8,0af9c9950fccdd12,0ee3be0e2e51164c,0f5369d02eddb1ca,0f968cabc224a8dc,106f8222132051b7,10835ef11fcd5a13,119670699e53f82c,1224e792cdda99f7,12858998987d18ef,128be9753bce3e7e,12afd6b7c5bd07c6,12c5b5d0ff9d6dda,13c404a77226d935,142a7e3d60a02716,144596fa27fb655b,14761fedc82129cb,1599b7fa198a5d09,1735c06d6be5fb36,17a442a91ad822ed,17ec64e079e1012b,186cf1fbe0a0025b,187750c97b69dd5b,1890d155c1fd0e96,19b50f6b0b4a4934,19b9e54be3b944cb,1a0b2bc0a43c8b58,1a6dee2001184d7d,1b29c9accbcd1e30,1bcc15b63fe01560,1cb95981413c868a,1cf5e035d93d11a1,1cf6d4f00899f2cb,1da5910c1be592c1,1e420d10338a5c63,1ec434f081377fe0,1fad4fea668ee2fd,20a2c603b1c31ab1,20e2ded169c35640,227539828427a87f,2745fb7a83e0f897,2799581c044389f3,2acdb9c66ee90f94,2af41c6501c19c36,2b79f8e7f87c4d3e,2bceb9834b712ff4,2cf735a4696a5d16 +P-1e8aa7eff08c 001bb9230f4b66ab,002a24192d9d1337,0064b3d5e0022b00,016878fa6d13a479,0181ea8fab19e87f,02895c00ab8471d5,0314bcb603970e05,032396e944b5a371,039a28f1fc465570,03ff3365a44b29ad,04746009a150208f,04b5e7af148ccd00,05030521a81c1bd4,05647f802fd8b0fb,05b1575267bdb28c,060ade7c8491f860,062733b2edba1e91,06413444c8da5691,06479e96b48491b0,065697412c69c5a9,0666bdff2b4199a5,066b0dfce8f4921d,0673538af7cf6dfa,06755e5685a5f4e1,072bb7315a41c4b9,076e65da7e47d8ff,07d01f880a60df72,08eae0ed37645b31,090ca0a9ba7dc265,09650c81711a06f3,099d1855ff23ae19,09ae8d5d472d6708,0a10a8f226acaf12,0ab7283f28d89ba6,0ab86fee0e86f198,0b959d1173c2bad0,0ba06deb2ed5778b,0c58c71690cb1073,0c6178152a731ea0,0c894cd622f76317,0cd93897a93d432d,0d91c001f4f58da3,0e4d67c0ba51a592,0e708530a2e5e936,0e79eae83fda2c80,0fbb55db0498bc49,0fc49d9baa7df049,0fcc99139d3bd9be,106f28706b0dbac5,10f23c2840f9c3c9,119d82db87824429,11de5e4849733d05,122df9eb5b804a29,14e649910862ab5c,155d7d114524aa81,160d4fad6054af19,162747fbc4ad8420,16309b282e32b815,167d0aeee3366078,1684e9c513c1c3b8,172e30b6c43037d8,1896fe89c7a1f13c,18d57c7a7fbb8492,191e85f11f960e35 +P-2f194b384bba 01323d4713ad31f0,0160b5079678e06b,03aceb90e3e74e88,051702c4fafa7f7d,0b94aeca2b5bc0d4,0bf39218346c3e82,0bfa54024e077b1c,0c079c92d741f075,0d91c001f4f58da3,0ee2e07e78323597,0f018b2d079a6a4a,112240fcc4aba239,117a485bc3858c8b,11e85243eb6c01fd,1292e28dd3432e19,131c1c1ff008a070,145e7f375d1590d2,149b6192a1e38cfc,14b9d9639665498a,1821a9eb20f4383d,19b1701e580154a1,1a5f92c6f0d35349,1c33d0339cc0904c,1c844c520dd7609f,1cdc7778fb21c62e,1eef643945d30e7b,23a23ab123bcad9f,26a85819c19d0abb,26e70e12486b2b08,2853c8783304bdca,2cbe998658e4e095,2cc9aa8f35a0bd53,2ce06473e7302dba,2f70edddeb2a6582,2f7e0f9c831f5aa7,30f9a1bcb0b3a18c,32238e3b12656414,34048c07b6f291c9,34181b450cb7a0e3,3937efc9dec95101,3957319237b3e3eb,3aa7fee6159c43d4,3bd6766dae4a92e3,3c847b319938885a,40bcc3082232c2f1,4104cbee8ccff71f,4199a5230c303e18,41cf645d8eeffc2c,424b5e1ef2cdee5f,42ad5e915284dfe6,42cb26eed468e6bb,4300a2225091b398,4313829df35c602d,4572fbc37a21975f,46303ee45192d1e9,4660468547983cf7,47e0151ab1b4ab49,4a00f6716de7f469,4b4745d8b656fb58,4cf3f2d8ad4ea2ed,4d890d943ea6aa40,4eb810e68e9c2c62,500ac9ada0f961dd,50999a0876a2852b +P-32457b5e3477 0044eb77685bdfeb,0068d1094c4bbe06,009c13b9fb220f21,0131f98cb3f2973b,016d1d5f1f85e4bb,043619a433ee4e4b,04db2c0565797bdc,04e38bbbb529599e,06b756356f509dce,07a5d18d75c9c2ab,08cc36975a1919fe,09304503038f384f,0bcc56263b4d3112,0c3bfc742e2001ed,0cc42ce1495481ed,0e5d4f83ede70455,0ece6dd69cf980b5,11639fda322f796d,12a3272ec4506ed3,1a23b1c5a3bd9bd5,1a2fdeb6db53bd23,1aef3a083549ca2d,1cbc5c71f9c04e3e,1dfce4e0040e6493,201b91b53e06c758,2313bef11dc4e224,25ca104dd3bd4f6d,262ac400790ee040,26bab762488aaa9a,2774a77dd72c19c2,27a9629ea3615f98,281a7bc6e619f3a6,29f69fc25d48f91b,2af63e44159a2f4c,2b1b01ef05465a83,2d47eb071d39c125,31a1329696c58316,3367c88a2c15326c,37a8b86c100e954f,386d2ad885004d91,3929aff31dc47e8b,39e912dacb4fbdb6,3ae9162735e2f3b1,3bbd057a84f849b2,3cd790a8e15ae847,40c406a656df8e93,42f8c61fbb8b3b0a,466b1c2c4c611fce,472dd492426b88be,478de2af90b07985,4a3c7861d7b26eb4,4b158bb9d221d4a9,4ec4f8564bf653c3,50858b72ebacb84d,51e155b0b3f8e2b4,53d1f235c232ebc8,54e72e24a136354c,59045be7ded00b63,5f57360bb99198e0,609e271475cd61a1,60dbbef73227241f,6107850ddfead303,616340277f3559fb,6818f733330f4b9f +P-344775d57610 02a221e54083e80f,0c974fcb247849da,0da89386c153688c,0f828021e6cc6b17,1094e49712511c73,15179a10bcd181d3,15d7e3e9292bed4b,17542c6c357de893,197bcb150953fcf9,1bb64b1eb1f56a14,1c1560a569a9a594,1c60916c2d49a2d6,1d137946367dc770,1e20d9dd2a388e6e,1f16fbd451c57ad9,2489af326514eff2,252df6726a5f83d9,253315362f2d2c80,26ac98ac053f2e1f,27e5004f7af8dfb3,2963fd7960abad12,2eec78ed24f24c90,3046aec4db9516e5,3061de40de31e272,3321bc714cc69303,34e5a83cfff242b0,3599050905afbf17,3725d898c3bb5d28,397f448b15efd7d1,3a2269b784b2f8e3,3a7ed6f2f31969e6,3f9c7a5468c67798,405c142f5747338f,409e583aaeeca829,42bc5ce51eacdc30,4e90ad0eea73df87,4efa1979243f8531,4f64c4dfe8d0f6d3,4fe923af2ac6ae08,5eb695412a7fb7d3,60179c72fb08d269,60b3c8a33394973d,627dcd359f587bd2,663b5cde17d8a6b2,6729c2127b07a0d8,6883eee514d121af,6a9d8a02ba7e7f41,6c9d307df6d1aaf2,6e30a063b6a6e30e,6e89bd72e3dd41bc,6eb8844fafd71750,7023558f450f96cc,702fc351aa456024,7090cd741d019ba2,739e933615d87334,747ba8045e569717,7605c21d09429ecc,767ecefd2d996983,771fbb806b335449,840bfadfddfb0447,866edd7a85a4bc78,90a7c38cff015690,911a80fbb341898f,915cbb040a4751eb +P-3b35f5a6934a 01015dc44f66ddd9,04790bc8de27db3e,054ade0a2341d529,06a43da00e557574,078dfe16313bb74f,0bb842bec0687d47,0bd41e3573ef56a6,0d340feda9119e7b,10e550809a0e141e,150975814aef6768,1772d1f9ffc8aadf,2094c8cbe1860526,213644c32d9f476a,2176ea3034f8f533,255f2324040afe9b,25d0a9247159fb29,2857f61a24ec017f,2a27ff7d653e4bfd,2b15e481204ac82c,2b9dc7b643cd4829,2bd5add2988c8520,2c78de0f9310b2ab,2cdc8f8fb8652c93,2d136fcd70103af8,2f24facb14dff51b,309b7c46b4267301,30e6b111d31ad9d1,30ea55652567d87a,31adf1eaf2374f35,33636748c55421e6,33ad21d4ce075bca,34bc85207ba4ae2f,34ef04a33ad5085e,3ada3ee3b6a46303,43dbd0b2c5bd48eb,48c062dcb7fc4474,4a5c23fc890fa969,4d1c33d8aee9b595,501e3a2abf88d4b8,516b785e7f0b7c64,539459d9a57bdb10,54364ee07ac3eecb,56439fead947d634,57534617133eeb3e,5b9c6b38b053f194,5cb2a57d35730c54,5d80cf2ca09f8f43,605d17f0eab1ac64,64653f85cf4faf33,66f991fe1f90d00b,6835dda703c1ff2d,6858a5520c24abee,6a8cc28c202dad5d,7047daa478d15588,70482ee1df7ea96a,70aac3dfc48339bd,767e26e0c7d46658,776d6fc6df07679b,7a04afa641367766,7aea878c2cd37a2a,7b36874a308d7578,821608760e54fdcb,85671fc0004fdb66,86cd22aa9080a4d4 +P-46e545d0621f 00118cac18ee991c,0b4b16713ce97d37,0e2164342fdcc1d5,10374532d61c8b13,1065ec3b043d9b70,1097072987be7950,112240fcc4aba239,11b77016a1272a00,12104448d3045c3a,1416cab495a8becf,149ecc82dffcb68e,1565fab5663c5de0,158cfc15ba372385,16da077b95cd3495,176c3bf75d07d5e6,1773b1f7d29f793f,17a2086ac76a2c1e,1c8ccc8892369562,1e63a1e0586edb7b,2021b6d2b99609db,21054130de4654f1,2290de3468ece873,27affb564f4e8d1e,2ce208b3ea459ce9,2e34b5039d77ab43,2f9d4731e700e685,303d93cebde360e9,31e11d7fa8ee8071,32bb4b7ab857414c,374bf8ea8a259cee,37a8b4e7b91090cd,38d2d46f4890f7cf,3a19c30deef84e15,3b0aaafff52c0e0f,3baa1dd2799a6a75,3e590f855c89ae72,3f96de1b18fdbe33,4084d954e0d5c5c0,40a1b23f9121321c,41cf645d8eeffc2c,41e650e26fb950d0,42250611e0d3e497,44ee6e415f6214da,4621dae25129038e,4690505c545a6472,472852748c73c40a,4863a9b45d8961bd,4ab6fae20b03f335,4db6ec771ae740e2,515f7c7dcc386c48,5400b54c0f958c76,546f8bbf44138e23,5536ce1c8cdfeb3b,5578d66d4b8a353b,5645aed7dc76b7c7,572b18cfb3acede5,589e3687e4f1b0d7,5a888650680aa032,5c48cab11c236dfa,5ce41f88b8eebf93,5d0b13a6c5ed4e13,5d95c79903a27910,612e0eb65c1e8bf9,61d42d60068c5756 +P-4cd71741f944 01980fac4b78be01,02fc555ff0ad31dc,031bb90f7158ed9c,0434d78219626721,043c17833444edf5,04a0d4e4660bcd60,04ac2e3f062a0071,058e099058d99433,060382af472256b3,06535bd65f4dd4f2,08fee363b4f95e91,0b17d7ea7bf62e86,0c2ed5d7c408d602,0c89174c1f13d87e,0cd395258832c34a,0cdac836bbe7b775,0d0df1d9b1bbfc54,0d0ed0c3d2594d72,0da7cc7734fc0a86,0e710aad3bd146ff,0f3c3aeab1415868,0f3ea8a6dbb318a6,0f5236c6a61df896,0fabd72598cbfbbd,129fb77bab30c232,12bdb354aab96ee6,139534ec8210626f,13daccfce055fb28,13f1655517b2eaf7,14ed44dc136b4f15,1672d51685f0e713,168c8bb95acf0a9a,1837e8fdb40265e1,18a422fafd02e5d8,18a8a59654eebc76,18cdfbaa1158556d,19b813961bc5b673,1b9e5d8ec0617887,1bd758397d56919a,1bda86eced303c02,1cdcf985b5463c10,1fcda79125c8cfe5,203fe8f4e6fe5a68,225a1c5e826b50cf,227e3830b75ca214,22a00e5f9d6e8f11,2304500cc8e20bbf,23b3754ec39fee74,241bedd4dbb7a033,24aea8ad322234e9,24b12cfe09dd517c,24bac76f1d183a92,2509cf68be266003,25757fbd848e7584,29854a0a4cc56319,29a2ed6ce9533ebc,29f55f29efdd405e,2a6f3d92919f0c45,2b17d1d83152ffe1,2cd2ba0f73c3149b,2d058376c124d6b9,2fa01b37b0c89309,3042fd9382e5ac0d,30d3d99650a58094 +P-50edd5a3123c 00c37d6836e782f9,01beaee7e4ded128,01ff0ed31b82bfa3,027be78db6c16c5a,02d2840cfbf74e13,0330e3796708ea93,04a8c77a71be1ef0,0bc84d4b31ad35cd,17084ec180f03a38,1b3863cffb374457,1fbefc05300bc4fd,228fff8b3d3de3a0,22a0829cbbee64c0,24bda3fa48a0e685,2555308784f255c5,2592aeb84b524aa3,2675b4d64d8a67ac,27dffac371a752f8,28272265f88df7aa,2c3a3ca78805df18,313121cf7c8c7979,3328a5d9d8b4bfd1,3669fc2b6d502428,36e80d3488648b06,36fd18569973e3c5,37922e18eb0d074e,3823159b39915189,3a904b71f445ac70,3b74b28b9fb3a5ea,3ecb64393f6d7bae,43476472ccb6f34f,453c496ba1f9d194,45f0de18b18a9235,4c330145def618fc,4dd3bb60e1824eab,51e1044d551fc2c8,5bc4b28e107ae66b,5cb91007725955b0,612295721cb8082a,62676c81154a3bb1,635b44b503beeb1b,63ba8742baf47929,65f0fcdebd1ced33,695b6d570d40e483,6a38f6d66b3b758a,6ffc7d2da91ef9a6,70d3e571d1ccf327,71656c90ded69eca,74a9adaa7257f37d,75e5acd3f0a85002,77858ddede6b73f9,798a0e4cd8bbee73,7efebd1eeba0bc00,80df0b3dd48cd14f,81e04775c4dfe077,8bfd1bf34b189666,8d0150e7f307ca44,8de320d33b1e64b7,8ec7a7f9178ef347,9170159ccd22f0bd,9268eb1df76d2456,9501d923adc391e7,98192e1f1762dc68,9a2bef9bbae90aa9 +P-553007c9399d 000225fdf97e68e3,0018eb871d51f099,0024bccf81086579,00996b9c9c81a36c,00ab8291b834a5c5,00ba9ffbbad550a9,00cbf097305b9d58,012a5051e19f36b3,0178e0b0f97809e3,01ba1c6b4ea5eaf4,01f456d85e38b7e6,02734c671ca0355d,02cdc379c829a168,0352929471566be4,0376f875828c48a7,0381a66b69019933,03c141af921d041a,03f2ad8835490d8f,047c6bf687798a59,04d6e0810b3d0b3e,04e012cd3c326b43,04ef190afefcbe4c,04fa7c1d2be22335,05670abc9251624e,058cdae4c3d2205c,059e5cc490506a78,059ef2497dca19f6,0607ebc0a2d7bc46,0653e223335510ac,0695199d2e2b3ea6,06dce231cbae26e5,071fcaf8b2e0fa1a,0722a4fc746f7058,07869a94e1801299,07f71cd5816edf7d,08834d0745bad713,088c95552bfef48f,0957bfaca8dad3a9,099a251faa2554ea,09aca2da6b83cc92,09d7b369b1c3851a,0a074d2ac6852d39,0a0d1aa8bb0f9684,0a3a9621918f2af7,0a87d3872625849b,0a9253c9faccded1,0b70bf16312c106c,0ba5a5abdbf1ec7d,0c6c18f22f061f00,0c94d9772d6e7c29,0c97b153ecf2bbe0,0ca3c9d0d39a9d4b,0ccbc86a180d988a,0d03f3c0640f10ee,0d9e8a779fe1f01c,0dd250b0aee6748c,0de09765da244367,0e3467332517be29,0e70c533be0ad4fb,0eea761c375a43be,0ef406bb3cc9ca8b,0ef8bef1e334c81b,0f5dc4169b52f061,0f82c555ecbcd58f +P-5a4122fffeae 00393025e408f812,029e51141e87c3d9,03c0b8c208ddfdee,04502870f231df21,05820c2c344b51d7,06039a70658d112c,0698165d6f4b0d22,074fd233a412a9b9,07d47ae16e6dfcc8,07e0617c2070303a,09fdf1c7b9170ae7,0b40a2ea5c06164e,0c50b8e3770f5e46,0d7f89c9a742b1e9,0e59472550434f69,0f4dccd9b07893ab,115430a278c336fb,1161119c00b0ece4,11a658ed24657b0b,136b59d6f61091c3,13c44873c7b6a354,13f198629e2b77de,15b5170220e24f5d,174f9b46af06ea6d,17c769a313f51820,18343bb6f0f936c7,18be858663df538f,1901c092c03027d2,193c2c1faf4742b9,196f669d5d4a3c00,1c67b1eca2472d44,1d8b2c6b66e1fc70,1e0fa8c9b4b037a7,1e6885a4c8e775db,1f92e9090932eb00,1fa1b0df37883d0c,211bb02872aaa5c5,2227e596f702913b,2480dd8c7c4b0e2e,2517cf862a37be2d,25b34a1701b06184,2767c6a73f91afa2,2be59df5475fb2bc,2c771753a2344826,2da31ab3551ce7dd,2e2a293f42c3767e,2e599fc515898fae,2f230e6fce20b7ae,310ac799b410b266,31c3d1cbd6beb5e8,31f40183302c3694,3247a1b1f7692e41,3261b2d0c2834cbe,341dede16ab28ae8,3551c1c0ff54c2d8,375123ac26e7e5f0,377d32675adf5df4,3b1718679bd5c5b5,3c15b059fca1bdd4,3c2b794eb10f2162,3d521d39008e17b5,3e32d6545fff6dbd,3edcdedf9b84762e,4084e3f75e19d88f +P-62b51b8e169b 0260830850ce2bad,055bfde2af378fe8,0742c588b37f8e1e,0dcc24bdb7b4fa51,10bdae29ddd11597,12f8b1c3a98e0d58,15e2d55cd0f5402e,1a7e28cbec3f5436,1e1c34548abc3b01,249d65af2d59747e,2555c241d198f1e4,261db2591c27e583,2c1cabad5ffeaf6e,2f5d1912a1443d0b,3165f61b492149ed,334dfa5f621c9838,3c1cfc36b265ace0,3d03a3da8adcf1f8,3dc14debccf79ecd,418a4dfa4b77bc45,42c53bf0e9cc220a,44debf5f0981b9bb,46734b2823f0b2a9,4b3f1e094d38c7b8,4ec0b8a2072d034d,4efb51c93de6551d,503a126ca1f52451,50e0ab3c3272a15a,54e72e24a136354c,55bc849085cd3f69,5611ef296d656af3,57b388b51e466f03,5d375f00570fa35f,5dac48231d19a76c,5f055eb90c46070f,61c79c0a33f72641,652a28ee4b7c5054,6556f678eb22a19a,65818f4c702aaa85,672acbd4ce376b93,67f3f4d5ace6a5fc,69b106a78d12565b,6a955ecb7ec061d3,6b1a3573f390eb58,6d6d88605c6c1540,6f30d9bed7764f64,6f84bcd4484c95f7,7072fc7100ee706e,70adf27ffaa5d358,778d28d50b674971,77fc3a2444cfcdb8,7bcb234124688f29,7c775fb6ad576170,7d443e133e438f07,7dfe426d77fc9e77,7f0c36d15d7ca65c,7fec2af503dc5f51,80c400269adcd380,84c81e30fb69156e,86612b8b10bb9f53,8f0be3237ef9ba27,90834c55360c9224,94ac24392b30ac94,955bbd91a54791f0 +P-69fb0db38747 00e7f2419e84d48a,010c47a4c5b38098,012bd1ff8154bf2e,021c75e85cb107bf,0350339ef0a764e3,03962064a814df81,04e26149ea49e61e,04ef61418c18068b,062c66af3bcb1277,063b3eb93ac79db0,079f373b917ac1a7,08b7c96cf861ca59,0e1812a710815d22,0e794b547a62523a,0f74c312326ed163,1022cf51e2560344,1151ed4c096e0e94,174ebe8c481871a0,1798ad779ce7a916,17be5feff72f73f0,19782bbbbcb897bd,1a060d005946aaa2,1a4d0e14dfb27e47,1b2a029788ccf60e,1d65959a334824b1,1f0860d34cd3bcb2,1fb5b1af6cb8022d,211bbd754d004d6a,222ec293ff255cd0,2246f01fd6cc556c,226753f21ae023b6,231908936b1992ad,23fa33015f3b1603,28d50e330e4b889c,2977cf81023570bc,2a6c92d1270689f6,2f6a742198056259,30b94b4fa3251aae,3263884d97472c9d,3452f5a9817b13ef,36c9011afd264f37,36d9ec7b13bd2ede,3cb5e7616df06828,3de9e17cc4e5ad2b,3e0482eb277f7b9f,41082459cf109aa5,423b0d288e57287d,427780aa9eaf686e,46b7c95aa245b9c1,480ccec5ddbcbce6,4a8d45a0d1874e5f,4c31ab9f013fe994,4f0e331038b28fa5,4f15d72292bba58a,500a7fec91a8857a,50904dfd0d4ecffd,50bd776a5da4accc,515300f7244b26b9,55d3e9b378063875,592ad79aa4942f53,59e096d2b2a6bd11,5c2ce1d22efaf44d,618aa4715f3eef9b,6217a3d5eb9d2ded +P-6ec735d37bbd 00cd29898af73e46,0272823c4e198a5e,031e8c88eb2c2666,04b1dff575c530f3,051d7941a18243c1,05c93fee5773b165,05e15311507b2ae1,06501619948b33b5,06f67e508dd9b9c1,0ce176c8ec1c64a6,0d16d2c952cc28e5,0e4d86e074be0d56,0f9ac9a4ee8249dd,118009fcf1a5682c,1357214f6a91690a,141e0bafdeef8d77,17d1216e04d03dec,1dd0b73b32d363b6,1f42b8cd7eda6091,1f98ca85638ae9f8,1fa19963da20b3c3,211487e056236f24,216e327c4436f446,22fc73e29fcbec4a,23562cf8594f9511,248ac732da8198b8,2b175fa4ed5452ae,2cd706975a77eacb,2cedab52df59c393,2da7ae7c61f40ca9,314c5594b4397658,31d8be545eabb234,32fd7be15c5e8e3b,358a7c3acd0b6195,38055fb674ac9bea,3a0e6d7a275f2dbc,3a9e00c5a3a0578b,3bed1d595f14f001,3e67a0dcee55c19b,41ae5c2943a5fc7b,43e4838ce3469e91,471767e0d8684b91,477d4df609306c05,48f88dccb5446b66,492245b9abd3b004,49b580ce49627f3e,4c611ff07925a58e,4c91f52a4a5b4b2c,4d7a5c61598358b2,51aa3d9595573571,528306d183766181,536357c647258cec,53cfe8dd5fbefc92,543b84f5e83ba86f,55e747f2c1383e75,56054ee46c0e18ee,57e85ee004032cd6,5843a5e06a7ef2b9,586035db692e62ba,5a39bdeb38f952a5,5ebeb9d3378e0f6c,618e10de9d4d0521,6294319a65d07657,636a9ced4e804272 +P-7a1b8a7554c4 021e62cb5fe0c47d,050369444fbbd65d,07711c34af64e362,08f062136d739d6d,1546658fa9607641,18fe7a4914ea6ada,1ca79f4ffa9c23ae,1fc5c6f2fbb774a8,21f3c1208912aa58,225aaf2d57396bc1,2351eb3a10ec48ed,23c71dbed37e5471,27e76509e93e5dbe,28c37f49cc67b92a,29728341542725b9,29776e8c3350b2aa,2ad9748755212c62,2c5455da7c6e2d33,2ed5a98ec39c02da,318dbd4a6980d210,321fdd4b2f8d2ec0,322d11a03e85d7a8,3286fcf3f5ec8db3,3338bae653a58143,34ad3b4f9a0e3ba1,35075309e4340b0e,3acdd26c3ac89217,44180a1cbf03dcb3,4a156cbadac3144e,4b6aaa9176fd8361,52d0c37c5916c785,52f7d830ff56d751,535ea8148d8f7ae4,537e3d3f14071a0e,54ab7892474f16ac,574860bfba04857a,585bae9be79cfdb2,5c4e07f837db8954,5ebdaf19919035c3,5f112fb49c5c6cdd,5f2d163dffb5c656,5f7bf6e0150b3586,6096d0acd6fb23f8,624773ad9dd6121a,63c3963534839cf3,6c75d5fb6977ba40,6e723590de5507d2,6e7b535b239083df,6ff850267f24921e,7231d7a81df18fea,73cbb73480fc3c8e,75703bd5bd27c59a,7cdd625a022a2a1c,7f98d4af47e63069,815ba06f2b6bc770,83c10a1e219dc9d3,845bbf843e641d70,8488bf4901a33b5b,8b2895b166703d5e,8ccbedcb7a88fed2,8d723b0f05c60455,8d767f19a98fc6aa,93c553ddeb4f3e19,96c177e11bc2eff9 +P-7c3e4fac1fd7 012e8bdc6d083717,02261312a27c7c10,03bfae335f2fe8cd,07cf1486e9cd2c93,07d1bca53d2f392a,0802f30132e93296,088da736fffcb163,0e6e15ecc2620390,1219e3ee0841af88,136c1c0d0529b944,13cf87def8cef580,156f7f043f99e97c,18d814afeb221454,1a79c366963ec6d5,1b07797fd9f6e4b3,1f0eb053deaee2e2,1ff9dba39dbfe46f,21858d963d057aca,24d29591a43291e3,27cdf3a3ed58a628,2d44e9dcaa203a60,34a4c1f65b603dd2,35453b27f28f61f9,37f1208c82ce8a7e,3a6c957d461db22e,3fa79dad1e94feb4,43460d583770e027,44da6b7b0ab8f6be,45bd894375c08223,461b425b5d638f09,462c55043c5afe32,47a99ff8bb259f2c,48da748fb6fa17d2,4ad6e1226dd0866a,4d452e1e113bacc8,532db5476200cada,55099abec2019d28,584f3f1d0131bc25,59d2c32020605657,5b1bb96384ab5188,5b8c3ed43e7868aa,5c6b36dd9d3809e0,5e4e1cd1f1834430,6711dd48a73660f5,67bdb5513f606fb4,681f079826df8625,6844f877ad7b2010,6891031fd9c77cae,6b4b64b88c6aa6df,6c404dfb1def3a77,6d7047ff072d6dab,6dce8cd78744b681,6ecc7e9a626fc38e,6f1839ba5317b312,6f74e25224246866,71a436045315f389,7524bf97f27459c3,75b2e8290142e55d,78808f897fc870a1,7b089d1f3c231eb9,7bd4d2234d591582,8097e1bc30882e8c,8178920f1bcd547b,8b737180bb2cd198 +P-7fffab1fddb8 005eb817826d29ca,07585b0ec0f0bdee,090f6d9f4cd5f894,0d9846991fc14892,13a7ea0ab281cb5f,155b1e7d2297e8c5,186343deb5a9c01f,2255ed97d7bed20c,290ed2b3cba840cd,2c2acb90eed8bd44,2d831e6d4cdd4466,2d9a88c6e8076aee,2fd4855602c42e08,32ac31c32d333ee9,382b7fcc03b61a7b,3b3066c9d84660a3,3c995c5f10523d4e,446820ba77879fe0,45efe01169afb637,529b0d15dfeffe2e,5338c2ee86617f1e,541e8c439d78ab41,54e72e24a136354c,5bec460e807c83dc,5e51f72a59676f04,5e706ad50175be78,62d0d88816e99303,678b2fe35afded14,70fbf3c3faa9c68d,76e501440ce24cf1,78c4f46dd89a2eea,798be064af9d6820,7f15db17acc06b97,801a9643b76953a9,83eef87cc7afa852,90aa60271a40740b,9563b92059931bad,960adc0ed6b05f3f,988bd2dcb06077ca,99b78e0f02e323b3,a16285cd17f10ef9,a5f0add267fb66c6,a91cfd9ebbd20149,aa3b82b1e9c7882b,aba3dad8e1dbf020,ae2d55f9bcdce5f8,aef9b03d8526c405,b198c5078a10797f,b83b6e54231bd9f5,bbd8ba40e5dbdb83,bc7e169b02412f5e,bcc484158bedd03a,bceb978cd43ea9f7,c0c0fdb3219a6f90,c14c7b9d7265bab6,c1dedae55a4e8c73,c477fc5a343240e2,c765007dbcc6593e,c8672d3e22c9b6a5,cc6d4dcb40bf0aca,cf1761db15100476,d10c1cd00e9b07e9,d244807cbb65b623,d98afb95919ecb6b +P-8453b248d79b 007616f6b3704e92,0131527b1197c74c,02b3f3b64805d96c,0478ace2d852516b,06a56450add853c5,06dbcc83b9796138,0741407d7e06d6d1,07cf1486e9cd2c93,08bb91df3dbd23b8,09a1bd87d122858b,0a8630530f3b8bc7,0b7028e3e65551bc,0d6cd8fd9402fd25,0e854b19b2b9688f,0f0a025c226002b5,0fd22b9d41f9fb34,10a624bb992db08f,10e523486a979dae,117a485bc3858c8b,11fe2fb73c86013b,12b73169969adc23,16cd91d680b768e9,191fefc13a3330e7,1976266ba36057f8,19bd2ab2228dbbeb,1a2cc165b3182185,1a76911af1154bae,1aac1e76de58f7de,1ab479812e2e14b3,1b14f6d5fd0d9372,1d19ce83dd50c7e0,1d6dbe3e989e030f,1d6fa14fac60ec5f,1dbce66e9ccbdf32,204dae9c10502d06,20d62d69c5f8fbbd,20f4a99b920c52aa,21f05d874f7ab71d,22e05a5e0914a47b,255f90b0dece6d94,2652c86e40ba7bd2,2660e1e458556edc,269d6023b7f9a5c2,289576d7507277f0,28e299132581b00a,2972945a695160cf,2adb8aeaeebbc3cd,2bc938a484a2342e,2da32abedc15e2d4,2dc617199dea0da0,2e45ba6074522fae,2e6af31f15712cc9,2f9f3500ce067112,2fb043f6ccd9693c,2fc82715084f600c,300429e602dd1e0e,30698ed934a034cd,340151f46bbaa5b7,346d574f947f64a0,34b34286c434c833,3563ac7b44aba752,35e769990d1cfc00,364f14d54fbe34a5,37de37157c48d9ea +P-9e598f519091 0220b2a5b8c00819,030ed48f3aab0160,04c00264e686353a,0684e4446190f88a,06f84b6c871224dc,0718becfb34a16e9,096939e9991f4a3d,0ec929985a997515,0face6f9b33228ad,12ee6b360620c048,17c2defa33c97a71,1bc7315a9bf0b299,228415b57683255c,2570628956e90b78,2c1910ac7a65eda8,309e96cf19ed9769,334e8782226a1bbc,352d4a1b3ad159fd,3566cc8111f7fa3b,366c00913c8c5212,3738a85ef932d483,373a084b2ef38ac1,3a4b691dea9d30c2,3d905fe34db3366e,411490e8b9eade9d,4564133c7d07d2e2,4566ed58732c766c,4aaad163588c5a7f,4ca5bd19485d3f65,507fb54275d1bfdc,5161a3eed7673f40,5447b5c30af2587c,548b72655fa79495,54c86850bbad2195,5d6c3ad63f53682e,5f46456e8424b004,603da60a69a1d78d,63bdbcbc1f7f74b6,65071f811169b715,68e4a872c59fb80e,6fd80286ee6de431,709011b565f38442,7205b0e9f02f578c,7276ce6402948ae5,73c91087ad8d760d,77245a6a4cd95b88,77314fee7eb4e4ff,7e5aa433e65d5a7a,87596242f079900b,879c3eb7e0491bb8,88a3765de6edd75b,8bcf211b56e70425,8ca7253157bf8419,922d797e2e53f2a7,955500c0eb1f470f,9e0a077851637ef3,a273e81f7837ebfe,a5ef17f4f93e5c34,a73d8e59982c6bd5,af7b6f564290a614,b04e410e59d58a80,b09fef37f9714d3a,b1958599d71250c6,ba06fd3be90cdb49 +P-ab41daaacc1b 054900cd523ac69d,06071ced49aab0d6,08297316c9ee2146,0cc83112a0b6b1c4,0dbaee2f0fb0dc57,12651305d439a3f4,1b0257e8ec119a62,1bd2fe3374b25bbe,1cbd9473521c0a6f,1cd3eeb42dfd18fa,1ecfd702065184d9,2128ee25d48f6980,21d4dbaeae1f83d9,302344cd51dc771f,34ae9fc55e98a907,368ea79275de11e2,38849359e5bab1e2,3d55324c337c6fbf,4623c72937dbc563,464e3db95bd1f183,4676f29f5dbc8700,4cdcd10cee887845,4d802703a9e9c83a,4db0e32d937eaa6a,53326150c8837171,594c9438472b878c,5af96fb7e6f82b83,5cb0800554b44b87,633631b95db3c09b,64c798362f145b69,6e3cce89341b47c3,71cdaa7063129d68,7554a3327b16e293,78994064840e6756,7b242c44bd385257,7cc7466a3b7525ce,7ddcaac7d3aac910,7e94caa03dfe2ab9,7f124ca8fc8de5ad,83567d4c2b78e701,899a74e71bc1f333,8ba4ef2e11b7b88a,904563cb2f3a1888,93bd7e54255e631e,943b20673d2567db,9cd68df52246a49d,9d1ed5d9707a835e,9ef4a29809ade0c4,a45baa84b691d8c2,a4a1aef98b9befd7,a4bfb2ab959c3b55,a8c9bf50f3bcc4ed,abeb16067a641a6a,afc540279a986f0d,b02f3a5c1790f540,b5edcb37c7c0a962,b89ea93c59281455,b928e36de995ae2b,bc258ea2090c222c,bc8e0b3de508880b,bd570c8b28521fa2,c1537d26947366c0,c69bd64f890fb23a,c7df31a335f5bfd7 +P-b18dc9a4b9d9 02d6f4877987af7f,05c3f128a0e3d047,0977c5093e4a5313,0aba8390829d4289,0c6807aac37a9bbb,0c8787ba3dd06c7d,0fc814047808f45e,11cf0221ada5328c,12a68410d97f57de,1383ec41bbcc5dd9,13e49166472aebf0,19c127fadc34678a,1a9ecc43a69a4948,1e4679d2496ea0df,1f109dff7911d8d6,1fe261630125d953,2207fd195b9996ce,248ac732da8198b8,260523a56e17acb3,2864bc34a2021cfe,2aac40bc9dec2aff,2c6789a5b4582bbe,2ff0e1bb8d9f1c72,303dee51c7f3b15d,31b36d46af795fe9,320c168f32d7a7b4,325861c396992337,337d5180cccb8b72,351ca057b522dc09,3555a80cc349298b,3c2b926fa8010b74,3c455df163c76fd0,3e051b0696f13dcc,3e5d22e8322cfda3,41a0bbb302ce30b4,421228205433a46a,44abcbb40c033a89,4773673aa203c6eb,49056aec94e63b3d,492df5c855151787,4a8cb737a5e74d5b,5203680689d9a131,526a0f17b97b9b0a,535f7db3b9535fec,537c52019c9ab393,56c2e47e27e5a117,5903a230f4704140,598b131a01e0f6b2,59a491b2dfe6119d,5ba46f457b048ac7,5c0085d9a34d76e5,5c9cc193a8160509,5da682de5a089db5,5de537695b4c62de,5fa8dd4f85ae04a6,61bcfcafc4df369d,653183914b58ec06,65ca02904aceec1f,66b2d30a2be55139,67661e7e3ca1d854,6acc42c12704806b,6b7b993ac64cd692,6f54f3b8f4920301,70f73a9b3530a7f8 +P-c3a0a45804a7 039955caa5380bdf,049c0dbc41752ed4,0b63901d6dbb4c16,1178d0778f4db08d,11ee5e1c89b3f09b,14a262311f0137b2,14bf8a3aa0438a68,14f67dbdea8e229b,1acf97c44f3e0b8e,22d09842528f3330,28954bd82531d2c9,2b452d6c1f2a4d82,2c4cd7e85e6ea4b1,2ff0e1bb8d9f1c72,32a4b6c0eb1b85e5,36510dbdde5cbf2f,36665aacc353f309,376921cd670ac2cd,398fb44e24ca6e2d,3a52e4059770fd80,3c7effa775264437,47c32ce51e9fbd07,48cf390d991c0d9d,4b23e6c46ee04544,4c5a169bd08835bd,503e812817b061a6,565b19fe76a00032,598b131a01e0f6b2,5be0a8a51ea9d76a,5db55555a2137a65,5f9b56e308909ef5,601c1f097def76d6,6120d11c446400ab,6a8f16a3d66a1dbc,6ba9471b67d5014c,6ce5c9503ca056e3,70b5d5ea96b7b18f,7124a1efdc9bb4bc,72641e670242acdf,73aaf8d492e6da6c,76cf1400545ef7c3,784e2401798a29c3,8627a0ff742ec38e,883d4fa556af8e92,8b688466154f5010,8be08c27e5179278,8e43b2eec73bd852,8fea59ad83008e7f,930574e7a93c7879,982adc1138fde00b,9a803bbd6cfcebe8,9d6626fe7de69db0,a6893a3f663a5790,ab6ddd316d668351,ad41d8f6b51bca3c,b9ad2f9355a0e7dc,bb5c952da957ec54,bdb7808c87752231,bdcb204c3634647c,c296a4b771a54aad,c5b97b850b1a9133,c8267c1b355a65ba,c9ecc44d2d4b803c,cb075c433c917178 +P-cc6f48e56473 00d44257951cdf23,03a15fc0635f6c13,0626a98bf0acc080,0863010cb501a874,09f159356396255a,0ec2bb35298bdb31,0f93bcb0a18d1c72,128b989871faffa8,149c0382f9daf24e,16e74244432b52bc,1882ec5acf8c2885,19ace5b3a43b3a5e,19eefa49f8af80d2,1d0c95c99853f501,1df7c61eb9d9fd49,223a920f9bba6ec6,22e37c108372624e,251f9d21956cca66,272ea2c61c99d394,27deff58d553e87b,2851a6c900bf15c1,29c67371f046e8e4,2c826961509d11d8,2c8c38e738e3fe1e,2e05b77e046e0409,34860521b11fb6bb,34998a8a0c8a0745,34ce25cfc73db1a7,350f699cd96f9f84,380eb61e669fd7f5,38109dc0f8cbc65f,39024779a2b16d43,3a9a9dd15e7d2cca,3c3c396fed2d97d6,3ca0f2c5d4e8bf0c,3ea6bb77c91d0fa0,3f000d29b1090427,3f696849593ad4f9,401784b0ad0fe43b,41ce33fdac763d8e,4225326ec0424685,423aa82a841eb72e,443c0a5366c9c7e3,44fbb44b81665f2c,454d4c57ef909c3b,459a52665d47f0a7,46582a08782d2c19,483d5e541f8a2be7,48a03ed74e3265c6,4e4c178e1ff68651,4f654d962a7362f0,500db2f8f4365bbd,5561ae369b1b6722,560b9cc186d16db2,56204a6b4a0611c4,56d16925db171fc9,5a600e993c6ff091,5a84954d5597c13a,5b0136a6a8e9c50c,5babab775ceacd2d,5d866d3484c08c54,5f502680ae929ddc,645ca91e7d42cd61,68132c015a8052b0 +P-d624d8619161 03ce2d428a12b89e,0455d60a82d905c1,06718120676fda8e,0742c588b37f8e1e,0970c6ae257fc078,0be6886c19a8279e,117307ff6fae0f0f,11f5fc85825fee40,12eb7b695f47cd5c,14311862e3cb53e8,155b1e7d2297e8c5,162bc4c6a3d502f7,2255ed97d7bed20c,255e1c6f443ff7a2,2bf83c3ccb0d49aa,2c2acb90eed8bd44,2d9a88c6e8076aee,32ac31c32d333ee9,3c0aa68d471fc6f9,3c995c5f10523d4e,3f35c4c25777307b,4982961b32a72205,4b23e6c46ee04544,4b3f1e094d38c7b8,5022c4bdda2f1429,5039f09ac8439c5c,50775c572b666f26,536ee112b669c3fe,54a5c06c2ccfdd87,54e72e24a136354c,562af5fd48ec8fe0,5bec460e807c83dc,5f4ca8811dc26c9c,5fadf188776ac994,5ff1a4d68dc9666d,63a35a8d0dec2d93,69f2bc10ec0f7044,73e75104abb8b8e1,74db35a4bff36ce9,76e501440ce24cf1,9563b92059931bad,987efb9d6b230dc5,98a5db8adeba337e,a5f0add267fb66c6,a91cfd9ebbd20149,aa3b82b1e9c7882b,aa69ffb5047baeff,ae0b120582503473,afde55338660f6a1,b5f2c5b7d52aea26,b6533b0569149c03,b83b6e54231bd9f5,bdcb204c3634647c,c1fb903350b105c3,c4cea9f41cf663c2,c56215c4a1b14f75,c7c41fd80c0a1c1f,c7e6e9f4cb3d98f6,c8672d3e22c9b6a5,d244807cbb65b623,d5184353bc99ed10,d71ce7c9d6ed2f49,d7c467d394ea6ddb,e6d228c7183c16cd +P-da4f5964dfb1 02db23b385874f56,04b1d15492795c6c,0742c588b37f8e1e,0b7fc581780e0ce5,14f43c01c982085c,14f67dbdea8e229b,18e702419051fa1b,19a593bdd056e5d0,19ea74894888ed14,1ac7a5fb5bea4da2,1acf97c44f3e0b8e,1bc7f400eedc128a,1db3fef5fbacd503,20df08009f5fd00c,227544c5c82eb083,2640c9e1fef70bed,26d92e10af1c9c3e,26fca831aaf3fffc,2dcde878be3a7e76,2ff0e1bb8d9f1c72,3234f16d6204318d,33a3ebf797a1ed04,390ac2fb88e37010,3922d53696bbc2c0,3ebd858d870ee71f,404be06fa06140fa,405fdc70f6ef9ad6,464f8c6d14f33e34,47c32ce51e9fbd07,4818c87512464fb2,489e50358a15952c,4943918331898aa8,4a8e3698e7fa3d26,4b23e6c46ee04544,4b3f1e094d38c7b8,50a163eed6399137,52325727ef96f573,54e72e24a136354c,56d377dda6a36eba,598b131a01e0f6b2,59d16a8b75fe93d9,5d1884ef3579372e,5f309e3013fc80be,612776c08d02136a,6557871ae0870635,6a416904a2f2afcf,6a52e6e19120b37b,6b834713b029e5db,71b3c102aef4d8f0,728ec789e0e49832,75ae1c5aa54c95d6,791102e88a5a6f69,7cd382e6f3c16eee,7cef93b29e8ae66e,7e5c437079a4b44d,7f81e103177865b3,81ed41d5e2ffc15e,82ac985868407080,83c5235bf23c997b,864d7ce626db3bd3,88ff89b85e91d752,89270651eae6d966,8cf0d6a5ca1a56c5,8d0671ab0d6f0876 +P-edb10eb7e5ed 029f8dae82b74443,03d8ad0fc2c1cfaa,0537b51e8091b2be,05c4f83789a9aa31,0bf9437b8fd6842e,0cd130c2fb1a1328,10b403685f5b764b,11ee7f0909695238,13336ed2fee6c589,17548cee6b3dc16b,1dc6f99b5a2c7e38,25a0cab4afa0e8ad,294e521a01adacad,2b7fad93b856bcd5,32e2cc9a6e66f0a6,331f68062027b4ef,33451e964c057820,35633cb23301252b,37a228b895386eff,37bd78c9fa521968,3afde2c21213c6f6,42f54e7db97f8cec,461849435ec6a6c9,47c19662070a9b97,54522b97005655b0,5511a920225ef3b3,557c95824dee5bee,579e14ba79226a85,5940a601c5f0d9b2,5c0275b5f82001ec,66ad03228b076dc7,6720526bf357160f,70008ca83e20590b,78385ce828d0430b,7bd3e4f384716895,7ec9fe55ff01b393,81cfcfceef78f771,8456768e6a5ae5ce,869f7dbf79e10574,8a5ef1222816f45a,8f219cce5b6f3a17,904a3e324c61dfe8,96bb44ccc95fadc2,96f556369c5726c8,9730253cd44b1c1e,994e686bdf666182,9b0e1d51ff4b5ccf,9bb38ad566c57d89,a3db01cb87f8a100,a4e748f1ade9568d,a5be354a78d9a433,a60bd1714db047d8,a9c53d06d227be02,a9d68b74234a0225,aa597127887bf8d0,ad7d3caf665fc6c8,ae9e4f5e0c3decfa,b0d3e4d9c1e4ab12,b2a985047158fc1e,b35cf00434367b8e,b654252e4ac2b60a,b8122adcf97c241a,c022ff086b8adaed,c3ba4c19deb61ddb +P-f3abd4438999 00bc3af7e3befc03,0148b55e73e147a3,06eb1f10e97579ad,094a163db2f24f3b,0b9120082568f958,0f24c2039f5be5c4,1115029aeb41643a,137064f55cd96443,1729888b6fd9f3e8,1738efb348605a70,18a236f3c30206d1,19fe50a0cbd4b75a,1cb8feca9511f44e,1ec2083290b5061b,232e19f1d38396fe,23629bbdec41d336,25ebeff2140684df,26ebee13176fc763,28ddbd94248f3a39,2916fa7b1ff79350,2ec9aece762ca819,322340a14a254124,3233a7a891e900b4,3481f09c44bf5fd2,3685ff163bc792b5,3883e592c088798e,392cb2b8ee16e14e,3a6936fd7a93b635,3e507add95ccd280,3f45b6ff6c3694dc,3ff85dacbe0f0d34,441bffaf742d1744,461f51802232470c,4a385ba5a9ea7305,4f0e146decc95e8d,5016d89bab4f666b,50272d02008c848c,572a27dbed854861,5a82cf7dc0170779,5deea4d9765ea2bc,5f0e3392b33df616,6116f72fa228fa10,61c4eaea7c16fade,6327dbc3887f6c35,646a683e776a543c,646d6ed0033d3234,67be53952ccac0ce,67c96baaa023fcc3,68764ce133a791ea,6d1e533be2f8e770,6ecd46006ee31ec4,74051a322b45355e,747f1d3c77fb883c,76acfc9d71f58bf7,7b0882adde4528e2,7cebbfe0b87f82d6,7efbece9608fe20a,7f35fc38819780ac,811cb5a3ee528dd5,86841aa12ddc96e2,88b4971598097fb5,916f699eb2fbde3c,953b0611d5ec67ab,97e65ad9e384c50c +P-f72984d91c0d 0001860d18401e2f,0038135aaacdb473,003cc64d201a42f8,00ee00d6f43c9099,025bc0a697fed8f1,029673c0e107df9b,02e2b5f201626f03,03081130ad9f82ae,030a7f3b6dc638b7,03843f034d5cfebe,03ad4b4466258820,03c92aead826751d,03f51637c9bf6a74,0423d77f58d7979a,0427e45347b86fe5,042996df428feea4,043b1d4033eb55e3,04a032d38c89f183,04c350faf1fa5a6f,04d08e0a929f5cb7,04e6ff7c88cfb8dc,0506effe8ebd6000,05145c10b0fd120f,054384296e88df24,0554ba984646ddbb,05603b0d4b0c9fda,05d0f3c8b814f57b,05dd45ffa999abf4,05e2fff15c34fde6,0607ccf2fcac27f9,060e19d5cd199f9e,06234af1685e73bf,0628368f88a5a686,06622a5383e3ed01,06c29a0d8db5337e,070f609b6548d822,076bc9c3699791f5,07a1edf25b93850b,07ad7b5e7087a170,07f89df5b31f676a,085573ca85a654b3,086cc5a718ee1888,08f298b9c37ca8b3,08f4e09fb5805b93,09048c84b60f8380,09166203ee465859,094b8b340961468e,096bdc0d09e33687,09e047fdf38f25d2,0a01aed3d9a0d72f,0a34978a829556c7,0a40d1665076ea43,0a5125c99217c470,0a53848930d4969f,0ad3902252134a6c,0b1000f365909033,0b1248e7491e7f71,0b19a1388a6818a2,0b1a65695c83d213,0b230bdfa3130191,0b95c3f927d6f7f1,0bc8074ea7e5303e,0ca11b208b4bc9d3,0d3271e0cfb340ef +R-pr1411 000db23b567240e6,00142ef715837e85,00324a7d39646939,005d9d5d7bbbc1ff,006efaf480c4794a,0078be7b4243b3ee,00827318c54b2cda,0098e83167b7777c,00997948eddeb9e1,00a3f6d2c32056eb,00bcb241e28778ae,011c01284b1aa3c9,01204d000bdb6bf2,0122bb50e2fbd521,0124321de6e8a1c2,015b9b000a39dfbd,017042731c36b6e7,018f14d6b5a40339,01ae3b1e7b4f8ef5,01b40f052534b878,01d0db2db40a6513,01e3b36afa02b2d2,01f3c2be20ed5fa1,02231a295c6d30be,0259b2dd5d4d2092,0265adf398bde1f1,026cd936c7d7b610,028c47a996d960fa,029b1dc13bd4185f,02a41ad582163726,02afb460b04c5593,02b0a501441bd0af,02c8acde50d9dbc8,02e80db3fa08dafe,030337bc74b41101,031a422a1a982c3d,0329e28224a561ba,0347b1d6de40c96f,0355139eccc615f2,0368b806d579fb41,03812c685895d133,0393505ce249d4c5,03e50c95eea7db38,03ead4fb976df9fa,03ebf6ce817944db,0412e6491b3b1a30,044bbd631c5e959f,0455e0c607a0de45,046620cef6f49f0f,046a0e39d53d8767,04936cc4b1bc7a90,0499f2967a32bac5,049ae8357367e84b,04d48d529153957d,04d4a28b8fdf16c7,04d8dde6e26bdc1f,04ee5620f85e9a90,05066df2fb9a83e7,0531cf53aa2bd067,0536fde2230e65ac,05452bfc1f7146fb,0549506e20313d98,0556414215952d21,056560b5a236f6c2 +R-pr1714 00f0d7cd3d36d00c,012cfed80b986c33,017240e6f81cb379,01c30897667039d3,020e1a5c88a4ed11,020ed2c347b0c6ba,023a6048612754a0,0388ac4daa3422d8,0392660dcc4a0444,039f3fdf4e5a6c2d,03b68d5b0b8accb3,03d7f60093ac16f4,04372671b0acff54,04a78b4b58db9956,04ba2483ba3c0e28,05bfc4dc0099e601,062737f52f697b98,06366fc160ca9ee6,06b2580b807055d9,07053c2321c06cf6,0711701b3a475229,07c1b8a54d655a2b,08297316c9ee2146,0864f7f37c44ff94,0881e4bfb1e034e7,08a0d26eb26540b5,0905f82758fd691d,091b34a194f975da,09877940146d2cf2,0993d021a56bbf40,0afcc7bc2efddb07,0b30fa84951b8775,0bedd5afae124417,0c4324fd936d0eae,0c462c8213672da9,0d1b62d5e60b941a,0d26cabffac5aa47,0d39a97d9cff6418,0d6fde0b49f5953c,0d8eb9141909a65e,0db591ccdff752be,0dd79670dfae45df,0e398d36960aa6d7,0e6b33a591d38076,0e7f05acd94b6016,0f9e4173bc9a108c,0ff37614110f7ea8,108df137d6e6c993,109512aeab8a86c9,10cc0eb52cca9adb,11235191655137a6,114bc0063a775576,116cd25fe6bedc87,120c2b83fa170d8f,12ede0b77f918b8c,132e1a073450cc93,134ffe6a6430882a,1367ed21fc6cdd6f,13e17e43733905ab,13fde03ca372a5f5,144230bcba30ca77,1460f6d1177d0ec1,1468efeb7ef9faef,14ab523c7b950ba4 +R-pr1782 000eab1208d8e4bd,0012e1e6e3208cef,001df239bebcfdad,00d44c9b3d6c75f6,00f4de554eddebc7,017dd38fe8db2367,0198ead498201acc,0220798665ee8785,023a6048612754a0,0266ff0b8f127c1e,027cdde38cc70ff3,028d9b500d0d7271,02bbf02546acbbd3,02c6a5da4839f2d0,02db1ea88a36f8e8,0360b58bebd078e7,03ecafc032eac3fc,040f1ccb7a2bc54f,047506c2b5baa934,047708599c58c8fe,04b712eb4138a31b,04ccbaca6617078f,0549c532c736059e,054cafaf8cd09298,0579d1482d293646,058a8fd9b3e9828f,05bac68259779996,05debca7c7775ddb,061bf66f90201862,06726f744510884a,06d363c654d9956a,06fbbfc5a4df755a,070119fc14639fa3,07aaefc6c5c83329,07cc694dc0bcd771,086ae5ca408d83c9,08d872dfc3be57e6,08e5d90088c3a90f,09877c2eff946267,09c44ffae0da684c,09ddd1c344754a0d,0a0a7f4d57d443fc,0a58d9d53b7aaea4,0a88f31f49a40ceb,0b39829d2edee103,0b4f270d24a517ef,0b5f09932e697bb6,0b76987d27a83627,0b7d5275fbb91c8d,0bb540c7e6ceae6a,0bedd5afae124417,0c2f7d2208287334,0cad15dcbfece303,0d53971bb7956b31,0d80d01b884b02b3,0d837b8c3201c969,0da022103e8747d3,0dc50daf522ab161,0dc5c641940fd49a,0dea1e4e18a2a8fe,0dedaa6bd0fa25be,0e12c76970be6311,0e6433578240323a,0e8c62be93639d58 +R-pr1806 0023eb1fad9de2eb,003d322577cb1fda,0051fbe2adef5e0c,005d342c3bf0e181,006bfbec16f4e759,00d80acce7efbe30,01015e7c2655a32b,017d97712e4fc07d,01c71e365dbdc6f0,01c7576e3982d74e,01d3c3422815d36f,022946db5ba9c799,0262572a80006cf0,028cf8f92d4a8068,029d94887afb72dd,02a3a6c3214de0aa,02e34e2dac6d13b4,034f7d1d01cb15ed,035c564f1fe29861,036dd8d6e3435128,0395b8302ba7413f,03a702e996f9e8ff,03a8a0cb2aee24d0,03b291bb138799a4,03cd88566e485ccb,03d237c1fc7eb61a,042468c308e966a5,04689273820a6668,0487e5a9081e4391,0488c03209605733,048e33cb7f227d0a,0492e742824bde46,049ebf4e7f3397b5,04b6ef3ede6e02fb,04e21f5ba20c3845,04f60ea8e494f3e8,05091bdd8488c164,054e33e7311c0714,0587bfbf0a51c9ae,05972843454e80fd,0597fc7d749a5710,05a5bb8816e51017,05a635aa85c57383,05d12b7d5d78a9ab,06298e2a043d3ca0,063030448d370a83,06366fc160ca9ee6,063fdbd189115926,0650f7203aa62dd8,0656ec6b7497c222,06765b6b91b9d623,067e3b3de8b5c9b0,068a51a61388cd4e,06b0d9ad0a72440d,06c3270a15270cb4,070f36dfb7b5a01c,073b6ae9a972a0ff,074bd9ca32a7ac5e,077857fd36e0f653,078dfe16313bb74f,0795f2c568b28b5b,0803abd4e6d69620,0815e67ad52660c8,0846ecec80eca79c +R-pr1808 014981f5e1e2aad6,016b27eafb638cdb,019ce3f6d1cb3721,028db1b0d5a77418,02cdc379c829a168,037843b7b5b90a01,03e56c2181a792f3,0426567bf964146a,0470b58f357177b6,04a8e946d0f635ad,057843716310898d,05c26f0799208233,05d877d166cb84ca,063a0807039e8cb4,0673538af7cf6dfa,068532b563ed505a,06a8aa4de39ec91b,074d3e79765a1129,07b03aee13b47cd0,07d0730e2ff2d335,09c686c8a571ffa4,0a2ea9bae4b3db63,0a82f2677c7e7133,0ad578c9dad988b7,0bf53e152647b40f,0d063bbf20526047,0dca1e8b2fe8add7,0e2ee967813c090f,0e530876c8667d1c,0e6b33a591d38076,0e844695b7b45a40,0ea92d829b9f4c1c,0eed2ed7736c00c3,1018e31c434b87b9,1027ca626ce020bc,1063539e46884712,107afed395504a35,1097f6404f44d348,11278420357c255f,112e46587b64fb8d,11e800408ee3dc33,130ab5f5f8c309ed,133853bd211db7ca,13819fd31de3ed56,15290aa969ba8ba2,15f48c93389749e0,17522f2e4dd22057,176f4f6ec3762cf7,17f737e4beb98402,18aac18579cdb03c,18c447287c091134,18c7d9fbd2f94bd2,19c310ea398ead4a,1a63852ffb3b693e,1a7e2d4096b83601,1b09cd39319ee06b,1b6abbba6c6e29cd,1b84c8fe3aaaed0a,1be5761698aa3f65,1bed9733528b0a20,1ca848ee0075481c,1cf4d8f7146d551c,1d5060335f2fb72e,1d763599f27d47be +R-pr1867 003c30bb613a26e9,012834525b9d2c75,016905f56832318e,01b80a8288cfd215,0202c893149915eb,02eea3580474ee41,034c7fac88b41a98,0397d7e6f85d2b56,03ac06fc2e037c14,03f8522bc241eb51,0438d82c52ce40d5,053ff12281c8cd80,05a6542cdffcf72b,05f9ba13e181a87c,06012114ba7c99d6,068afef7a141c2a6,070620ab25841cd4,070f0361e1ff396c,072ced5f791c6270,07b1f194c67b0d09,07ddace8d1023fbc,07fa5b2419484adf,082652a7cdcb7791,0883b6a69b1be7c2,08e662327e2db032,08ea0ef4243291a4,096728a15ee9daba,09b4508af0280d8e,09e3f31f790db022,0a007c109f848c21,0a2322c0119169b7,0a7407eca6782bf3,0a8eab89070af523,0ab7283f28d89ba6,0abeb747b6415ab3,0af464f501313709,0af50b8c906d1a3e,0b1e8f323c37373a,0b4f270d24a517ef,0c0ea893dba10591,0c4caafd17923c18,0c6b46f8ee728632,0cfefd376fd51e9c,0d13cd0e0dd8e615,0d43e6d7b2f287bd,0d93ad7b06df7c9a,0da15bcd48b9d6a7,0e423c76d407fd57,0e7477249049c997,0ecc5a964de19846,0ef980b265ddc576,0f3d818f72a09c5b,0f7ea7a146e891fe,1005f02b7be39687,1016d4b54d9c9a9e,10a10f0478da0eaa,10a5a1454ca04962,10b54e2e38500d73,10d276ee88fbc055,1121fbc3314aedd7,113239aa38ab70ad,119ac679d0776562,11a0554430182516,11b85c1f87bf9054 +R-pr1870 0052b37fe93613b3,0069fa76f86efa11,00cf881bc0462bb7,00f26cad022f71db,011ee4529c56fb4e,0135ac9fd194d255,014adc7266eed685,017f538d1da84067,018f8ae4d66a1c32,01b17b4e9967542d,021fcd78a1b96b71,0252c69d62d2f000,0262315d687adcad,02c7cd93c5f18baf,02e71b895c68bcdc,03149c91f18c1c7d,034330fb8682dad9,0364e9f341934c04,03a58016b99006f0,03c8809f4ebc0e68,03edefaee6f506a1,03ff3dfb94dc52a5,04144f3dd2814192,047f9758a8cc2044,04a08d6e7aaf3848,04bd604278d73575,05452bfc1f7146fb,054c3a1db89ce716,05c545e9a848459a,05d6aedabaa66e38,05d8995b84c201ed,060fbb08ac58b6e0,07088c6e343e6151,0769962f148ba915,076f44d20bfb8b4c,0789b52639d48217,07d65a4b5b33a144,07f846d32062e6ff,081395f05c751028,0827f3fa525574da,084308e18a0cfdcd,08554f400a34068d,08a3bab54ce977d8,08cd65c6351fc940,0935feb169158718,099b1671e085227e,09b98a759bceb011,09df477282146bd8,0a26d2dccc5617a9,0a28d0c6ef0a2f47,0a9a85e786a50d52,0ab6d2f85d123ce3,0b085eb462543de6,0b0f1b23632a8d57,0b4d6affe71404fc,0b5417a338206901,0b6b02324f707b72,0b711a41363060d2,0b74869930bf2f63,0b8719d7ab3a834d,0ba18c194551fff2,0bf53e152647b40f,0bfb3fe0e74155d4,0c24a52dcc198f37 +R-pr2092 00082d54883afbb3,002230bf46e47704,002353a84839d6c7,00386464bed2db16,00737712c96d00aa,00a5fa644eddc22a,00ca2534e8364be2,00f90dabdd4b822e,00fcc623f7b6957b,01a27805f0f5d74f,01be7178e1a24345,01c1befae503380c,01ee430f2bb4d452,02013ba50d4d540b,0203a0e15fe7b848,02061519d3b9d1a5,021529c039a7e345,024e286e08829c60,02b1b1cbdcea7bc9,02bacb1881b3af43,031a652155261149,031f2165de8b61ff,03594ac30ec64d4f,035adbc356c270b3,03ca42efa038e4ce,044f854b2dc8dd3f,0482efc8e4cd2a2c,049c00df74225709,04b07765f4924c1c,04c0793a5c67eede,04c437e8577b446d,051bff2f645b263b,053cb0b01a64c328,0542e69f7b17b85b,0553458824380438,0556ca89aa44cf8d,0559ade66825fa52,056c245f14b3021b,057a49ab2b21272b,05a50db36e4b0a51,05b57dd687e36961,05b9649b4549adee,05ba1511f274d37a,0630c56cb95b3931,06403f0e9d2bd232,0670f26c93f6091f,06cfbfc800bce37b,0739233c1dbed73a,074217780df69dee,07957f6935c5e4eb,07befd3cf2d131a9,07c496800997ad88,07de7ca535f1a8b5,081d1d2605c01ce1,085e10b2fa9b9f85,08618691e72b90fe,0866207f2df62a84,08c0a5513e9ea1cf,08cd73c57fff2244,08e9a0d0165f1869,08f55d04bb086867,0906659fd5d3da08,093202833cfafca4,093ddb5f7c5088e7 +R-pr2321 000a72385555714a,0058909e1c3e847d,0062227aeee88345,00bca3904dee4194,00ebf127c8aa2c6f,01115ab50070b542,0159e9867cede317,017b3d0704b2d823,01ab5c2bc89b13b9,02b4a06a4475f40b,02d3ae952dfe3cef,02fc877fc0ff788c,033c64e0462b6e12,03538268cfa8f78c,041e6ff068fa9e33,0465d6bacbd75537,046b4871d6c11fe1,0470b58f357177b6,04e73ad16086f489,05f35e5a71b3b69a,065e1f15095e9445,068d8a817767ea92,06ceed1f8f4a1efa,06f85ef4ba056441,08afad969a5775bd,08e006b2bca0d6ed,094cfcea801b8924,0975f78c3eb2f53e,09b536f4a8b2f292,09f1a134bb54349f,09f9fd794d901dbf,09fbdeabfa0538e9,0a4b57c03bb72c68,0a8979d5d387b404,0a9190a1f520fb51,0ad1c7271697dbdc,0afedb6d3abe1693,0b6eb55995b35d5d,0bb49643d17b7599,0beec10d87e62aa7,0bfa53d6fa6a0308,0c2f22530f4e3b74,0d6b68bfc3241f99,0d987323605156bf,0e60057963e52091,0eee9d83426b8c8f,0fc989f487b3c82c,0fc9fb8ef7b20b8b,106e8b7edb10a73d,1114e93ed4d3c6c8,113f01c4c72bd295,114d709eedc81145,1259ee8ca1f76d08,12f7f92604322098,13a9f1321b6f195b,13b0530b131c49d2,14566529df6b983b,14c5b2b511f8e592,1529f9a6fdceac9b,1539029655fd1a2d,1546edebc0b3cec8,15891fbb4159e5a8,1593fb4db0cb42e9,15a67d2d98491b87 +R-pr2365 003a2f2b20bc68fd,009990c4b465d908,00b3e1228fd9d5a8,0257e3e6b1385a18,02de1af095e023cc,02fb0cca11caf2b5,03323978c8de74b8,03549fbb4624786b,0359c968e9bea367,035bd7187ec03a74,03a4501cbebef24e,03cc25bd1ca51cf9,03cd6121efe2c0ae,041ab42b3b9a3bc7,042cd0418882693a,0432b44c73a5709a,043f2874bbbdb677,044586f47ad5d881,04521b6abd575cda,045d656ecd6eb92d,04c1ef17b2c84b91,05184d12da21b71c,0534b6fbec622ce1,055d96360d6baac8,05b3f0051b656162,063fb6ae59f2d6c4,064ee3aaf756c130,069861526f167389,06a15535ac5c9b9a,0710254cc1c07a63,079fff870cfa6497,07b5c6584eb66268,082d2b1390f173f2,08be2f73b4d607ca,08fbb9df22811837,091a3d68eb34cb8a,09490b0ee5069a40,096db8d6286a7553,097c545b5902c57c,099a8615a0d9f360,09bc6028e9cd9f3b,0a09a7da012af4fb,0a0f83f299856f39,0aa0d4bbb77d7c15,0af7868b578c304f,0b296b2ebe549354,0b5ac43ed415f384,0b660eaff63baa68,0b8bbf9700fc070f,0beed5b2ceda3cfb,0c5265bc329f63b7,0cad2320e681e2d4,0cc90dc43d62d83d,0d0c0c7174963e76,0d866edfc3e7af3f,0dcaa71447cadacc,0e071cc6b3e025f2,0eb48a055a4b59b6,0f1211857a7e4b74,0f22fda670a6af6a,0f3de889e1c34d9f,0fd259608c5c5cad,0fdde5639c43a115,102f05e9d181c2ac +R-pr2368 0026cf9694399318,00485ffc60aa622d,006a6aacae910f0c,008cee9d9004ba42,00a568114fc79a26,00b69102db69abc0,0121cf9865f7bda0,01682489dff6e7fc,0173ff7ce5b5eafa,019c299a24872a63,01dd7f1a8e3f9c62,02341e150f671695,028cf8f92d4a8068,02e0e7b6059e9b87,02f709be017dee35,031c20bde258c9f2,034313d50271d4cf,034de4bfea2ebcfd,0387c3f9fb6e04bd,03f30ac81b70cefa,0472662c2b6d7836,0483846f0b6c6796,053e7d4e0d3e2544,054e2754ba0be9fa,058e1bc33d181fd6,05a20624ef32614d,06476c3941b446de,06506a33393c6cd3,06580555b9b3b4f3,06894bf7d6b3eac6,06931ed4e89daf99,06a2f3cfee6b2eba,06c8a32db7eeb35a,06dce231cbae26e5,07037a33cf175710,0736e5de60e6d8e5,074834006e7cef10,07520cedc6c147b9,077a02528f96fae6,077de05d6ec6c92e,078cc6846a2ae242,078fa382cbb124c3,07bc6c6a9703afed,07d371c28af36446,0801692dec88ed5b,080184008f373e0d,08257b63c0ef4aea,085661564bd5e7de,086aed6e0d21dc9c,08a4bc5505149105,08c6a4101b128e35,08defcd8df138123,08e9a0d0165f1869,0928f40b9644dc50,09504d6ca4de146d,096a4adb215d341f,0980da7628ff8ffc,09ffea155daebef8,0a40ad4cd86e6da7,0a520cc619759e71,0a9b9e134c255855,0ae4b4c586f5f63e,0ae70e4e1ca532d3,0aeb7216ee4a2cad +R-pr2415 0011b17d2f3e1475,001e2dd4bba4643c,002a18e8cfc19a2e,002d379a96fc8d61,0031f041b11d641c,0042e14020488c84,0043f8b2c508abe7,0077ccb9de6b5b5b,00a81d57ce9b63a6,00bc9b5ffe486a95,00be53d69e3177b1,00cdc9fb596b01b6,00d788f38e1234e5,01002177601d7afc,010bf6cddc4b1c6f,011a123eabc00b73,0121cf9865f7bda0,01354f49c2b4ff9b,013e7693ba15fddd,013ec33cda811a81,01415be737b33365,014bf25828552899,01b5ceb50af83bed,01f2fe06065d7fb8,01f8aafb6a6322e3,0203ed2b56ef984a,0207df85c81f0303,020a8eceb83a9310,021529c039a7e345,022e820df73e3869,024007b51de227a9,0246eace4e4cd098,02477421db56692d,0263af243f6b113e,0290f807240e13d8,02948f4521859bc9,0298cb33133309bb,02a3299adadd9d7a,02a84a7c40e02c8e,02ac9ce648fdef9a,02be02ab3535ff34,02c92d7bb8f41290,02cb5c5ae71a6c0a,02d6990fd694723c,02d8a2c693e2b4ae,02ec4ca0ca9d6401,02f5e8a11bfc844d,030a1ea27c0c3fd4,034005fb40e356fc,0343795e53af105b,03518b2c362ea3fe,0360d13f104a1d0a,0384dcf9b8807913,0393b5993d720367,039845cd561fd850,03a26f631109082e,03aec6b5687f7d2a,03bdca125bb6bb99,03dbfa2295b02501,03e50c95eea7db38,03e7c166d857f81e,03eeb52f14d37917,03f2ad8835490d8f,03f3a9c566ef17d9 +R-pr2416 000225fdf97e68e3,000a72385555714a,0018e7e8b46f516d,00247dc312188f23,002666fd3da61777,00276524afca7f5d,0027661dd7e657cc,0028813ef5c2ff17,002e94808c0c14d7,003657da38ea4d2a,00634f7dd2390c39,006486f33ad6bf62,006a6aacae910f0c,006afbe41611c1d9,008ba62859e3d507,008e14a797224c7e,009218be48141ff4,009a708c98372d2c,00a568114fc79a26,00a6979711f604d4,00d13bb6a5ea037d,00dd2c758255b487,00f0e6ab440dec5e,0118d598eeed4a08,011a123eabc00b73,0131f316b6032326,01426c921d06e922,014d044fea050691,015038568c8dc380,0157d971a83db2fd,0158a16bce01b20f,017c2e009d3ca919,018c2eb012a33e04,018e8959fb9a276c,0192e0c6b08ef707,01957e7759289c15,01bfcc318991ba13,01f4594c76f21afd,01f84d46d8af33e6,021958691835f9c3,021bbb14ee3b4046,021ca01b7aa302da,022b63f850987b93,023c30b0da2579d6,024007b51de227a9,0248676c2251f182,0248c41f443c3ee9,025381dcacc76422,0258d4a3cf3e17cf,025d282a04d78c58,02857d44b3d00357,028cf8f92d4a8068,028d1971cd52ed3d,02998b50fcc4d9ba,02a2b95c422d098a,02a748efdf5c1c58,02b0ff3aa8f2db05,02b3133dd4a26c27,02b7eb923ff7e661,02b8fb77809eae0d,02cdc379c829a168,02d35621ef6c8799,02e89053fd5351c5,02f4315143b40774 +R-pr2420 00010d430eb15502,0003829b0cdb2bd3,002d8428655b8aa6,003b3fc011f2c640,003c25d849ddd55c,004e99c832a3207b,00555e82c752ec1a,006b55259e9b2b3a,0070c6b2a3ad39f7,00736570e017174d,008f1a004c9ebd76,009730c4912fb837,00bf29128020cd20,00c5ad6bf9ae66ef,00d0fd104ac6152d,00d9db6f034b2cf6,00e29a3938208baa,00e9c954a8536074,00f6cdf2e667ab71,00f73fb7a0181e99,00f7f84a84eeaab7,0100e007463b8116,0110ed9bbd825b43,0120a8321db31d4e,0125fac73b0963d9,013930a19cba1bd0,013dce974f4ea2e5,0147f89897f49d4e,014dfbad5a0cb553,016caeecd0bb5e74,01741fa0e9875897,01759f7782bb87bc,017b1589ee119e53,018c3d7a691b11c6,019a7bc02b796ea5,01afd20bbed64947,01b5c51bc9426598,01e440713c8609d1,01e88de998c80350,01eaf8250fa930c7,01fa229283896381,01fd8302fb9f8063,020482526f7f6cc6,02070f9931023b67,02083f59e366ba2f,02171380f2545e9b,021b5f8d9db7e846,022a398eacb6a831,022df2024e6833e7,0230360cd9c341a8,023e27952e442241,023ff4822a4cf13f,024759767b3708fb,02620f2efec1eb73,0263d7a8397b3a5e,0290a3a884a92c81,02a25d45f3aa6906,02acc7e177fbabd0,02b12a118b1bc223,02b41a248b69d848,02c2c262a14b6f85,02d2840cfbf74e13,02db1ea88a36f8e8,02db3954d8ef6f61 +R-pr2423 000376060e53e37f,000428eeab98df09,000b8ca2d59d9090,000fadfcb8cd2cc1,00196bd7f7d96cc4,001e9c4b59501c17,001f0306114f17a2,002326560fdcc3c6,0025731c94fb1277,00262b2ad5af0951,00315bad2d55cdad,0049a3fd386a8d2b,00669e4e16f8f712,006e65c93a21caa2,006f5800ab5f80f7,0074b5858cfa61b4,008cee9d9004ba42,009c53fcd8689f36,009fd48eaa9a83cc,00a10e43752fdda5,00a4d7a7e958365f,00aed8e5e067fda6,00b5cf6363c85270,00bcb241e28778ae,00c7c69912e9e346,00cb7a09b3abe8a5,00d2be101b08c4e4,00df9ec02ba323fd,00dfcb73ca766a54,00e0098e3f2fe0d3,00e04027adba74f4,00e27e519d968dee,00ea70af15a1ae94,00fc8f85d642b31b,0104b7bab6fbc513,010b571384a150b2,0110727f696382d6,011849209124deac,011a123eabc00b73,011a136578cc4f3d,011e47f4aa116566,0121cf9865f7bda0,01251dc3f0f9d315,0127eeff18006e7d,012fe11b89c1af5c,0131240d476d9e3b,014884802a5f187b,0150abe6069897de,015d2b3027131799,016185a811786722,016d27b055116382,01706da425bbfb1a,0176adcdc44128ee,017e5be76f54675d,018ea6e7d3662447,019c1eba3667ddb8,01b3af2648ce8aab,01b3d77e55513fd4,01cb56dfb7ea609b,01d1fb367b472cc9,01d897513c1d67d5,01db39d87ab0440c,01df0041bd3db1b0,01df230aef7496cf +R-pr2424 000aa1761bcafe07,000da4d68896f219,00175093eec276e2,0018fe5b79049aa4,001d57b19e9c750a,001f49da07fd2286,0026726d270f1c92,002e707652b7ef9a,0031af6b5f291e00,004429af72687d77,005c073c46919741,0076d027443431a0,007bed9fddce790a,009471d2b4945d56,00990b5e9c1dd5ee,00a2aa4a08d5e4bb,00a81d57ce9b63a6,00b5b00c9155e509,00c3df6d8f91cf01,00cc8af616584e30,00cdc9fb596b01b6,00e99b48a887b9a5,00eaa25bfe83cd05,00ecd62897596dba,00ef6bb8b608c4ee,00ff51f1082fb14f,011a123eabc00b73,0121cf9865f7bda0,01303be754da34d5,01338fd39d354f3a,013c148dcf45a569,013ec33cda811a81,014d2c115e6ad163,01522b5aa8f5834d,015728fe3cf2885d,0174fa4ac6022e67,01770d3e8465d585,017c1464121046f9,017d56b0d2680052,01bb5feaef9f4f74,01cb5ec6bb46d722,01e440713c8609d1,01e61fb27b0dccb3,01eb76edf2679701,01ec1c395f4b964f,01eeba2553e5539a,01f3f907de046dc6,02118de52c994dbc,0213f5db93e0edaa,02193dbd5d8ad39b,021d35e7d5c9d214,021dfc5c34ad68e6,02251a89997fdc44,0236d45084b1afb5,0246eace4e4cd098,02503f237231d16d,02647c3785964652,026861225139a34f,027be78db6c16c5a,02a01eca17280163,02acfd707eca0271,02b0ad6640e04b39,02b21d3f4df7ac38,02b5a5bfd728477e +R-pr2617 00050980e9ae0993,0021ddc13d9de0f4,0023a4f07741fb1d,0030d7707c81c191,00402307ef22557a,005fdd97f7bff0fe,006eca08cacb855a,007d9247d273bbbb,0080151320ae63e9,00806ca19ee16e21,0082fd0c0949ea19,008cee9d9004ba42,0094a81ff1ce25de,00983e8986c8c242,009ad01bc282c858,009cc7cde1fa9ee7,009fc8741b3d28c5,00cc8af616584e30,00f89e17b54c8539,00fc7a10f65dc48e,00fcc0f0d8ebe408,00fd519ae79b62da,0106d8d56a91bbe0,0108a061e59af50d,010fbbea125d5e81,0122fb74a920a862,0124d4d490ea66cc,012702cc28bd0121,0133dcff25bd8d6d,0134a2ddc9b8885e,0139974bbb180109,013b0dd2d27f4679,013f398390de3061,01432e07f8caba5c,014e2e00cf6af437,015359fed064dfa1,015b28cf9ddaaff6,015cdd2d2f976b92,0172cc181f610221,0173989ed52325d7,0180a9d484310e90,018a47b1926ffb3a,018b7e5933899760,01a8540f380fb8e4,01b19eae8abeb0c6,01b77a224df8cc64,01bc5ac160ae7610,01c71e365dbdc6f0,01caf0e85636a5b0,01ceb1fa35fb2876,01d6af920d5fc4fc,01e1e33945d76949,01eb51f66f1f19fb,01f9492599161069,0209cb59ce275b98,021e8912fad16c2c,022d861caaa2d2ac,02329f11a45318c5,02350d9b222b602a,0244ae3db5cd6224,0246eace4e4cd098,025174a70626f816,027d2ce6afd4b13f,0281a61ca6ac8cea +R-pr2707 001d608fa96299da,001df5c2d9cbf476,0044af679c0e9adc,0046623124216c9c,00762798caef55e5,008322d6e0da4580,0087c68cff298807,00ac2df35955a0c2,00b2f8fec0e05e0e,0107be2726deb662,011a123eabc00b73,012a96e2a3258f40,012b7bcbf7b86276,013862e0a434bc88,014df8062336e9ed,015021fda9f66fe5,0157021fd769e3f1,015825aa1fa2f07e,0159e9867cede317,016185a811786722,01657b2a6a24f167,0166c2c1d3d947d6,0169ccf5c5f1b188,018c175084d82554,018d0ec9a768ab48,018f8ae4d66a1c32,0193c9e81c47dc81,01d03a4cf5bc9947,01d1a26e5fa5e77b,01eaf32b11b5ebee,02011563de07f524,02020f5ef1ecac8d,020a69bca52cd368,020c519a02074ab5,021509ad964ddf99,0217d67a6b20062a,021cdc1fa8a5b6b0,022540674671d483,024715fbeedfad2c,0247e55f430e1c8a,0254566db50cc1ee,02618ffa31d157a8,027970e78c23fbfb,0290a3a884a92c81,029f8dae82b74443,02aa58e2ad5df0e0,02cdc379c829a168,02d2d860f8e5dc47,02d74e050df69d7d,02e017a8870594e0,02e622c8daf90d1c,02e7ff4290a628cd,02e9b53d6ce9a3d2,02fd83cac0b7f57b,0315a4d5ec9d2c26,03187bb6585bd63a,03293f93c91bc58e,032a1239a1bf6c33,032df00de5f686f8,03374d4c440f7466,0344445f5c1bc6b9,035a3ba1918aa688,0380d6b4cef9bd67,0386094f314db25a +R-pr2809 000c3cff3019f92b,001deea66d81828a,00312a6e63841306,0042e14020488c84,0046623124216c9c,006f80510c67d863,00947101448d439b,009e2ab89b70525f,00c389cd4c797a16,00c8a76df06f2387,00cc8af616584e30,00d1a945d39b7962,00e543e3f8906169,00eb3707d727c487,010cc11fcf72cb9e,0111e5800393b860,0113992a176d369c,01182fed5269ca5b,011941de18debf22,011a123eabc00b73,0134128dbbee17a1,014925197461abed,015977f67291e711,015d16d289c4230b,016185a811786722,01676c24940c55f9,017241ce7860dcb4,01774e4b1541bfe8,019ddafced4e60d9,01a4d80c2edb65c3,01abbdcb1344bca2,01b594d1632b1721,01c3176d5ed47f05,01c805da3cd343fa,01cebbc8a3843af1,01cf0b660e95095f,01d94534daf8846e,01e1fbeeb557ba96,01e34fe514e1678b,01e38a17c25153a5,01e968b582f12cf7,01ea653fe7657a11,01f38306a268900f,01fa0f3685360053,0203917f6129639b,0208b38aab006b30,02099324349cf0e6,02194f74b0d1ba9e,02294d7d9b2190ac,02373ff42e1372f3,0240b2852c655981,024715fbeedfad2c,025ab0bdb83b8187,026889ad8d590b28,026d7004670b0de8,0276c005afd7a7f9,028119a4db0c9bbf,0288810bd310f552,0290d1623f37d4c1,02944ce89e95aed0,0295ae5258a7cf60,02afe07cfc0bdf9c,02b794423bb64466,02bc91ee9aeb9972 +R-pr3008 000ea424553a1029,00464b09d1e76b93,0081b3d496b517ee,008aae4fc7e9c415,008d385edfa82aef,008ef6598ca58ce6,00ae5135474dc7a1,00d8871bfadb1376,00e8984de73a4a7c,00f8d7792139dac5,011b047470d13fb4,01461a529723e13b,0172e89ab0968e0d,01c2a489b9682555,01ecf7d903630b1b,01f8c4baa509ce0e,01fffc04faf1440c,0264cac0953b8ffa,02680c2a3661b06b,0295103ec6bf494a,029fe5a72c4a2be7,02d6b9db9f1cdf8f,02db1ea88a36f8e8,02f32e05afb98fb4,02f7d548100f3051,03187bb6585bd63a,034f7d1d01cb15ed,037345cb4f21ce06,03a7bb99afef476f,0464dac639b4c4c8,0487e5a9081e4391,05066979137f7034,05202e9ae43528a4,0530f751165909db,054b68a15b654713,05ad8b52a9d40116,05d16d95e0365085,05de81b31616869d,05f0945a98c932e0,060e3588ec68cc6d,060e886dd0d357a3,061d5daefbde49c7,06396a76392e1e4d,06b44c7e5033e476,06bb382ce508216a,06c0c8dcb6cc472c,071fcaf8b2e0fa1a,074adb7bb82598a9,074e80a7b20a9b2f,0766dc625f3034d5,0786db7d54eaa4f5,0794b2bae776ef70,07c1b8a54d655a2b,081f4596bdc9c826,0826d62bc62d4c51,082b22f1774d0991,085fc19587bd8048,08624567741407fc,08a28185e33dd2f9,08d6c361ec4c6438,08e6136fe9ad2e7c,08f82c2b6a837c3b,09156f280aff0c45,093eb396e741dce3 +R-pr3030 003a2f2b20bc68fd,00762cfd5bf5b9b9,008e1b0b6895c3c4,0093efca36ac8dca,009b404381ae896b,00a72fc4b3a011b9,00bfffdec6bce9f6,00c6de07cf8b6173,00ca22013263cc0e,00d0bc3d28a48819,00d0db831c302fc0,00ebf03fc5fc2238,00ed0fef1078d34c,00ef762217915c23,010f265b4fd00993,011a264c09b3ab80,011d3e660dcf8626,011da93886f76fbe,01276a15be6a93f8,0146edc4b77d9972,017aa4601be3a9fd,017b4e29bbc3f8cd,0202c620605c5fc0,020bf87ee4eb735e,021a40cfc1484ad7,0242a7c3c74872d4,025fb49338a19392,0261b80103d834c2,02780d2d47a0410d,0284004274c46cca,02a175a15a1c7249,02be7864cdc91663,02c6ef6248a97cc8,030c26afe4683f13,038c6627b332eee9,039410ba20bde9d3,03b537a6eb25476f,03cc90671f98e060,03d9f85c78c38067,03f59a9f7e20a6a3,03ff6e165df6a4a6,0408289dc0e07ef9,040f6b627b3af082,04301fe8099e030e,04928fca16465ec0,04a907a116aed9fd,04feedb38acebfa3,052650fc2f06b1cf,057fbc7d4dc70747,0583dbdd289c96ab,058b9842a8b0e3cb,0599ff0578c17f99,05df3f5162702aa6,061cd9731cec1e33,061f6c5a9e3270c4,062979ca7e3271c2,062fbdffc302ee2d,0652c062e1856a8c,0699ab2a27a46842,06a459133d3cee3d,0708f03d29e1a3b1,0743276f82a0b126,0745738cf7256ed7,07852ec9db157c21 +R-pr3048 003a2f2b20bc68fd,0056dcd0ce2a2aa6,0089da9fdf49982a,00fa56df2074256c,0111b374f180d4d8,012428ed8a0111d4,0145f1dda22398bc,016d5d3a0f23e1c1,017f8b784497985c,019514d82f9f7779,01cdac2080f77fd3,01f04c6f7218bc92,021385a4a5f84669,022109d345046f9e,024007b51de227a9,02aebb2da415a0e1,02b8e4d6110bee39,02eab0e4445306aa,02efe15b8a76d9be,0317311310f3947a,032ad0766196e5c3,0369fb0a8de212a8,0377c80473b3f02f,039d7b1bd49116b5,03b32691cfa47451,03bd015fc25b7065,03d0973688a13359,03d5fe966baa1c6f,03e407de2c93520a,03e76d5e72d32c71,03f504f698fb7987,04077a85e6cbbfc6,04083c61e2fc82a2,04093c16ffbe4fd9,042282c0d20cd25a,046e2ac6adcb7001,0474781a6af7c647,04761d3e881d17ac,049e02d8b32a120a,04c0d6a7475396fe,0500ed3da5e99ad5,0502e397a34c583d,0520371c248cf9b8,05325b0364e4e378,053406ef474ede3d,053e9846514b7c83,053fb3b106f69c53,0552942598f41133,0570259681a2439f,05720243d25b262a,057c010c6eaa5a54,0587795e82ce472f,058d23117371e5d4,058d83528b44503a,05a244cb54880108,05a7383f7db99220,05bb0b0a02596b86,05bcbbf4a9f2c297,05d5dd01d5b349b4,062067d69ff4de3c,0623025ed009b810,0626d4124a7100da,06351d3a3973cbb5,06366fc160ca9ee6 +R-pr3106 0020ae53a0bebcd6,0058e04219858f44,00b5b00c9155e509,00cc8af616584e30,00f5fe018c814dd7,01123d891bec72d9,017e31d5959fb5f0,019b9f1bb359111a,01ebda77f678f127,020385ec21389ef1,026185c92af394fa,02ea384b3e981e02,030f2fe2b0048a98,031e8c88eb2c2666,0337dbae45e55d03,033a7cce80522fc2,03b812d5cac5da37,03cf88fcad938802,04042cbf6382a26b,0435df33580eed2e,04486dac5fd7d9bd,047bef5f4ad99026,04ba490273d2302f,04c7db12c352ac37,057a089a571402e9,05851cdc593e8460,05c798a512deedba,0627dc9e7f581942,06481fe3a3152248,0650c71fe6c732bd,066bd10f862d85d0,066f69e5d3fee6b2,06a6b450526b8784,06abecc2eaf17b39,06c3270a15270cb4,0743890c0a0e9b26,0760c2029ba60667,078025e6d67bf650,0786a76b9952ab22,079aae2f3958aa3d,07a3dbc022ee2f4e,07b17c7a62a0f930,085d90408a90c01d,0877fdca68b598f9,08815bbf1928bc87,08950cf538ecbe41,089fb416bf93a9a3,08b3aea6e8e6f63e,08d7338f93e4bf5f,08defcd8df138123,090af63abee1cb3a,090c40deb918540f,092b2409c9ba4576,0974e2620d3d9521,097a7f707f082c92,097cf08f2faa4447,097d7ae6f9b075ad,0993d021a56bbf40,09b782d40c89adbc,09d02a5f722712d9,09d0af57e7f18226,0a0f2322bf93f3aa,0a1f221d1651b3f1,0a439b6fc9742f92 +R-pr3216 00019eb5fba780e1,000fd00c1e0d17fe,006284906cb24496,00868c0863d20de5,0087d4096dbaaca6,009b99476b1a8ff2,00d118e892efbe3c,00d1a3ba4e865b38,010403319667523b,011200e9e34cceec,01554f8780b7a007,0160a99ca3ec5267,018fdfd171f72e95,019d17ebbe86e697,01be60b081c6b9b7,01cdbac2efbbb213,01e6bdd6ac73d608,01e8cf4ddc583a99,021f5f74e70ffe47,022ded5b39c32bd9,023c6a4715791814,024007b51de227a9,02612d431b7feccd,028a7bd454efe447,02a2616489db1669,02a7db2858d3627d,02ae6a8608f08ae5,02d67d29dba6cb34,02e98ec6c4d954a7,02eabf23bdf7a699,02fc67427adb97b3,03126a0c0b268c37,03180d8defa9996f,033a03523e38e3bb,0380167f06605046,03a82d3dddfaabac,03c1ad8d577ba951,03f65990c62995c8,0401ab468d896469,0408289dc0e07ef9,0432b44c73a5709a,046dbd16ea1e9b1c,0474e00669d03cbf,0477cd0345d7f449,048f9495e27e9d51,04a094cfa04b1cef,04b96831275a8bf1,04ce29c60ae1aa83,04d31a41ae3942d4,04d674ae243bb7be,04f0d4779c71cbd8,04f8c5f603877131,04fcd11d925053e9,0506a8aa172b08af,0515b3cfa142caf1,053e4e3f170bb5cb,054d2ccb03b662f5,055f473ade88f462,0565546c1c523144,0587cec42859afb7,058826932c209eb8,059ea0925f0d449f,05a119ac0d16f626,05a244cb54880108 +R-pr3307 001e21626b2c27c8,0025be343fb07f53,0038f5626735499d,00397d65bd634edc,003ea3a9666fd412,0040ba39b06168f7,006bd1a5290d5b77,0075625b0dc1200e,0088034c5aa7e56a,009ba2ba7ec08051,0106c6e0c08f5884,0121015d9d31a9b2,0121fa743d401401,0148c05380f90d9b,0169e86aeb5673e1,016a4b2af498848c,0197065fde403ed1,01a54189fea8a104,01b08be593c4d17e,01d0f83f52e81ba2,01f9ee55cd78b93c,0224237eac883c0c,022552fa43608792,0271eb8924e04314,028efb7a87b096eb,02e66170870fbd1c,02e6ee6f4a07390a,02fad7c4fe2b0235,030b3d56666d5a02,033cc8215ff1fc52,03543ababcdb5a47,0357fb205d6abba5,036518e4d0932832,036880ad8bc4319b,03851e7c88496a10,0396825a967a3e17,039c5247f0b70af8,03b9974ce6c3b154,04521b6abd575cda,0454aa78c7238167,046a5efb7b099137,046adc2c985dbe64,046cb895c3b8ca77,0498906d18a1b2e9,04b336f01446650f,04c728cddb439823,04eff842a73e6879,04f8499d18534a86,050095a61b3f14e9,0502e397a34c583d,05106bdd2e2bcb3b,0528d0bad51fd902,053628071ba6f49b,0543ddb1634bce76,0545cbe713ac42d1,054b620659b668f9,058d600e9de334d7,05a37f9866527025,05e273b083a22dd2,060c24a648fce438,060e1b77c1b0c000,061cca727592349b,063517a108983cf3,06366fc160ca9ee6 +R-pr3309 0020faf88152da38,0059caebe73bb9d6,00948317cbf37b11,011998e0cdd5ab11,01975124ca6e0413,0218ecb8eff7bdde,0232d5968f14ca15,027c7c1abeb7afa5,029befc60486eb78,02b418e101d7742a,02c9825881595445,02e26a508e574bd7,0338d70485da77fd,035bb87eaf458d4b,039779c9ab4d1005,03b43f5f8f038325,03e82e23954575b6,042ea0f5ac69a87c,045846c60268160f,04bcf6618d48ba4d,04be19ff742a0058,04c02b78ff345852,050c82742f598c60,054323103c24b0a3,0563c8cb285d116a,056741f642a05d13,05b875c54118c49b,0628d7e8ba509fa8,063cfdb56badd7a7,0647827c2c6c99a7,06816a2d7ed5d0ff,0691863a21596b0f,06932aea9b372ed3,06c3270a15270cb4,06ce60a639f07447,06dcc2bf2e46d449,06f3e03d53b18b83,079fcdeea20ac8c0,07a4e1621391692d,07b155f9b7c3ce12,07b77b6b92386f0f,07c144a11ba000d0,07d18886512fd1bb,0836a18904d7d910,0849123916cd5c05,08be45dc535f7c01,08cb073b8c644880,08df1e3f1d6aa979,091212638aa979eb,0913c8942893654b,0924b2b38909f1bb,093bffc20fac98ca,09d13b0a517f3d47,09f5634abda3e102,0a0178a36900f919,0af9c9950fccdd12,0b2a289b21c20b47,0b4cc927a341144e,0b5b969af2bfc705,0b9202be0df8da45,0bc1d7ec6e449a5b,0bced0b5617cef52,0bf40590117c0ff8,0c21f0a231b92117 +R-pr3319 0042aca04c13249f,0055e2ef0a830326,005a425e495c11ee,00fb49a9a92f3c5c,00fe3e932d2a30b2,012274011e96eadb,012c6548b3e3c628,014889db31acbc19,015af4212dcb3924,0160b851c5dccd6f,01a4ae8feec4e1fc,01e43e3b448e67e7,02275c0ab0baf75a,02f22a65b9f163db,02f76b86d8469ea9,02f9ebb0ac1974e6,03537cafe0a1950a,03b11014d5a82afc,03cfcde06a1f8020,03d8a1c9b890c8a4,03f65990c62995c8,041f523c1b7f60d9,04539ae9a6f25e91,048c0c7939390c98,04fc9bc01402522b,050d76ab4612d329,05461a5f021c3c58,05461a5ffdaee1f0,05ae3d0b3c7d7664,05dc1186b048d6b6,067aa9e7dc840606,0692fbf11ba6ca2e,06a516f8c8d10330,06c3270a15270cb4,0710e47e1e37a73c,075270522d1f47d5,07b77b6b92386f0f,084add288c91e521,0856c760cf32c04e,0859be71351ec919,085a8e0635d2b1d6,086b57e53ad31254,08784e5b671d2ff9,08b5b4793e61f61d,08ba2cf70a23c916,0956d5019eb1cc74,09dce12d468e504d,09e79b28dfcf217e,0a4f1e3e964c0608,0a931f35b368513c,0ac963bd77c25c63,0af9c9950fccdd12,0b2fbc8892c63674,0b33cf682428b0b0,0b7384b75b09ebfe,0bc30789797f2330,0c764a2362443479,0cccb209fdec457f,0cddc2c15ec7b9cd,0d25963ed02a8cdc,0d4587a040c7ef83,0d97a61662998d99,0d9e468ff2a8a60b,0dce2e395da5878b +R-pr3339 0008d97feca332d7,0011ad774d6e004a,002289f9dbd9bf65,00258ec4e1efde6f,003137f1af43e04e,00381630ee16590e,00397cdcd3b4b641,00422c2462571cdb,004a9aaa623ef781,005c0395e94d5252,005fb4599a5f0451,005fcc2a54ba8334,00628926ed33982d,00637ca5ec3a7dde,0064a9e7ebc2c8de,006f2ae379d86b5c,00759c0ca79fd2cf,007795cadc3a3c5e,007941692a53749e,007caabf0259943a,0085dd09433a66e1,008a9dd84193c42b,008e5b8c49fbb0e6,009679b75a8ed4fd,009ac80d6e3b3684,00a4204995352ec5,00b412217a8f939c,00b7aeafa08923f8,00d89dfa3d93bab0,00ddc31ab4e73482,00ddd4ce8119ec9a,00dee0f13587c6e9,00dfbff2a536d1b3,00e7b918330a53cd,00f0dc14b6620818,00fe31e53059b35e,010b7f064a171875,011421e156bde1a1,012274011e96eadb,012eb05c1cc2216c,013ffc9fff9e0d91,01419157d14d2490,01568d632a616b3b,0156f02d7c7f4788,015c42db7e9a4766,0164728d52916479,0169cd6f926977af,016e78637d45d253,016fc3fb285b6ed9,017f3a2a62b30a0e,018488e339c7ee2f,0191322a645b8642,0194b65a5cc107eb,01a0f6e9b968b904,01a3c1c2f94c75a3,01a6d905c9dea7cb,01a71a3af8a4ca85,01a7a117bc71f75f,01acd023cdea2674,01bb5cbdf4f85d69,01be01c93288972c,01be95ce2eba060d,01beab3daada21fb,01c120eb4cd35ca0 +R-pr3405 001094c976a02ea3,0029dadb7fd2757e,0079101d2cda375a,00bd334a416cccc6,012702cc28bd0121,0170c8a3f7ad6935,018706a1d235d154,019ceded31948b17,01a6cbe6fd8c2abe,01b4e8154031155a,01bdcb70058e5551,02183ee4eb1e1e3e,023270b750efe0f7,0247a63bd62eaae6,025dd00086395b55,0294f05077de30d9,029baf9861c7289b,02c1cbc332772424,0301223f81bc3eac,0312948258da49de,0347f6c031d237a5,034efdea84a97894,040bb5cffd4dcafe,040f257a9a0f8d12,04204cb69ff00ae1,0422a5f20897aa7b,046baa73aeec28dd,04a1e061a57655cb,04b8a2ebeced6e81,04bdbdc199118017,04c89f23387a16d0,04dc72f6206e9954,04ef92977947f245,05027b7b98fe5d30,0523fa23ca5613f4,054e3da628beec58,05918e1097908e5f,05e8622363fbbdca,05f61de20539a111,06115dbd2196f791,06366fc160ca9ee6,064cb47dfd25beae,069e19d5816d5ade,06cb161f14225c05,06d72285f40ea2d8,0710ce664545d8c5,0711701b3a475229,0727cffbe2b27434,073921278e97a27e,075cf52aa5e64161,07bc74fbdcd10021,07d3a4f115f380d8,07efbd511e849e52,07f4fb7f4413c503,0805dc10ecf3426d,082ff4516168369a,083f27f7c979035e,08453297bc4f87fc,0848b20face83413,0867e6711c00f705,08720bf9fd31c59a,08c27d869bbd246d,08c74f6ccc05966f,08ca669ff7bb4f5c +R-pr3469 0006948803e05fc9,0012e630745be5ff,001838027e9029d6,00a3255a9ce2d44a,00b477e674537773,00d7a455130283f6,010dd4adbf80acc0,0116214a60a81616,011da93886f76fbe,0130fb41526f544a,016609380949f16f,017189cf5429e687,021656f435d11779,0227336d02c12673,02275c0ab0baf75a,023e67e6a0b3616d,023f6125d9d78340,02aa1c04fefbc5a8,02e5256a50a746f1,02f89dc6b074efaa,0324f7adc6ed3ee6,032546164129d33d,0339945c3efa6035,038489dca54a7573,0399ea2c450ee7e4,03dde332704bde24,0422a1756dd4b002,0438e4975b7072d9,046515d4d714654f,047ca951ff7785ec,04f37fc9861e71d2,0535ead7615abc0a,057f65b285fe6b33,05e22a3dc2ea9a2d,063353a177c20ce2,068aebd250d40230,06b6caa1c1daf295,06d2677a5e1f2234,06d60312a181da08,06eea42e1946df8e,06f43a28c97084fa,06fa7979d88ad25e,075035bea08a5929,078e49066f545d74,07c19e6e4dca2127,07e9c210684eb5c7,07ed228c3923dab2,07f1260c65aca3e4,084095461514744d,08461ab227913814,08608de778786229,0923d5960c333f4c,0962e07add8d11fc,096fb40d3dfd2f6a,0997d7e58462177a,09a327df704ee1d6,09cde1bedfa401e2,09fa17a1b5be2bb9,0a054fe0694046da,0a2f243b8f303d20,0a30f925756f8921,0a7a59dc51884ee7,0a8215cd89a202c0,0acfb853569bbcbc +R-pr3481 0019670d41b88abf,0025c56febee3c1e,00788e8d381460d8,01085921af69bb5d,013bca984a588ae1,0182e1b43ec91412,01cb96661f7b379a,01ccaa7646113fe8,01e4751e9b8bd8bc,0214edf14e5c4caa,0241ace00023c3e1,0244d81eb1cf7df8,02e5e5bfae58eb6c,02fb3e51262e5bbe,030bd71d498d9f4a,034e95700532a72e,0384472ee790c2c9,042fa3b2826ba57b,0435b1a995776cc4,046cc2c718b6528c,04a205dfbf10419d,04a367224e385395,04cc40602bffc4c6,053531982b8adfc6,057f5d03ff3e57f0,05d4d5e3726e34e2,060e868cfc469e2e,0613cda7474e8688,06193b7f79732539,06321240f05ceb16,06366fc160ca9ee6,069ec35d75ef6f01,0720b99da8a88e81,0747cd138cbddd7e,074d7b2f84301eb6,0847021ac1142824,0861ffef1777e211,08fb20476c465092,09114a13d9a90e4c,0962e07add8d11fc,09636a6a3989b083,0979801c1e506c33,09acfc476a0945df,09b494acf11144f2,09efd89737fd902d,09f47bbd76622410,09fa17a1b5be2bb9,0a2b49c2135ff1fa,0a719fd423c4b892,0a9c1cd928670479,0ab08887572c6a26,0ab9be6807c06e34,0ad49228e993d18b,0bf53c7e6d93fff0,0c0e28c81036cc45,0c143fc96aba0e46,0c7bab9ff098f00c,0d93d04a1bc87b8d,0da76a5968661ae9,0ddbfc9368972e9d,0dddb4a21c485cae,0e40f43d140b9546,0e5a8c058142bb26,0eb226430eb1a835 +R-pr3706 00228f94d1fa58a7,002eda1ccbb589b6,0106b31b3deafed8,01150dec476bf84a,01263884d1f841e6,01b9a45c82226f68,01c44aac1d2f433e,01d43a9122b79597,0224869aeb7de738,026793365a3884a4,027511fcb3c655c3,02772f3deafb6bfe,029b7964d4a1b855,02c3fdff305140c1,02cdc379c829a168,02cee8490c3ddad7,0311a38fba9c8f2e,0314797653f64681,033a3c567f55b5f3,033db279cd21002e,034c7fac88b41a98,03b68d5b0b8accb3,03b6db8ac70e37a6,03bbe92d6958fa2a,03ed05e37c27a685,03f0e444fdd21d05,03fbc2f620c93cf6,04018b28947cdd87,0414a34d81eb2da9,0493f30870e7acde,0495d2c2e0e893c6,04a45cbb6feac706,04b838941d6e978d,04ce560bb174f704,04fdf9f74278dd74,050151741bb81e2a,0532a364839f62a4,0535e905921c53c2,05823efbfbafc743,05c58a28b77e74e0,05ef16ab21f508fa,05f7b84c57193d65,060fabc3b9fee435,062c26906fe1ba39,06366fc160ca9ee6,06556845dc09f5a2,066a7dbb695329d6,06795e8d15fc966b,06972c297c42a2db,06af808f3928207b,06be85d69d5b04dc,06c0c8dcb6cc472c,06c24b4369b706eb,06ecf5878811c6ab,0718238bad800dd3,0728b6546a3a99c3,073361f6bef316f1,0736f832c8113635,0745cd696f572c80,07542c3b925ecbc7,0767c4dcc2f1f4c1,077ba73b7647941b,07802c8551530e03,078a18614778fe96 +R-pr410 0281954ddf265646,02acfd707eca0271,059d755dc32c8117,08297316c9ee2146,09418f8fbb0515dc,0bb540c7e6ceae6a,0d3a28f35f0e92ce,13ba0a82959947af,159b51d4b93fdd88,1890138da69a8c98,1bb45f169f1c8baf,1bef78b242ec1fbc,1cc63bab9bf15e9d,1cd93a2993bc3e7d,1d52a2daf390ba37,1e50db9e924e7cfc,220760523de20ce6,2b3918781f144885,2bc3ec81f8b013ca,2ddc677d5cade068,31a25e568426aeff,31e93c288289f291,328d1b156612f17e,37c1381e5e82d2c1,37d9a7ac7b3d9f5d,383e26d1cf626836,3d97d59349456420,3e06464c36cb5641,446dc15d11143a3a,4a42e9a9075cb408,4b3f01884b33b916,4b400438720ab9cc,4d602f4aaeb19368,4e0a8e94d9a271c9,4e756ea45e327439,4fc361f6628c3224,50c78221a6449f10,541eae15e4030aa6,55148ac6e6d79373,563b98ac379a5a47,57a11377a1d78a78,5d5c4a2325c7f671,5f68af7a94abad89,60368c51bf845293,64581b1211048c86,660272e2094f2d98,6baefd7c1f83fe13,6cd32d59fa7a6d90,6eb2efe4fdd43bb1,6f1552583fb4a09d,7090e682425432ff,7e9a8783ee1a72ee,80086687df462118,857ecd4bd508651c,887ec04a6a5db015,8e309af7d656dc51,8e654b563860104f,9174e8fb242076f8,9686bad31c5704dc,97bb544ce7c6277e,99d183442be952bb,9d1271536b2c5c1f,a07ebce1b585a5cf,abaf22d278434400 +R-pr411 000225fdf97e68e3,000397491624e4c0,000577fbd5435cd5,000b8ca2d59d9090,000d061e08faf9cf,000f3eef78c96c4b,00102e9a21aeab3f,002326560fdcc3c6,002a4ecc191c660d,002aa38d9c99b5d0,002adc8ac35affb3,002e94808c0c14d7,0032d5c491abbcfe,003355484380ced9,003cc64d201a42f8,004be879c75215d5,004d8df16623891e,0051674d31770ac1,00547e800e5a91d6,0054d52c2a37652a,00595bf7945b2a7f,0059b01f12afd7a3,00602d81954341ed,006430a94294546a,00677b08b79d852e,00685867cead7e61,006c1ce19c1e32f9,007093e6fb67f5b7,0071660f8bd5509d,0074a2a4cbe25698,0074f1af57f8b225,0076a9fb096efac6,0076d1efe1f915fc,007adfd44681a49a,0081adc748f0c322,0083b3b48c00cebc,0086f0976c21edb2,008760c133e12b51,008b7d47b1962d7e,00907fc6599b6952,009218be48141ff4,00986b11639d0f07,009931d69b18701a,009ce54f9d530624,00a5c266c1c89ddc,00a657fe0e892a0f,00a849520a8f799b,00ae68cb257e1715,00b3c0e8f3565313,00b3e83624f71565,00b5cf6363c85270,00b88ff4d14b7e04,00b9b6fb9c779999,00bc0950075213ea,00bf6f7949734446,00c2ecfd927b5896,00c3554ac7e689e5,00c45287ea5482e4,00c5bcbaf8ed0862,00c7141a5107a04c,00d5f21b5bb906f2,00d78147b9b5c1e3,00dd2c758255b487,00e109d3367fc646 +R-pr750 0002afe7ec7a17d1,000306f00acefdaf,001dbb6c96eb720c,0020a420afbcfe6c,00501a8c81900f37,007c977a0a045202,009471d2b4945d56,00a49f2d15238619,00af6c049db4b78f,00b658a11b4b703e,00d8035dc5dc7bc9,00f83879b27d7224,011f12b5bd9c9fea,01213880213f6f6e,012a5051e19f36b3,0157d971a83db2fd,015ca00f9963dfd7,0187f14af61e67a8,01c275e21e7716e7,02057794a3980890,0246eace4e4cd098,024d8fe5731a5e2a,025c959dafabb761,02890af4945d9192,028cf8f92d4a8068,02ade39fdceed4b5,02cc18ee46b90583,02cc72e142bcef2a,02d749cf9740be99,02da283eb95ff1d0,02e9a8f476c71d30,03153ab8b9e6fa53,0317c16ee5621e14,0322175bd6ebe8db,032d1a970205a307,0337bc3852bb41cf,0364cc0ce2759782,0366b24b2c879b7a,036b6636c2d90f71,0386d97c753f1a08,0387c3f9fb6e04bd,03c971a5517fb83b,03cae8494e8bf21a,03fa614f60a69a91,0413113c3a802676,04177c9f93dae2de,0427204bfcc645c5,0463e5adf4e2300a,046782b0b0707e3c,046b856bb1211c6f,046f7ac84280c524,04b5bc4036e75aa6,04b6f14727b1fefb,04c401f8dced6259,04cb92881ccedf13,04fc757f905d10e9,052e67191b5927b5,05302fb7243c62e8,056883305a489514,057d1f7ce4fb13a3,059a2629a28f7c4a,059c37ef5ecb56ce,05a58a06ee14eb22,05ac5fcc433b816f diff --git a/docs/audits/rex-001/prm-04-admission.jsonl b/docs/audits/rex-001/prm-04-admission.jsonl new file mode 100644 index 0000000000..5b7fce7c2f --- /dev/null +++ b/docs/audits/rex-001/prm-04-admission.jsonl @@ -0,0 +1,6 @@ +{"schema":"rex-cell-admission-v1","cell":"C1","host":"intel","backend":"wgpu","apr_tag":"v0.69.3","apr_sha256":"8a67a0103cbb036332908cc184d5a8a8ae422fb9025bd5c78e42d65df86cfbd7","model_id":"Qwen3.5-4B-Q4_K_M","weights_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","prereg_sha":"f0b4e4ac07e605e313e61ce5efa9e43f7339ca5a5e5275b0d5f06b0277011041","at":"2026-09-25T20:01:59Z","status":{"state":"NotRun","reason":"NoDeclaredExecutor"}} +{"schema":"rex-cell-admission-v1","cell":"C2","host":"intel","backend":"cpu","apr_tag":"v0.69.3","apr_sha256":"8a67a0103cbb036332908cc184d5a8a8ae422fb9025bd5c78e42d65df86cfbd7","model_id":"Qwen3.5-4B-Q4_K_M","weights_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","prereg_sha":"f0b4e4ac07e605e313e61ce5efa9e43f7339ca5a5e5275b0d5f06b0277011041","at":"2026-09-25T20:01:59Z","status":{"state":"NotRun","reason":"NoDeclaredExecutor"}} +{"schema":"rex-cell-admission-v1","cell":"C3","host":"lambda-labs","backend":"cpu","apr_tag":"v0.69.3","apr_sha256":"8a67a0103cbb036332908cc184d5a8a8ae422fb9025bd5c78e42d65df86cfbd7","model_id":"Qwen3.5-4B-Q4_K_M","weights_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","prereg_sha":"f0b4e4ac07e605e313e61ce5efa9e43f7339ca5a5e5275b0d5f06b0277011041","at":"2026-09-25T20:01:59Z","status":{"state":"NotRun","reason":"NoDeclaredExecutor"}} +{"schema":"rex-cell-admission-v1","cell":"C5a","host":"mini","backend":"cpu","apr_tag":"v0.69.3","apr_sha256":"8a67a0103cbb036332908cc184d5a8a8ae422fb9025bd5c78e42d65df86cfbd7","model_id":"Qwen3.5-4B-Q4_K_M","weights_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","prereg_sha":"f0b4e4ac07e605e313e61ce5efa9e43f7339ca5a5e5275b0d5f06b0277011041","at":"2026-09-25T20:01:59Z","status":{"state":"NotRun","reason":"NoDeclaredExecutor"}} +{"schema":"rex-cell-admission-v1","cell":"C5b","host":"mini","backend":"metal","apr_tag":"v0.69.3","apr_sha256":"8a67a0103cbb036332908cc184d5a8a8ae422fb9025bd5c78e42d65df86cfbd7","model_id":"Qwen3.5-4B-Q4_K_M","weights_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","prereg_sha":"f0b4e4ac07e605e313e61ce5efa9e43f7339ca5a5e5275b0d5f06b0277011041","at":"2026-09-25T20:01:59Z","status":{"state":"NotRun","reason":"NoDeclaredExecutor"}} +{"schema":"rex-cell-admission-v1","cell":"C4","host":"gx10","backend":"cuda","apr_tag":"v0.69.3","apr_sha256":"2f274f477275c578a511fd5fd90e8c9e36e686df88b8eff0529d80ced5e2508c","model_id":"Qwen3.5-4B-Q4_K_M","weights_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","prereg_sha":"f0b4e4ac07e605e313e61ce5efa9e43f7339ca5a5e5275b0d5f06b0277011041","at":"2026-09-25T20:01:59Z","status":{"state":"NotRun","reason":"Inadmissible"}} diff --git a/docs/audits/rex-001/prm-04-receipt.md b/docs/audits/rex-001/prm-04-receipt.md new file mode 100644 index 0000000000..a71326972c --- /dev/null +++ b/docs/audits/rex-001/prm-04-receipt.md @@ -0,0 +1,64 @@ +# PRM-04 — cell admission re-run under `parity.oracle` (rex-cell-admission-v1 1.1.0) + +**Outcome: receipted refusal. 0 of 6 cells Admitted; S-7 stands** (`rex admission-check` exit 10). +C4 has a passing declared-oneshot parity receipt, but it measured 7 positions, and admission requires 64. + +## What changed + +- `admission::parity_from_receipt` derives an `Admitted` parity block from a receipt's bytes and never from typed numbers. It is covered by FALSIFY-RCA-004..006. + - It reads two receipt shapes: + - the `apr-review-serve parity` oneshot (comparator `apr-cpu…`, recorded as oracle `apr-parity-gpu-cpu`, the §4.1 rule-1 interim oracle); + - `apr-parity-oracle/v1` from `apr parity-oracle` (#4444, 57's branch @03579caa9; oracle `llama.cpp@d1d3c3396`). + - The cosine is the **minimum over the receipt's positions**. The receipt's summary `cosine` is never read. + - The threshold and basis are the **declared** ones. A receipt's self-chosen threshold is never read. + - It refuses on any of these: + - fewer than `min_positions` positions; + - exit ≠ 0, a verdict that is not passing, or failed > 0; + - any position below the threshold, or a missing or non-numeric cosine; + - an undeclared oracle, an empty basis, or a non-finite threshold; + - a binary or weights sha that differs from the row's. +- `rex admit --cell C --parity-receipt R --threshold T --threshold-basis B --min-positions N` appends the row. It admits one cell at a time, never `all`. When the receipt does not admit, it writes nothing, exits 1 and names the reason. + +## The run + +| Cell | State | Why | +|------|-------|-----| +| C1 intel/wgpu, C2 intel/cpu, C3 lambda-labs/cpu, C5a mini/cpu, C5b mini/metal | NotRun{NoDeclaredExecutor} | unchanged since REX-04; resumes on infra#1088 | +| C4 gx10/cuda | NotRun{Inadmissible} | `rex admit` refused: `7 positions < min_positions 64` | + +The C4 evidence is `prm-04/gx10-parity-20260925T124817Z.json`, sha256 `1f7102e52f974106ebbe9e6c3de49a0d16332ae0faf05d1f7e233034dc88b339`. It was read from gx10's declared state dir `~/.local/state/apr-review-serve/parity/`. +- It is the output of the declared oneshot `apr-review-serve parity`. +- Its two absolute weight paths were redacted to `/Qwen3.5-4B-Q4_K_M.gguf` on 2026-10-10, before it reached main. Nothing else changed. The capture as read hashed to `54e2b72a12f2413e8b82ecb1a93b38a7b01a3ea671c5d4702365328b60844b67`. +- Binary: apr v0.69.3 `ad5d07a1`, binary sha `2f274f47…`. +- Weights: Qwen3.5-4B-Q4_K_M, sha `00fe7986…`. +- Result: exit 0, verdict pass, 7/7 positions passed, min cosine **0.99970** against the 0.98 threshold. +- **The only failing condition is the evidence floor.** + - `evidence/parity/thresholds.yaml` sets `min_positions: 64` (basis I8 / CF-4, #1864: "any autoregressive gate validates over >= 64 positions"). + - The 0.98 default was itself measured over ≥ 64 positions. + - A 7-position smoke check cannot use a threshold whose basis is 64 positions. + +Positive control: the same receipt with `--min-positions 7` does admit (cosine 0.9997043609619141, oracle `apr-parity-gpu-cpu`). So the refusal is the floor and nothing else. The row it produced was a scratch file and is not part of this audit. + +## Why not the llama.cpp oracle + +`apr parity-oracle` (#4444) has only been measured on Qwen3.5-**0.8B**. The prereg-locked model is **4B**. Its threshold row `qwen3.5-0.8b@cpu-vs-llama.cpp` deliberately has no `min_cosine`. No 4B llama.cpp reference exists yet, so this oracle cannot admit a cell. + +## To admit C4 (resume conditions) + +1. The declared oneshot must measure at least 64 positions. The fix is a longer parity prompt, such as the 78-token corpus prompt the 0.98 basis used. Then re-run: + ``` + rex admit --cell C4 --parity-receipt --threshold 0.98 --threshold-basis "evidence/parity/thresholds.yaml default.min_cosine" --min-positions 64 --apr-sha --weights-sha 00fe7986… + ``` +2. The serve declaration (infra e872544c) and the parity oneshot (infra 3f66d253/3d59dd3e) are live on gx10, but only through the open infra PRs #1142, #1154 and fold #1054. **They are not on infra main**, and infra#1088 is open. An Admitted C4 still rests on a declaration that has not landed. +3. H1 (verdict invariance) is evaluated in the analysis (PRM-08), not at admission. + +## Note + +The prereg lock the tool reads today is `f0b4e4ac…`, while the REX-04 rows carry `ef51087d…`. This file uses the current lock. REX-04's file is left untouched. + +## Gates + +- `cargo test -p aprender-review-experiment --lib`: 167 passed. +- `cargo clippy -p aprender-review-experiment --all-targets -D warnings`: clean. +- `pv validate contracts/rex-cell-admission-v1.yaml`: valid. +- `cargo mutants --re parity_from_receipt`: 21 caught, 1 unviable, 1 missed. The missed mutant was an unreachable `is_finite` guard, which was removed because JSON carries no NaN or ∞. diff --git a/docs/audits/rex-001/prm-04/gx10-parity-20260925T124817Z.json b/docs/audits/rex-001/prm-04/gx10-parity-20260925T124817Z.json new file mode 100644 index 0000000000..2f8e18ed1c --- /dev/null +++ b/docs/audits/rex-001/prm-04/gx10-parity-20260925T124817Z.json @@ -0,0 +1,112 @@ +{ + "host": "gx10-a5b5", + "at": "20260925T124817Z", + "apr_tag": "v0.69.3", + "apr_commit": "ad5d07a182b15e70527d3f33f18ceb1211ba4c80", + "binary_sha256": "2f274f477275c578a511fd5fd90e8c9e36e686df88b8eff0529d80ced5e2508c", + "weights": "/Qwen3.5-4B-Q4_K_M.gguf", + "weights_sha256": "00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4", + "comparator": "apr-cpu (apr parity: GPU vs CPU; no llama.cpp comparator in this apr, aprender#3576)", + "exit": 0, + "verdict": "pass", + "parity": { + "model": "/Qwen3.5-4B-Q4_K_M.gguf", + "tokens": 7, + "passed": 7, + "failed": 0, + "parity": true, + "metrics": [ + { + "position": 0, + "token_id": 3710, + "cpu_argmax": 369, + "gpu_argmax": 369, + "max_abs_diff": 0.2318652868270874, + "mean_abs_diff": 0.03146722540259361, + "cosine_similarity": 0.9998131990432739, + "kl_divergence": 0.00021212672537373304, + "sigma_level": 100.47431010234187, + "cpk": 33.22796605385059, + "verdict": "Pass" + }, + { + "position": 1, + "token_id": 369, + "cpu_argmax": 279, + "gpu_argmax": 279, + "max_abs_diff": 0.14195120334625244, + "mean_abs_diff": 0.02431115135550499, + "cosine_similarity": 0.9999650120735168, + "kl_divergence": 0.000028142869161619233, + "sigma_level": 131.23602099621212, + "cpk": 43.479465434592946, + "verdict": "Pass" + }, + { + "position": 2, + "token_id": 220, + "cpu_argmax": 16, + "gpu_argmax": 16, + "max_abs_diff": 0.24833190441131592, + "mean_abs_diff": 0.03066864237189293, + "cosine_similarity": 0.999754011631012, + "kl_divergence": 0.00010543855908146571, + "sigma_level": 103.40762116359619, + "cpk": 34.20492610853252, + "verdict": "Pass" + }, + { + "position": 3, + "token_id": 17, + "cpu_argmax": 15, + "gpu_argmax": 15, + "max_abs_diff": 0.20449680089950562, + "mean_abs_diff": 0.030058100819587708, + "cosine_similarity": 0.9998959898948669, + "kl_divergence": 0.00007435307399205818, + "sigma_level": 105.21864173972742, + "cpk": 34.80932453478307, + "verdict": "Pass" + }, + { + "position": 4, + "token_id": 10, + "cpu_argmax": 17, + "gpu_argmax": 17, + "max_abs_diff": 0.23898255825042725, + "mean_abs_diff": 0.03911856189370155, + "cosine_similarity": 0.999718189239502, + "kl_divergence": 0.0002524354532544672, + "sigma_level": 82.05736118788325, + "cpk": 27.084956565755913, + "verdict": "Pass" + }, + { + "position": 5, + "token_id": 17, + "cpu_argmax": 30, + "gpu_argmax": 30, + "max_abs_diff": 0.23314523696899414, + "mean_abs_diff": 0.03875268995761871, + "cosine_similarity": 0.9997207522392273, + "kl_divergence": 0.0005191280388814299, + "sigma_level": 82.29276982289848, + "cpk": 27.165167758074453, + "verdict": "Pass" + }, + { + "position": 6, + "token_id": 30, + "cpu_argmax": 271, + "gpu_argmax": 271, + "max_abs_diff": 0.23418569564819336, + "mean_abs_diff": 0.03857100382447243, + "cosine_similarity": 0.9997043609619141, + "kl_divergence": 0.00028003667922200324, + "sigma_level": 82.17311717474466, + "cpk": 27.126914090180225, + "verdict": "Pass" + } + ] + } +} diff --git a/docs/audits/surface_audit.csv b/docs/audits/surface_audit.csv index 9e537d7804..ed7f3dcfa9 100644 --- a/docs/audits/surface_audit.csv +++ b/docs/audits/surface_audit.csv @@ -35,9 +35,9 @@ aprender-data,aprender-data repl,6,UNKNOWN,huggingface-transformers,no,4,data-pi aprender-data,aprender-data schema,6,UNKNOWN,huggingface-transformers,no,4,data-pipeline,crates/aprender-data/src/cli/mod.rs:66,high aprender-data,aprender-data view,6,UNKNOWN,huggingface-transformers,no,4,data-pipeline,crates/aprender-data/src/cli/mod.rs:140,high apr,DELETE /api/delete,6,x86_64-linux,ollama,no,10,apr-core-commands,crates/apr-cli/src/commands/serve/ollama.rs:654,medium -apr,GET /,6,x86_64-linux,llama.cpp,yes,3,http-apr-serve,crates/aprender-serve/src/api/router.rs:258,high -apr,GET /api/tags,6,x86_64-linux,ollama,yes,3,http-apr-serve,crates/aprender-serve/src/api/router.rs:170,high -apr,GET /api/version,9,x86_64-linux,ollama,yes,3,http-apr-serve,crates/aprender-serve/src/api/router.rs:172,medium +apr,GET /,6,x86_64-linux,llama.cpp,yes,3,http-apr-serve,crates/aprender-serve/src/api/router.rs:271,high +apr,GET /api/tags,6,x86_64-linux,ollama,yes,3,http-apr-serve,crates/aprender-serve/src/api/router.rs:178,high +apr,GET /api/version,9,x86_64-linux,ollama,yes,3,http-apr-serve,crates/aprender-serve/src/api/router.rs:180,medium apr,GET /health,9,x86_64-linux,llama.cpp,yes,3,http-apr-serve,crates/aprender-serve/src/api/router.rs:89,medium apr,GET /health/live,9,x86_64-linux,llama.cpp,yes,3,http-apr-serve,crates/aprender-serve/src/api/router.rs:90,medium apr,GET /health/ready,9,x86_64-linux,llama.cpp,yes,3,http-apr-serve,crates/aprender-serve/src/api/router.rs:91,medium @@ -47,19 +47,19 @@ apr,GET /models,6,x86_64-linux,llama.cpp,yes,3,http-apr-serve,crates/aprender-se apr,GET /ready,6,x86_64-linux,llama.cpp,yes,3,http-apr-serve,crates/aprender-serve/src/api/router.rs:93,high apr,GET /realize/model,4,x86_64-linux,llama.cpp,yes,3,http-apr-serve,crates/aprender-serve/src/api/router.rs:105,high apr,GET /tensors,6,x86_64-linux,llama.cpp,no,10,apr-core-commands,crates/apr-cli/src/commands/serve/safetensors.rs:170,low -apr,GET /v1/audit/:request_id,7,x86_64-linux,llama.cpp,yes,3,http-apr-serve,crates/aprender-serve/src/api/router.rs:146,medium -apr,GET /v1/effective-config,7,x86_64-linux,llama.cpp,yes,3,http-apr-serve,crates/aprender-serve/src/api/router.rs:161,high -apr,GET /v1/gpu/status,7,x86_64-linux,vllm,yes,3,http-apr-serve,crates/aprender-serve/src/api/router.rs:149,medium -apr,GET /v1/metrics,6,x86_64-linux,llama.cpp,yes,3,http-apr-serve,crates/aprender-serve/src/api/router.rs:156,high +apr,GET /v1/audit/:request_id,7,x86_64-linux,llama.cpp,yes,3,http-apr-serve,crates/aprender-serve/src/api/router.rs:148,medium +apr,GET /v1/effective-config,7,x86_64-linux,llama.cpp,yes,3,http-apr-serve,crates/aprender-serve/src/api/router.rs:169,high +apr,GET /v1/gpu/status,7,x86_64-linux,vllm,yes,3,http-apr-serve,crates/aprender-serve/src/api/router.rs:151,medium +apr,GET /v1/metrics,6,x86_64-linux,llama.cpp,yes,3,http-apr-serve,crates/aprender-serve/src/api/router.rs:161,high apr,GET /v1/models,9,x86_64-linux,llama.cpp,yes,3,http-apr-serve,crates/aprender-serve/src/api/router.rs:130,medium -apr,HTTP 404 fallback route index,6,x86_64-linux,llama.cpp,no,3,http-apr-serve,crates/aprender-serve/src/api/router.rs:279,high -apr,POST /api/chat,3,x86_64-linux,ollama,yes,3,http-apr-serve,crates/aprender-serve/src/api/router.rs:165,high +apr,HTTP 404 fallback route index,6,x86_64-linux,llama.cpp,no,3,http-apr-serve,crates/aprender-serve/src/api/router.rs:292,high +apr,POST /api/chat,3,x86_64-linux,ollama,yes,3,http-apr-serve,crates/aprender-serve/src/api/router.rs:173,high apr,"POST /api/chat (stream:true, NDJSON)",3,x86_64-linux,ollama,no,3,http-apr-serve,crates/aprender-serve/src/api/ollama_handlers.rs:610 (framing :383-421),high -apr,POST /api/embeddings,6,x86_64-linux,ollama,yes,3,http-apr-serve,crates/aprender-serve/src/api/router.rs:175,high -apr,POST /api/generate,3,x86_64-linux,ollama,yes,3,http-apr-serve,crates/aprender-serve/src/api/router.rs:166,high +apr,POST /api/embeddings,6,x86_64-linux,ollama,yes,3,http-apr-serve,crates/aprender-serve/src/api/router.rs:183,high +apr,POST /api/generate,3,x86_64-linux,ollama,yes,3,http-apr-serve,crates/aprender-serve/src/api/router.rs:174,high apr,"POST /api/generate (stream:true, NDJSON)",3,x86_64-linux,ollama,no,3,http-apr-serve,crates/aprender-serve/src/api/ollama_handlers.rs:665 (framing :383-421),high apr,POST /api/pull,6,x86_64-linux,ollama,no,10,apr-core-commands,crates/apr-cli/src/commands/serve/ollama.rs:650,medium -apr,POST /api/show,6,x86_64-linux,ollama,yes,3,http-apr-serve,crates/aprender-serve/src/api/router.rs:171,high +apr,POST /api/show,6,x86_64-linux,ollama,yes,3,http-apr-serve,crates/aprender-serve/src/api/router.rs:179,high apr,POST /batch/generate,2,x86_64-linux,llama.cpp,yes,3,http-apr-serve,crates/aprender-serve/src/api/router.rs:99,high apr,POST /batch/tokenize,2,x86_64-linux,llama.cpp,yes,3,http-apr-serve,crates/aprender-serve/src/api/router.rs:98,high apr,POST /generate,2,x86_64-linux,llama.cpp,yes,3,http-apr-serve,crates/aprender-serve/src/api/router.rs:97,high @@ -70,23 +70,23 @@ apr,POST /realize/generate,2,x86_64-linux,llama.cpp,yes,3,http-apr-serve,crates/ apr,POST /realize/reload,4,x86_64-linux,llama.cpp,yes,3,http-apr-serve,crates/aprender-serve/src/api/router.rs:106,high apr,POST /stream/generate,2,x86_64-linux,llama.cpp,yes,3,http-apr-serve,crates/aprender-serve/src/api/router.rs:100,high apr,POST /tokenize,2,x86_64-linux,llama.cpp,yes,3,http-apr-serve,crates/aprender-serve/src/api/router.rs:96,high -apr,POST /v1/batch/completions,9,x86_64-linux,vllm,yes,3,http-apr-serve,crates/aprender-serve/src/api/router.rs:152,medium +apr,POST /v1/batch/completions,9,x86_64-linux,vllm,yes,3,http-apr-serve,crates/aprender-serve/src/api/router.rs:157,medium apr,POST /v1/chat/completions,4,x86_64-linux;nvidia-cuda,llama.cpp,yes,3,http-apr-serve,crates/aprender-serve/src/api/router.rs:134,high -apr,"POST /v1/chat/completions (stream:true, SSE)",4,x86_64-linux;nvidia-cuda,llama.cpp,no,3,http-apr-serve,"crates/aprender-serve/src/api/openai_handlers.rs:878-948,1017-1151 (stream:true branches 1234,1332; cuda stream branches crates/aprender-serve/src/api/cuda_chat_backend.rs:147,441,602,755,1335,1366; ignore_eos fail-closed 16,561,712)",high -apr,POST /v1/chat/completions/stream,4,x86_64-linux,llama.cpp,yes,3,http-apr-serve,crates/aprender-serve/src/api/router.rs:139,high +apr,"POST /v1/chat/completions (stream:true, SSE)",4,x86_64-linux;nvidia-cuda,llama.cpp,no,3,http-apr-serve,"crates/aprender-serve/src/api/openai_handlers.rs:913-983,1052-1186 (stream:true branches 1282,1380; cuda stream branches crates/aprender-serve/src/api/cuda_chat_backend.rs:147,441,602,755,1395,1426; ignore_eos fail-closed 16,561,712)",high +apr,POST /v1/chat/completions/stream,4,x86_64-linux,llama.cpp,yes,3,http-apr-serve,crates/aprender-serve/src/api/router.rs:141,high apr,POST /v1/completions,2,x86_64-linux,llama.cpp,yes,3,http-apr-serve,crates/aprender-serve/src/api/router.rs:131,high apr,"POST /v1/completions (stream:true, SSE)",2,x86_64-linux,llama.cpp,no,3,http-apr-serve,crates/aprender-serve/src/api/gpu_completions_handler.rs:405 (SSE :340-378),high -apr,POST /v1/embeddings,4,x86_64-linux,llama.cpp,yes,3,http-apr-serve,crates/aprender-serve/src/api/router.rs:142,high -apr,POST /v1/explain,4,x86_64-linux,llama.cpp,yes,3,http-apr-serve,crates/aprender-serve/src/api/router.rs:145,high -apr,POST /v1/gpu/warmup,7,x86_64-linux,vllm,yes,3,http-apr-serve,crates/aprender-serve/src/api/router.rs:148,medium -apr,POST /v1/logprobs,7,nvidia-cuda,llama.cpp,yes,3,http-apr-serve,crates/aprender-serve/src/api/router.rs:222,low -apr,POST /v1/perplexity,7,nvidia-cuda,llama.cpp,yes,3,http-apr-serve,crates/aprender-serve/src/api/router.rs:223,low -apr,POST /v1/predict,5,x86_64-linux,llama.cpp,yes,3,http-apr-serve,crates/aprender-serve/src/api/router.rs:144,high +apr,POST /v1/embeddings,4,x86_64-linux,llama.cpp,yes,3,http-apr-serve,crates/aprender-serve/src/api/router.rs:144,high +apr,POST /v1/explain,4,x86_64-linux,llama.cpp,yes,3,http-apr-serve,crates/aprender-serve/src/api/router.rs:147,high +apr,POST /v1/gpu/warmup,7,x86_64-linux,vllm,yes,3,http-apr-serve,crates/aprender-serve/src/api/router.rs:150,medium +apr,POST /v1/logprobs,7,nvidia-cuda,llama.cpp,yes,3,http-apr-serve,crates/aprender-serve/src/api/router.rs:235,low +apr,POST /v1/perplexity,7,nvidia-cuda,llama.cpp,yes,3,http-apr-serve,crates/aprender-serve/src/api/router.rs:236,low +apr,POST /v1/predict,5,x86_64-linux,llama.cpp,yes,3,http-apr-serve,crates/aprender-serve/src/api/router.rs:146,high apr,apr attn-parity-lint,1,x86_64-linux,NONE,yes,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:1007,high apr,apr attn-viz-lint,1,x86_64-linux,NONE,yes,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:1021,high apr,apr audio-inspect-lint,1,x86_64-linux,NONE,yes,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:984,high apr,apr awq-lint,1,x86_64-linux,NONE,yes,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:878,high -apr,apr beat-run,2,x86_64-linux,NONE,yes,10,apr-core-commands,crates/apr-cli/src/commands_enum.rs:429,high +apr,apr beat-run,2,x86_64-linux,NONE,yes,10,apr-core-commands,crates/apr-cli/src/commands_enum.rs:435,high apr,apr bench,6,x86_64-linux,llama.cpp,yes,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:88,high apr,apr canary check,4,x86_64-linux,NONE,no,10,apr-core-commands,crates/apr-cli/src/commands/canary.rs:45,medium apr,apr canary create,4,x86_64-linux,NONE,no,10,apr-core-commands,crates/apr-cli/src/commands/canary.rs:31,medium @@ -121,14 +121,14 @@ apr,apr chat,6,x86_64-linux,llama.cpp,yes,1,apr-lint-diag,crates/apr-cli/src/ext apr,apr chat --backend cpu,6,x86_64-linux,ollama,no,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:85 (BACKEND_VALUES crates/apr-cli/src/commands_enum.rs:16),high apr,apr chat --backend cuda,6,x86_64-linux;nvidia-cuda,ollama,no,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:85 (BACKEND_VALUES crates/apr-cli/src/commands_enum.rs:16),medium apr,apr chat --backend wgpu,6,UNKNOWN,ollama,no,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:85 (BACKEND_VALUES crates/apr-cli/src/commands_enum.rs:16),high -apr,apr check,2,x86_64-linux,NONE,yes,10,apr-core-commands,crates/apr-cli/src/commands_enum.rs:781,high +apr,apr check,2,x86_64-linux,NONE,yes,10,apr-core-commands,crates/apr-cli/src/commands_enum.rs:787,high apr,apr check-finite-lint,1,x86_64-linux,NONE,yes,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:1041,low -apr,apr code,2,x86_64-linux,claude-code,yes,10,apr-core-commands,crates/apr-cli/src/commands_enum.rs:836,high -apr,apr code --input-format json,2,x86_64-linux,claude-code,no,10,apr-core-commands,crates/apr-cli/src/commands_enum.rs:891 (CodeInputFormat crates/apr-cli/src/commands_enum.rs:120),high -apr,apr code --output-format json,2,x86_64-linux,claude-code,no,10,apr-core-commands,crates/apr-cli/src/commands_enum.rs:797 (CodeOutputFormat crates/apr-cli/src/commands_enum.rs:31),high +apr,apr code,2,x86_64-linux,claude-code,yes,10,apr-core-commands,crates/apr-cli/src/commands_enum.rs:842,high +apr,apr code --input-format json,2,x86_64-linux,claude-code,no,10,apr-core-commands,crates/apr-cli/src/commands_enum.rs:897 (CodeInputFormat crates/apr-cli/src/commands_enum.rs:120),high +apr,apr code --output-format json,2,x86_64-linux,claude-code,no,10,apr-core-commands,crates/apr-cli/src/commands_enum.rs:803 (CodeOutputFormat crates/apr-cli/src/commands_enum.rs:31),high apr,apr compare-hf,9,x86_64-linux,safetensors,yes,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:563,medium -apr,apr compile,9,x86_64-linux,llama.cpp,yes,10,apr-core-commands,crates/apr-cli/src/commands_enum.rs:682,medium -apr,apr convert,4,x86_64-linux,llama.cpp,yes,10,apr-core-commands,crates/apr-cli/src/commands_enum.rs:604,high +apr,apr compile,9,x86_64-linux,llama.cpp,yes,10,apr-core-commands,crates/apr-cli/src/commands_enum.rs:688,medium +apr,apr convert,4,x86_64-linux,llama.cpp,yes,10,apr-core-commands,crates/apr-cli/src/commands_enum.rs:610,high apr,apr data audit,3,x86_64-linux,huggingface-transformers,no,4,data-pipeline,crates/apr-cli/src/data_commands.rs:54,high apr,apr data balance,6,x86_64-linux,huggingface-transformers,no,4,data-pipeline,crates/apr-cli/src/data_commands.rs:126,high apr,apr data decontaminate,2,x86_64-linux,huggingface-transformers,no,4,data-pipeline,crates/apr-cli/src/data_commands.rs:96,high @@ -171,12 +171,12 @@ apr,apr data x schema,6,x86_64-linux,huggingface-transformers,no,4,data-pipeline apr,apr data x view,6,x86_64-linux,huggingface-transformers,no,4,data-pipeline,crates/aprender-data/src/cli/mod.rs:140,low apr,apr dataset audio-inspect,6,x86_64-linux,huggingface-transformers,no,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:1441,low apr,apr ddp-metrics-lint,1,x86_64-linux,NONE,yes,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:953,high -apr,apr debug,3,x86_64-linux,safetensors,yes,10,apr-core-commands,crates/apr-cli/src/commands_enum.rs:292,high -apr,apr debug embed-viz,6,x86_64-linux,safetensors,no,10,apr-core-commands,crates/apr-cli/src/commands_enum.rs:978,low +apr,apr debug,3,x86_64-linux,safetensors,yes,10,apr-core-commands,crates/apr-cli/src/commands_enum.rs:298,high +apr,apr debug embed-viz,6,x86_64-linux,safetensors,no,10,apr-core-commands,crates/apr-cli/src/commands_enum.rs:984,low apr,apr decrypt,9,x86_64-linux,NONE,yes,10,apr-core-commands,crates/apr-cli/src/tool_commands.rs:140,medium apr,apr devices,8,x86_64-linux;nvidia-cuda,ollama,yes,10,apr-core-commands,crates/apr-cli/src/extended_commands.rs:647,high apr,apr diagnose,9,x86_64-linux,NONE,yes,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:834,medium -apr,apr diff,4,x86_64-linux;nvidia-cuda,safetensors,yes,10,apr-core-commands,crates/apr-cli/src/commands_enum.rs:313,high +apr,apr diff,4,x86_64-linux;nvidia-cuda,safetensors,yes,10,apr-core-commands,crates/apr-cli/src/commands_enum.rs:319,high apr,apr distill,9,x86_64-linux,unsloth,yes,10,apr-core-commands,crates/apr-cli/src/model_ops_commands.rs:143,medium apr,apr dry-sampling-lint,1,x86_64-linux,NONE,yes,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:872,high apr,apr embed,6,x86_64-linux,huggingface-transformers,yes,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:1358,medium @@ -195,36 +195,36 @@ apr,apr eval --task mbpp,6,x86_64-linux,NONE,no,8,rag-eval,crates/apr-cli/src/di apr,apr eval --task plan,6,x86_64-linux,NONE,no,8,rag-eval,crates/apr-cli/src/dispatch_analysis.rs:1448 (flag crates/apr-cli/src/extended_commands.rs:145),high apr,apr eval --task verify,6,x86_64-linux,NONE,no,8,rag-eval,crates/apr-cli/src/dispatch_analysis.rs:1445 (flag crates/apr-cli/src/extended_commands.rs:145),high apr,apr experiment view,6,x86_64-linux,pytorch,no,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:1568,low -apr,apr explain,6,x86_64-linux,safetensors,yes,10,apr-core-commands,crates/apr-cli/src/commands_enum.rs:447,high +apr,apr explain,6,x86_64-linux,safetensors,yes,10,apr-core-commands,crates/apr-cli/src/commands_enum.rs:453,high apr,apr explain-token-lint,1,x86_64-linux,NONE,yes,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:1066,low -apr,apr export,6,x86_64-linux;nvidia-cuda,llama.cpp,yes,10,apr-core-commands,crates/apr-cli/src/commands_enum.rs:476,high -apr,apr export --format coreml,6,x86_64-linux;nvidia-cuda,pytorch,no,10,apr-core-commands,crates/apr-cli/src/commands_enum.rs:481,high -apr,apr export --format gguf,6,x86_64-linux;nvidia-cuda,llama.cpp,no,10,apr-core-commands,crates/apr-cli/src/commands_enum.rs:481,high -apr,apr export --format mlx,6,x86_64-linux;nvidia-cuda,huggingface-transformers,no,10,apr-core-commands,crates/apr-cli/src/commands_enum.rs:481,high -apr,apr export --format onnx,6,x86_64-linux;nvidia-cuda,pytorch,no,10,apr-core-commands,crates/apr-cli/src/commands_enum.rs:481,high -apr,apr export --format openvino,6,x86_64-linux;nvidia-cuda,pytorch,no,10,apr-core-commands,crates/apr-cli/src/commands_enum.rs:481,high -apr,apr export --format safetensors,6,x86_64-linux;nvidia-cuda,safetensors,no,10,apr-core-commands,crates/apr-cli/src/commands_enum.rs:481,high +apr,apr export,6,x86_64-linux;nvidia-cuda,llama.cpp,yes,10,apr-core-commands,crates/apr-cli/src/commands_enum.rs:482,high +apr,apr export --format coreml,6,x86_64-linux;nvidia-cuda,pytorch,no,10,apr-core-commands,crates/apr-cli/src/commands_enum.rs:487,high +apr,apr export --format gguf,6,x86_64-linux;nvidia-cuda,llama.cpp,no,10,apr-core-commands,crates/apr-cli/src/commands_enum.rs:487,high +apr,apr export --format mlx,6,x86_64-linux;nvidia-cuda,huggingface-transformers,no,10,apr-core-commands,crates/apr-cli/src/commands_enum.rs:487,high +apr,apr export --format onnx,6,x86_64-linux;nvidia-cuda,pytorch,no,10,apr-core-commands,crates/apr-cli/src/commands_enum.rs:487,high +apr,apr export --format openvino,6,x86_64-linux;nvidia-cuda,pytorch,no,10,apr-core-commands,crates/apr-cli/src/commands_enum.rs:487,high +apr,apr export --format safetensors,6,x86_64-linux;nvidia-cuda,safetensors,no,10,apr-core-commands,crates/apr-cli/src/commands_enum.rs:487,high apr,apr finetune,2,x86_64-linux,unsloth,yes,10,apr-core-commands,crates/apr-cli/src/model_ops_commands.rs:6,high apr,apr finetune --task classify,2,x86_64-linux,unsloth,no,10,apr-core-commands,crates/apr-cli/src/model_ops_commands.rs:48,high apr,apr flow,6,x86_64-linux,NONE,yes,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:668,high apr,apr fp8-lint,1,x86_64-linux,NONE,yes,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:884,high apr,apr gbnf-lint,1,x86_64-linux,NONE,yes,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:1131,high apr,apr gptq-lint,1,x86_64-linux,NONE,yes,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:896,high -apr,apr gpu,6,x86_64-linux,NONE,yes,10,apr-core-commands,crates/apr-cli/src/commands_enum.rs:829,high +apr,apr gpu,6,x86_64-linux,NONE,yes,10,apr-core-commands,crates/apr-cli/src/commands_enum.rs:835,high apr,apr gpu-memtrace-lint,1,x86_64-linux,NONE,yes,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:1102,low apr,apr grad-norm,6,x86_64-linux,NONE,yes,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:1167,high apr,apr hang-trace-lint,1,x86_64-linux,NONE,yes,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:935,high apr,apr hex,3,x86_64-linux,NONE,yes,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:588,high apr,apr imatrix-lint,1,x86_64-linux,NONE,yes,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:1167,high -apr,apr import,6,x86_64-linux,llama.cpp,yes,10,apr-core-commands,crates/apr-cli/src/commands_enum.rs:506,high +apr,apr import,6,x86_64-linux,llama.cpp,yes,10,apr-core-commands,crates/apr-cli/src/commands_enum.rs:512,high apr,apr inspect,3,x86_64-linux;nvidia-cuda,safetensors,yes,10,apr-core-commands,crates/apr-cli/src/commands_enum.rs:224,high apr,apr kernel parity,6,x86_64-linux,NONE,no,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:1472,low apr,apr kv-timeline-lint,1,x86_64-linux,NONE,yes,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:1085,high -apr,apr lint,2,x86_64-linux;nvidia-cuda,safetensors,yes,10,apr-core-commands,crates/apr-cli/src/commands_enum.rs:414,high -apr,apr list,6,x86_64-linux;nvidia-cuda,llama.cpp,yes,10,apr-core-commands,crates/apr-cli/src/commands_enum.rs:596,medium -apr,apr manifest,9,x86_64-linux,NONE,yes,10,apr-core-commands,crates/apr-cli/src/commands_enum.rs:439,medium -apr,apr mcp,1,x86_64-linux,claude-code,yes,10,apr-core-commands,crates/apr-cli/src/commands_enum.rs:773,high -apr,apr merge,6,x86_64-linux,llama.cpp,yes,10,apr-core-commands,crates/apr-cli/src/commands_enum.rs:709,high +apr,apr lint,2,x86_64-linux;nvidia-cuda,safetensors,yes,10,apr-core-commands,crates/apr-cli/src/commands_enum.rs:420,high +apr,apr list,6,x86_64-linux;nvidia-cuda,llama.cpp,yes,10,apr-core-commands,crates/apr-cli/src/commands_enum.rs:602,medium +apr,apr manifest,9,x86_64-linux,NONE,yes,10,apr-core-commands,crates/apr-cli/src/commands_enum.rs:445,medium +apr,apr mcp,1,x86_64-linux,claude-code,yes,10,apr-core-commands,crates/apr-cli/src/commands_enum.rs:779,high +apr,apr merge,6,x86_64-linux,llama.cpp,yes,10,apr-core-commands,crates/apr-cli/src/commands_enum.rs:715,high apr,apr modelfile parse,6,x86_64-linux,NONE,no,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:1590,low apr,apr monitor,9,x86_64-linux,pytorch,yes,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:458,medium apr,apr mono archive,6,UNKNOWN,NONE,no,10,apr-core-commands,crates/apr-cli/src/commands/mono.rs:39,high @@ -254,7 +254,7 @@ apr,apr prune,4,x86_64-linux,unsloth,yes,10,apr-core-commands,crates/apr-cli/src apr,apr ptx,4,x86_64-linux,NONE,yes,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:362,high apr,apr ptx-map,6,x86_64-linux,NONE,yes,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:330,high apr,apr publish,6,x86_64-linux,llama.cpp,yes,10,apr-core-commands,crates/apr-cli/src/tool_commands.rs:46,high -apr,apr pull,4,x86_64-linux,llama.cpp,yes,10,apr-core-commands,crates/apr-cli/src/commands_enum.rs:542,high +apr,apr pull,4,x86_64-linux,llama.cpp,yes,10,apr-core-commands,crates/apr-cli/src/commands_enum.rs:548,high apr,apr pv audit,6,x86_64-linux,NONE,no,6,contracts-pv,crates/aprender-contracts-cli/src/cli.rs:67,low apr,apr pv book,6,x86_64-linux,NONE,no,6,contracts-pv,crates/aprender-contracts-cli/src/cli.rs:373,low apr,apr pv census,6,x86_64-linux,NONE,no,6,contracts-pv,crates/aprender-contracts-cli/src/cli.rs:99,high @@ -356,11 +356,11 @@ pv,pv verify-structure,6,x86_64-linux,NONE,no,6,contracts-pv,crates/aprender-con apr,apr qa,6,x86_64-linux;nvidia-cuda,llama.cpp,yes,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:238,high apr,apr qualify,4,x86_64-linux,NONE,yes,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:686,high apr,apr quant-preservation-lint,1,x86_64-linux,NONE,yes,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:1215,high -apr,apr quantize,4,x86_64-linux,llama.cpp,yes,10,apr-core-commands,crates/apr-cli/src/commands_enum.rs:742,high -apr,apr quantize --scheme fp16,4,x86_64-linux,llama.cpp,no,10,apr-core-commands,crates/apr-cli/src/commands_enum.rs:747,high -apr,apr quantize --scheme int4,4,x86_64-linux,llama.cpp,no,10,apr-core-commands,crates/apr-cli/src/commands_enum.rs:747,high -apr,apr quantize --scheme int8,4,x86_64-linux,llama.cpp,no,10,apr-core-commands,crates/apr-cli/src/commands_enum.rs:747,high -apr,apr quantize --scheme q4k,4,x86_64-linux,llama.cpp,no,10,apr-core-commands,crates/apr-cli/src/commands_enum.rs:747,high +apr,apr quantize,4,x86_64-linux,llama.cpp,yes,10,apr-core-commands,crates/apr-cli/src/commands_enum.rs:748,high +apr,apr quantize --scheme fp16,4,x86_64-linux,llama.cpp,no,10,apr-core-commands,crates/apr-cli/src/commands_enum.rs:753,high +apr,apr quantize --scheme int4,4,x86_64-linux,llama.cpp,no,10,apr-core-commands,crates/apr-cli/src/commands_enum.rs:753,high +apr,apr quantize --scheme int8,4,x86_64-linux,llama.cpp,no,10,apr-core-commands,crates/apr-cli/src/commands_enum.rs:753,high +apr,apr quantize --scheme q4k,4,x86_64-linux,llama.cpp,no,10,apr-core-commands,crates/apr-cli/src/commands_enum.rs:753,high apr,apr rag demo,6,x86_64-linux,huggingface-transformers,no,8,rag-eval,crates/aprender-rag-cli/src/lib.rs:107,low apr,apr rag eval compare,6,UNKNOWN,huggingface-transformers,no,8,rag-eval,crates/aprender-rag-cli/src/lib.rs:455,high apr,apr rag eval gate,6,UNKNOWN,huggingface-transformers,no,8,rag-eval,crates/aprender-rag-cli/src/lib.rs:466,high @@ -378,7 +378,7 @@ apr,apr react-trace-lint,1,x86_64-linux,NONE,yes,1,apr-lint-diag,crates/apr-cli/ apr,apr registry aliases,6,x86_64-linux,llama.cpp,no,10,apr-core-commands,crates/apr-cli/src/commands/registry.rs:12,high apr,apr registry-quota-lint,1,x86_64-linux,NONE,yes,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:1161,high apr,apr rerank,6,x86_64-linux,huggingface-transformers,yes,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:1253,high -apr,apr rm,4,x86_64-linux,llama.cpp,yes,10,apr-core-commands,crates/apr-cli/src/commands_enum.rs:599,medium +apr,apr rm,4,x86_64-linux,llama.cpp,yes,10,apr-core-commands,crates/apr-cli/src/commands_enum.rs:605,medium apr,apr rm-gc-lint,1,x86_64-linux,NONE,yes,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:1186,high apr,apr rosetta chain,6,x86_64-linux,safetensors,no,10,apr-core-commands,crates/apr-cli/src/commands/rosetta.rs:70,low apr,apr rosetta compare-inference,2,x86_64-linux,safetensors,no,10,apr-core-commands,crates/apr-cli/src/commands/rosetta.rs:109,high @@ -392,7 +392,7 @@ apr,apr run,2,x86_64-linux;nvidia-cuda,llama.cpp,yes,10,apr-core-commands,crates apr,apr run --backend cpu,2,x86_64-linux;nvidia-cuda,llama.cpp,no,10,apr-core-commands,crates/apr-cli/src/commands_enum.rs:164 (BACKEND_VALUES crates/apr-cli/src/commands_enum.rs:16),high apr,apr run --backend cuda,4,x86_64-linux;nvidia-cuda,llama.cpp,no,10,apr-core-commands,crates/apr-cli/src/commands_enum.rs:164 (BACKEND_VALUES crates/apr-cli/src/commands_enum.rs:16),medium apr,apr run --backend wgpu,2,UNKNOWN,llama.cpp,no,10,apr-core-commands,crates/apr-cli/src/commands_enum.rs:164 (BACKEND_VALUES crates/apr-cli/src/commands_enum.rs:16),high -apr,apr run --gpu,2,x86_64-linux;nvidia-cuda,llama.cpp,no,10,apr-core-commands,crates/apr-cli/src/commands_enum.rs:794,high +apr,apr run --gpu,2,x86_64-linux;nvidia-cuda,llama.cpp,no,10,apr-core-commands,crates/apr-cli/src/commands_enum.rs:800,high apr,apr run --no-gpu,2,x86_64-linux;nvidia-cuda,llama.cpp,no,10,apr-core-commands,crates/apr-cli/src/commands_enum.rs:90,high apr,apr run --stream,2,x86_64-linux;nvidia-cuda,ollama,no,10,apr-core-commands,crates/apr-cli/src/commands_enum.rs:123,high apr,apr run --task transcribe,2,x86_64-linux;nvidia-cuda,llama.cpp,no,10,apr-core-commands,crates/apr-cli/src/commands_enum.rs:113,high @@ -423,8 +423,8 @@ apr,apr sim run,6,x86_64-linux,NONE,no,11,simulation,crates/aprender-simulate/sr apr,apr sim validate,6,x86_64-linux,NONE,no,11,simulation,crates/aprender-simulate/src/cli/args.rs:92,low apr,apr sim verify,6,x86_64-linux,NONE,no,11,simulation,crates/aprender-simulate/src/cli/args.rs:97,low apr,apr sim version,6,x86_64-linux,NONE,no,11,simulation,crates/aprender-simulate/src/cli/args.rs:119,low -apr,apr stamp,2,x86_64-linux,NONE,yes,10,apr-core-commands,crates/apr-cli/src/commands_enum.rs:628,high -apr,apr tensors,4,x86_64-linux;nvidia-cuda,safetensors,yes,10,apr-core-commands,crates/apr-cli/src/commands_enum.rs:352,high +apr,apr stamp,2,x86_64-linux,NONE,yes,10,apr-core-commands,crates/apr-cli/src/commands_enum.rs:634,high +apr,apr tensors,4,x86_64-linux;nvidia-cuda,safetensors,yes,10,apr-core-commands,crates/apr-cli/src/commands_enum.rs:358,high apr,apr test llm bench,6,x86_64-linux,vllm,no,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:1696 (dispatch crates/apr-cli/src/commands/test_llm.rs:482),medium apr,apr test llm bench --band,6,x86_64-linux,vllm,no,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:1831 (protocol crates/apr-cli/src/commands/test_llm_band.rs:348),medium apr,apr test tensor,4,x86_64-linux,NONE,no,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:1648,high @@ -436,7 +436,7 @@ apr,apr tokenize plan,6,x86_64-linux,huggingface-transformers,no,10,apr-core-com apr,apr tokenize repair-manifest,6,x86_64-linux,huggingface-transformers,no,10,apr-core-commands,crates/apr-cli/src/tokenize_commands.rs:239,low apr,apr tokenize train,6,x86_64-linux,huggingface-transformers,no,10,apr-core-commands,crates/apr-cli/src/tokenize_commands.rs:63,low apr,apr tool-use-lint,1,x86_64-linux,NONE,yes,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:1125,high -apr,apr trace,6,x86_64-linux,safetensors,yes,10,apr-core-commands,crates/apr-cli/src/commands_enum.rs:370,high +apr,apr trace,6,x86_64-linux,safetensors,yes,10,apr-core-commands,crates/apr-cli/src/commands_enum.rs:376,high apr,apr train apply,1,x86_64-linux,pytorch,no,10,apr-core-commands,crates/apr-cli/src/train_commands.rs:76,high apr,apr train apply --task classify,1,x86_64-linux,pytorch,no,10,apr-core-commands,crates/apr-cli/src/train_commands.rs:88,high apr,apr train apply --task pretrain,1,x86_64-linux,pytorch,no,10,apr-core-commands,crates/apr-cli/src/train_commands.rs:88,high @@ -450,13 +450,13 @@ apr,apr train submit,6,x86_64-linux,pytorch,no,10,apr-core-commands,crates/apr-c apr,apr train sweep,6,x86_64-linux,pytorch,no,10,apr-core-commands,crates/apr-cli/src/train_commands.rs:201,high apr,apr train watch,6,x86_64-linux,pytorch,no,10,apr-core-commands,crates/apr-cli/src/train_commands.rs:172,low apr,apr tree,4,x86_64-linux,NONE,yes,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:666,high -apr,apr tui,9,x86_64-linux,NONE,yes,10,apr-core-commands,crates/apr-cli/src/commands_enum.rs:810,medium +apr,apr tui,9,x86_64-linux,NONE,yes,10,apr-core-commands,crates/apr-cli/src/commands_enum.rs:816,medium apr,apr tune,4,x86_64-linux,pytorch,yes,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:384,high apr,apr typical-p-lint,1,x86_64-linux,NONE,yes,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:1137,high apr,apr unified-search-lint,1,x86_64-linux,NONE,yes,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:1190,high apr,apr unshard,9,x86_64-linux,llama.cpp,yes,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:1229,medium -apr,apr validate,2,x86_64-linux;nvidia-cuda,safetensors,yes,10,apr-core-commands,crates/apr-cli/src/commands_enum.rs:280,high -apr,apr validate-manifest,9,x86_64-linux,NONE,yes,10,apr-core-commands,crates/apr-cli/src/commands_enum.rs:299,medium +apr,apr validate,2,x86_64-linux;nvidia-cuda,safetensors,yes,10,apr-core-commands,crates/apr-cli/src/commands_enum.rs:286,high +apr,apr validate-manifest,9,x86_64-linux,NONE,yes,10,apr-core-commands,crates/apr-cli/src/commands_enum.rs:305,medium apr,apr zram benchmark,6,x86_64-linux,NONE,no,8,rag-eval,crates/aprender-zram-cli/src/lib.rs:47,low apr,apr zram create,6,x86_64-linux,NONE,no,8,rag-eval,crates/aprender-zram-cli/src/lib.rs:38,low apr,apr zram remove,6,x86_64-linux,NONE,no,8,rag-eval,crates/aprender-zram-cli/src/lib.rs:41,low diff --git a/docs/findings/2026-09-26-aprender-fb.jsonl b/docs/findings/2026-09-26-aprender-fb.jsonl new file mode 100644 index 0000000000..bd7396a72e --- /dev/null +++ b/docs/findings/2026-09-26-aprender-fb.jsonl @@ -0,0 +1,2 @@ +{"id":"FND-20260926-serve-qwen35-gpu-no-timings","title":"apr serve omits the server timings block on the Qwen3.5 hybrid CUDA chat path, so replay TTFT/prefill/decode are null for apr","evidence":"PRM-S1 review-replay-v1 set b8a58011 on gx10 (GB10, gpu-layers resolved=32/32 backend=cuda, server log 'qwen35 hybrid forward, #3090'), apr 0.69.5-rc.1 (5660b0877), Qwen3.5-4B-Q4_K_M sha 00fe7986: 200/200 /v1/chat/completions replies have usage (input/output tokens) but no timings, so prompt_ms, prompt_tps and decode_tps are null in all 200 rows. llama.cpp d1d3c3396 on the same set and cell returns timings on 200/200. Timings::from_phases exists and cuda_chat_backend.rs builds PhaseTimings; the hybrid path apparently does not call it (suspected, not traced)","repro":"apr serve run Qwen3.5-4B-Q4_K_M.gguf --gpu-layers all --context-length 36864; POST /v1/chat/completions non-streaming; the reply JSON has no timings key","suspected_epic":"PRM-001 (#4354)","severity":"P2","found_at_sha":"5660b08775c1115e9951960638e54a6d83baabd6"} +{"id":"FND-20260926-serve-qwen35-gpu-4x-slower-wall","title":"apr serve on the Qwen3.5-4B hybrid CUDA path is 4.6x slower wall-clock than llama.cpp d1d3c3396 on identical prompt ids (PRM-S1 v2)","evidence":"PRM-S1 review-replay-v2 set a4910a65 (200 items, enable_thinking=false both sides) on gx10 GB10, Qwen3.5-4B-Q4_K_M sha 00fe7986, apr 0.69.3 @96eb79f51 --gpu-layers all vs llama.cpp d1d3c3396 -ngl 99, ctx 36864, greedy seed 4354 max_tokens 512. Pre-run gate: prompt-ids sha llama=apr=66ef3684, ids-diff differ 0/200; input tokens 1,941,322 both. Wall p50 apr 26.84 s vs llama 5.85 s (4.59x); wall sum 6008 s vs 1248 s; wall per output token 131.8 ms vs 28.2 ms (4.68x; output tokens 45,573 vs 44,275). Peak RSS apr 8419 MB vs 9566 MB (0.88x). parse_rate 1.0; verdict agreement 195/200. apr server TTFT/prefill/decode are null (FND-20260926-serve-qwen35-gpu-no-timings), so the prefill-vs-decode split is not measured; llama prefill 3.7-4.1k tok/s, decode 63.5 tok/s.","repro":"replay build --version review-replay-v2; replay prompt-ids per engine; replay ids-diff (exit 0); replay run --ids per engine; replay summary (set a4910a65, speed-block prompt_ids_sha f3c20136)","suspected_epic":"PRM-001 (#4354)","severity":"P1","found_at_sha":"96eb79f510707c3ee30178272f823b555e356b69"} diff --git a/docs/roadmaps/entries/PMAT-4354.yaml b/docs/roadmaps/entries/PMAT-4354.yaml new file mode 100644 index 0000000000..ad899bd56c --- /dev/null +++ b/docs/roadmaps/entries/PMAT-4354.yaml @@ -0,0 +1,18 @@ +- id: PMAT-4354 + github_issue: 4354 + item_type: task + kind: code + title: 'PRM-L: land PRM-001 v3 (PROMETHEUS) + rex/001 to main' + status: in_progress + priority: high + assigned_to: aprender-84 + created: 2026-09-25T18:38:33Z + updated: 2026-09-25T18:38:33Z + spec: docs/specifications/PRM-001-prometheus.md + acceptance_criteria: + - Refs paiml/aprender#4354. Spec PRM-001 v3 byte-exact, rex-prereg-v2 lock, v1 lock frozen, superseded banners, v3 figure. + phases: [] + subtasks: [] + estimated_effort: null + labels: [] + notes: null diff --git a/docs/roadmaps/entries/PMAT-4446.yaml b/docs/roadmaps/entries/PMAT-4446.yaml new file mode 100644 index 0000000000..3323407258 --- /dev/null +++ b/docs/roadmaps/entries/PMAT-4446.yaml @@ -0,0 +1,16 @@ +- id: PMAT-4446 + github_issue: 4446 + item_type: task + title: PRA-001 T12 terms/provider tagging (P-TERMS) + status: planned + priority: medium + assigned_to: null + created: 2026-09-25T17:08:31Z + updated: 2026-09-25T17:08:31Z + spec: null + acceptance_criteria: [] + phases: [] + subtasks: [] + estimated_effort: null + labels: [] + notes: null diff --git a/docs/roadmaps/entries/PMAT-4447.yaml b/docs/roadmaps/entries/PMAT-4447.yaml new file mode 100644 index 0000000000..d62f697fd3 --- /dev/null +++ b/docs/roadmaps/entries/PMAT-4447.yaml @@ -0,0 +1,16 @@ +- id: PMAT-4447 + github_issue: 4447 + item_type: task + title: PRM-S1 review-replay-v1 benchmark + runner + status: planned + priority: medium + assigned_to: null + created: 2026-09-25T17:46:46Z + updated: 2026-09-25T17:46:46Z + spec: null + acceptance_criteria: [] + phases: [] + subtasks: [] + estimated_effort: null + labels: [] + notes: null diff --git a/docs/roadmaps/entries/PMAT-4524.yaml b/docs/roadmaps/entries/PMAT-4524.yaml new file mode 100644 index 0000000000..ab1b5ad5e7 --- /dev/null +++ b/docs/roadmaps/entries/PMAT-4524.yaml @@ -0,0 +1,18 @@ +- id: PMAT-4524 + github_issue: 4524 + item_type: task + title: 'PRM-L land rex/001 C6+C11+C14 on main: merge main, reconcile with #4655 crate snapshot' + status: planned + priority: high + assigned_to: null + created: 2026-10-08T12:49:26Z + updated: 2026-10-08T12:49:26Z + spec: null + acceptance_criteria: [] + phases: [] + subtasks: [] + estimated_effort: null + labels: + - kind:code + notes: null + epic: 4354 diff --git a/docs/roadmaps/roadmap.yaml b/docs/roadmaps/roadmap.yaml index 0bd9fdbcec..659b1d91a1 100644 --- a/docs/roadmaps/roadmap.yaml +++ b/docs/roadmaps/roadmap.yaml @@ -23710,6 +23710,24 @@ Not done, pending the ruling: moving the 31 fixture files to an obviously synthe labels: - kind:code notes: 'DPO x3 by pvl-b; NF4 x3 and f16 by aprender-57. The old dpo_gradient_formula axiom was false for beta < 0; now a HasDerivAt theorem plus dpo_gradient_scale_nonpos (0 <= beta).' +- id: PMAT-4354 + github_issue: 4354 + item_type: task + kind: code + title: 'PRM-L: land PRM-001 v3 (PROMETHEUS) + rex/001 to main' + status: in_progress + priority: high + assigned_to: aprender-84 + created: 2026-09-25T18:38:33Z + updated: 2026-09-25T18:38:33Z + spec: docs/specifications/PRM-001-prometheus.md + acceptance_criteria: + - Refs paiml/aprender#4354. Spec PRM-001 v3 byte-exact, rex-prereg-v2 lock, v1 lock frozen, superseded banners, v3 figure. + phases: [] + subtasks: [] + estimated_effort: null + labels: [] + notes: null - id: PMAT-4369 github_issue: 4369 item_type: task @@ -23844,6 +23862,38 @@ Not done, pending the ruling: moving the 31 fixture files to an obviously synthe estimated_effort: null labels: [] notes: 'Adopted from aprender-ea orphan fix/4445-bench-fresh-prefill @65fd973170 (09-25, never PRed). The bug exists only with the #4214 checkpoint, i.e. car/0.70.0 (#4429); main before #4429 does not reuse an identical prompt. Cop ruling 2026-09-27: not folded into the car; own PR on main the moment #4429 merges, before the rc.1 tag. Superseded 2026-09-27 ~14:00Z by operator "make it faster" (via cop): open on main NOW, in parallel with the car. Until the car lands this change is inert on main (an identical prompt already prefills whole there); it pre-lands the bench wiring and the API so the car merge needs only the checkpoint line, which the exhaustive destructure forces.' +- id: PMAT-4446 + github_issue: 4446 + item_type: task + title: PRA-001 T12 terms/provider tagging (P-TERMS) + status: planned + priority: medium + assigned_to: null + created: 2026-09-25T17:08:31Z + updated: 2026-09-25T17:08:31Z + spec: null + acceptance_criteria: [] + phases: [] + subtasks: [] + estimated_effort: null + labels: [] + notes: null +- id: PMAT-4447 + github_issue: 4447 + item_type: task + title: PRM-S1 review-replay-v1 benchmark + runner + status: planned + priority: medium + assigned_to: null + created: 2026-09-25T17:46:46Z + updated: 2026-09-25T17:46:46Z + spec: null + acceptance_criteria: [] + phases: [] + subtasks: [] + estimated_effort: null + labels: [] + notes: null - id: PMAT-4453 github_issue: 4453 item_type: task @@ -24022,6 +24072,24 @@ Not done, pending the ruling: moving the 31 fixture files to an obviously synthe estimated_effort: null labels: [] notes: null +- id: PMAT-4524 + github_issue: 4524 + item_type: task + title: 'PRM-L land rex/001 C6+C11+C14 on main: merge main, reconcile with #4655 crate snapshot' + status: planned + priority: high + assigned_to: null + created: 2026-10-08T12:49:26Z + updated: 2026-10-08T12:49:26Z + spec: null + acceptance_criteria: [] + phases: [] + subtasks: [] + estimated_effort: null + labels: + - kind:code + notes: null + epic: 4354 - id: PMAT-4594 github_issue: 4594 item_type: task diff --git a/docs/specifications/PRM-001-prometheus.md b/docs/specifications/PRM-001-prometheus.md new file mode 100644 index 0000000000..04938e22dc --- /dev/null +++ b/docs/specifications/PRM-001-prometheus.md @@ -0,0 +1,441 @@ +# PRM-001 — PROMETHEUS: make local Qwen 3.5 fast enough, and good enough, to break quorum ties + +**Spec id:** `PRM-001` v3 (Prometheus). **Supersedes:** PRM-001 v2, REX-001 / PRM-001 v1 (experiment protocol), and PRA-001 (trace-corpus design). Old row ids are kept in the §7 `was` column. +**Target:** `paiml/aprender` → `docs/specifications/PRM-001-prometheus.md`. This is the single source of truth. +- The infra copy becomes a one-line pointer. +- The v1 file `review-experiment-protocol.md` stays byte-frozen in §2–§5, so its v1 lock still verifies. It gains one superseded banner above §0. + +**Runner:** the aprender traffic cop, sole minter. Tracking is one sub-epic under E4 (#4381) per APR-EPIC-001 §4d, with its phases replaced by §7 here. Epic #4354 (rows #4355–#4367) is collapsed into its checklist. +**Launch:** from `~/src/aprender`, run `Implement docs/specifications/PRM-001-prometheus.md autonomously.` +**Figure:** `docs/specifications/figures/prometheus-flow.png`. Re-export it: title "PROMETHEUS", remove the silver pool, and add the tie-breaker rung. +**Related:** +- E2 Verbs Are Fast (0.71) and E6 llama.cpp Parity (0.73): Prometheus is their real-workload acceptance test. +- ARB-APR-001, ARB-SELF-001, EXT-001, FLOW-001, APR-PERF-GATE-001 (J3 receipts). + +**Provenance marks:** `[V]` verified · `[C]` computed · `[A]` asserted · `[O]` operator ruling · `[U]` unmeasured · `[X]` third-party. No invented thresholds. + +--- + +## §0 Operating assumptions + +1. **One goal.** Qwen 3.5 4B on `apr` becomes the **4th-lane tie-breaker**. That needs three things: + - **fast:** it never lengthens a quorum; + - **available:** it answers ≥ 95% of rounds; + - **measured:** its accuracy on sealed gold is at least that of the voters it would overrule. + Speed is necessary; it is not sufficient. +2. **Speed is the critical path.** Review is prefill-heavy: a diff of 2k–32k tokens goes in, and a verdict plus at most 5 bullets comes out `[U until PRM-S2]`. The binding constraints are prefill and TTFT, not decode. +3. **The shadow has zero weight until promoted** `[O 2026-09-25]`. + - Every quorum receipt carries an apr row (`Verdict | Refused | NotRun{…}`); a receipt without one is RED. + - The apr verdict never blocks, breaks a tie or escalates until §5.4 grants a rung. +4. **Hosted-model outputs are never training targets** `[O 2026-09-25]`. + - Claude and agy/Gemini outputs are captured and stored (you own your outputs). They are used only as **replay inputs, latency baselines and evaluation comparators**. + - Training uses gold labels plus locally generated rows only: Qwen's own outputs and the local 27B teacher. + - This makes the program clean under Anthropic's help-center rule on Outputs, Anthropic Consumer §3(2) and Commercial §D.4, and the Google ToS ban on "using AI-generated content … to develop machine learning models" (§1 G19). It replaces the S-6 ruling. +5. **Capture first.** A quorum whose I/O is not captured at dispatch is lost for replay, evaluation and outcome joins. The order is capture → measure → speed → promote. +6. **A scientific experiment.** §2–§5 are frozen by the pre-registration lock `rex-prereg-v2` (PRM-00b), which locks **this v3**, before any measurement. v1 collected 0 data rows `[V]`. +7. **Producer is never the gate.** + - Released, sha-verified `apr` tags and sha256-pinned weights only. + - The writer of a trace never labels it. +8. **A refusal is a result.** `Refused{removed_by}` is recorded and filed, never retried under the same cell name. +9. **Genchi genbutsu.** Re-measure §1 at HEAD before relying on it. A false premise is recorded as `premise-falsified`. +10. **Foreign work is transferred.** + - Capture at dispatch → paiml-implement #436. + - Executors, the almacen store, the second copy and transcript retention → paiml/infra. + - Speed defects are aprender issues in E2/E6; Prometheus links them and does not own them. +11. **The operator has no pre-steps.** Pure Rust + bashrs-clean shell, no Python; provisioning by `forjar apply` or make only. + +--- + +## §1 Ground truth (frozen baseline; never quote as current) + +| # | Fact | Value | Mark | Source | +|---|---|---|---|---| +| G1 | n=1 pilot | 4B Q4_K_M, apr v0.69.3, lambda CPU at load 33–40: **175.8 s**; Haiku 4.5: **17.8 s** (≈ 10×) | [V] | diff `0c6932fd5` | +| G2 | Recall and precision of every lane, the voters included | unknown | [C] | — | +| G3 | apr v0.69.3 sha256 | `8a67a0103cbb036332908cc184d5a8a8ae422fb9025bd5c78e42d65df86cfbd7` | [V] | REX-04 receipt | +| G4 | Weights sha256 | 4B `00fe7986…69ef11a4` · 9B `03b74727…2b7e8` · 27B `84b5f7f1…d2a806` | [V] | REX-01 receipt | +| G5 | Train themes | 0.70 fast (`--json-schema`, resident serve) · 0.71 wider + training (qwen35 finetune/distill; WGPU/Metal) | [O 2026-09-20] | `06x-release-schedule.md` | +| G6 | `apr parity` v0.69.3 | GPU-vs-CPU only; no released llama.cpp comparison | [V] | REX-04 premise check | +| G7 | rc.1 lane on gx10 | F2 guard (cos 0.8425) → CPU fallback (#4313); `--context-length` ignored, 262144 used (#4443); decode and **32k prefill** both slower than ollama (figures in #4313, #4443); temperature-0 divergence at ≈ 55 tokens | [V] cop report | #4313, #4443, `evidence/4252-promotion.md` | +| G8 | TTFT at 0.68.2 on a 4090 | slower than llama.cpp, TTFT and end-to-end (figures in #3596) | [V] | #3596 | +| G9 | Epic speed targets | E2 TTFT target ≤ 2× llama.cpp · E6 target ≥ 1.0× llama.cpp for decode, prefill and TTFT across the certified matrix | [O] | APR-EPIC-001 §2 | +| G10 | Executors | apr pin on gx10, lambda-labs, yoga; no `apr serve` unit, weights or review label anywhere | [V] | infra#1088 | +| G11 | paiml-implement | v1.8.1 has no advisory lane; #420 unreleased | [V] | PRM report | +| G12 | Ledger | findings stored as a count; `weights_sha256: None` hardcoded; free-text `LaneUnavailable` | [V] | `crates/aprender-review-experiment/src/ledger.rs` | +| G13 | Sealed corpus | `review-corpus-v1@787d2026256cc08b`: test = 35 P (size S, mutants) + 35 R + 35 G | [V] | REX-02 receipt | +| G14 | Corpus diffs | single copy on a 94%-full raid (`items-v1.tar`, sha `70dffe56…8bba`) | [V] | REX-02 receipt | +| G15 | Branch | `rex/001` @ `da7b6386f` folded into **closed** #4317; nothing on `main` | [V] | review package README | +| G16 | Volume | 50–200 quorums/day; ≤ ~0.45 TB/yr | [O] / [U] | PRM-C14 | +| G17 | Storage | almacen 3×16 TB RAID 5; re-pull time and scrub policy unmeasured | [V] / [U] | `almacen-storage-integration.md` | +| G18 | Transcripts | Claude Code deletes after `cleanupPeriodDays` (default 30) | [X] | Claude Code docs | +| G19 | Terms | Anthropic: training allowed only for non-competing specialised tools; "using Outputs as training targets" is prohibited. Google ToS: flat ban on developing ML models from AI-generated content. Claude Code under Max = Consumer Terms. **Not legal advice.** | [X] | support.claude.com #12326764; anthropic.com/legal; policies.google.com/terms | + +--- + +## §2 Design + +### §2.1 Lane dispatch and states + +Order `[O 2026-09-25, R-9 option b]`: `gx10-cuda → lambda-cuda → intel-wgpu → mini-metal → intel-cpu → NotRun{NoExecutor}`. + +- **lambda-cuda is rung 2 and shadow only.** + - Dispatch time only, and only when rung 1 returned NotRun **and** `/tmp/apr-gpu.lock` is free. + - Never a resident serve, VRAM preload, experiment cell, teacher or finetune. + - If the lock is taken mid-round, the row is `NotRun{Busy}` and the ladder continues. + - **Once the lane holds tie-breaker or vote, lambda-cuda leaves the ladder.** A deciding lane runs only on declared, non-contended cells (§4.1). +- **Rows record the backend that actually ran** (`gx10-cpu` on the F2 fallback). +- **train-active falls through; it never blocks.** +- **One typed state enum**, shared by receipt-lint (#436) and the ledger: `Verdict{PASS|FAIL}` · `Refused{removed_by}` · `NotRun{NoExecutor|Busy|Timeout|ContextOverflow|TrainActive}`. +- **Timeout per rung** = 2 × that cell's p95 once it has ≥ 10 rows; until then 300 s `[U]`. + +### §2.2 Capture (`agent-trace-v1`): the substrate for replay, evaluation and outcomes + +- **What:** a `TraceSpan` in the quorum rail (#436) hashes the exact wire bytes and streams the full reply into a blob, for every lane of every round. +- **On failure:** `trace_status = capture_failed` raises an andon. The quorum never waits on capture. + +| Field group | Contents | +|---|---| +| identity | `trace_id` (ULID), `quorum_id`, `round`, `lane`, `counted`, `provider`, exact `model_id` or weights sha, `access_channel`, `apr_tag`, `served_by`, `lane_state` | +| work unit | `repo`, `pr`, `head_sha`, `base_sha`, `diff_sha256`, `input_sha`, `input_parts`, `prompt_version` | +| result | `output_sha` (full, never truncated), `verdict`, `findings[{file,line_start,line_end,severity,category,text}]`, `parse_status` | +| **timing (first-class)** | `tokens{input,output,cached?}`, `latency_ms{queue,ttft,prefill,decode,total}`, `prefill_tps`, `decode_tps`, `load_snapshot` | +| local only | `logits_sha` → `sparse-logits-v1` (k=20 `[A]` + `logsumexp_full` + `topk_mass`); used only for local teacher → student | +| lifecycle | `outcome` ∈ {pending, merged, reverted_le14d, regression_escape, closed_unmerged}; `split_guard{repo#PR, split, dedup_cluster}`; `secret_scan`; `train_eligible` (computed by admission, §4.2) | + +- Second stream: Claude Code transcripts (including `subagents/`) and agy logs, harvested hourly `[A]`. +- These are stored for audit and replay only. They are never training data (§0.4). + +### §2.3 Storage + +``` +almacen:/traces/blobs/sha256/ab/cd/.zst write-once; address = sha of uncompressed bytes +almacen:/traces/index/agent-trace-v1/YYYY/MM/DD.jsonl append-only +almacen:/traces/transcripts/{claude-code,agy}//.zst +almacen:/traces/replay/review-replay-vN.jsonl frozen replay sets (§2.4) +almacen:/rex-corpus/v1/items-v1.tar the sealed corpus leaves the 94% raid +``` + +- Keep forever; quarantine only, never delete. +- **Second copy:** nightly, incremental by sha, to separate media. +- **Weekly restore drill:** 100 random blobs `[A]`, 0 mismatches. RAID 5 is not a backup `[C: P(≥1 URE) ≈ 0.92 on a degraded 32 TB read at 10⁻¹⁴]`. +- **Transcript retention:** a forjar-managed Claude Code setting, `cleanupPeriodDays: 36500` `[A]`. + +### §2.4 Replay benchmark (`review-replay-v1`): the speed instrument + +- **Source:** real quorum diffs from the trace index. Inputs only; no hosted output enters the benchmark. +- **Stratification:** by input tokens, 2k / 8k / 16k / 32k. +- **Set size:** 200 items per version `[A]`. Frozen per version; a new version each train. +- **Exclusions:** sealed-test hashes and clusters. +- **Run on every `apr` tag,** on the primary cell and on llama.cpp `d1d3c3396` with the same GGUF and cell. Output is `review-replay-receipt-v1` rows. +- **Measures:** review wall-clock p50/p95, TTFT, prefill tok/s per stratum, decode tok/s, peak anon memory, parse rate, and the verdict-identity rate vs the prior tag. +- **Queue budget reference:** the p95 of the **slowest counted voter on the same diffs**, taken from captured `latency_ms.total`. The speed target is measured, not asserted. +- **Workload profile (PRM-S2):** the measured distribution of input and output tokens per lane. It confirms or falsifies §0.2 and weights the strata. +- **Consumers:** this receipt is the **acceptance workload for E2 and E6**. Their `done_when` numbers are read on replay-v1, not on synthetic pp512/tg128 alone. + +### §2.5 Sealed evaluation corpus (`review-corpus-v1`) + +- As built (G13). +- At most 20 promotion decisions per sealed version `[A]`, then 50% rotation. +- Escapes feed the next version, never test v1. +- `review-corpus-contamination-v1` covers exact hashes and near-dup clusters. + +### §2.6 Experiment cells and arms + +| Cell | Host | Backend | Notes | +|---|---|---|---| +| C1 `intel-wgpu` | intel | wgpu | `Refused` expected before 0.71; must be observed | +| C2 `intel-cpu` | intel | cpu | **reference cell** | +| C3 `lambda-cpu` | lambda-labs | cpu | the GPU is never an experiment cell | +| C4 `gx10-cuda` | gx10 | cuda sm_121 | primary candidate; `gx10-cpu` on F2 fallback | +| C5a/C5b | mini | cpu / metal | C5b `Refused` expected before 0.71 | + +- **Arms:** 4B primary; 9B control (test split, reference cell plus top cell). +- **Evaluation comparators on the reference cell:** Haiku 4.5 and agy, exact ids. Evaluation only; no training. +- **Fixed:** prompt sha, greedy decoding, `max_tokens`. Overflow is `NotRun{ContextOverflow}`, never truncated. + +### §2.7 Metrics (deterministic; no model judges) + +| Metric | Definition | +|---|---| +| **Review wall-clock p50/p95** | request → parsed verdict, per item; primary speed metric | +| **TTFT; prefill tok/s by stratum; decode tok/s** | from `apr` timing and the serve metadata | +| **Speed ratio** | apr ÷ llama.cpp on the same cell and GGUF, per metric | +| Availability | Verdict rows ÷ rounds, rolling 7 d, by cause | +| Parse rate | parsed ÷ attempted | +| Recall, per class | FAIL on P ÷ \|P\|; FAIL on R ÷ \|R\| (reported separately) | +| False-refute rate | FAIL on G ÷ \|G\| (the HRQ cost; does not depend on prevalence) | +| Precision | descriptive | +| Localization | a FAIL finding names the defect's file | +| Cross-cell invariance | verdict identity with the reference cell (bytes are descriptive) | +| Determinism | byte-identical on a 10% same-cell re-run | +| **κ_err** | Cohen's κ on binary error indicators vs gold, for every lane pair (voter–voter and qwen–voter) | +| Interference | slowdown of a reference workload when the serve is co-located vs alone | + +--- + +## §3 Pre-registered hypotheses (locked by `rex-prereg-v2`; Holm family H1–H6, α = 0.05) + +| # | Hypothesis | Test | Consequence | +|---|---|---|---| +| H1 | **Verdict invariance** across admitted cells vs reference | count of divergent verdicts > 0 rejects | the cell is inadmissible; one issue per item | +| H2 | **Determinism** on the same cell | byte count on the 10% re-run > 0 rejects | the cell is inadmissible until fixed | +| H3 | **Size:** 9B correctness exceeds 4B | McNemar exact, one-sided | if rejected, §5.4 weighs 9B's gain against its wall-clock | +| H4 | **Precision ≥ min(Haiku, agy)** | paired bootstrap of `p4B − min(pH,pA)`, **min inside each resample**, 10k resamples, seed 4354; Holm-adjusted one-sided p decides; CI reported only | the tripwire gate | +| H5 | **R-class recall ≥ the Claude lane** | same machinery, R items only | the tie-breaker gate (with H4) | +| H6 | **Interference** | 5 runs alone vs 5 co-located; slowdown > 2 × sd(alone) rejects | a rejected cell ranks below every other cell | +| H7 | **Independence** (outside the Holm family; a gate) | κ_err(qwen, each voter) ≤ **max κ_err over voter–voter pairs** on the same items | the tie-breaker gate. It is δ-free: a tie-breaker may not be more correlated with a voter than the voters already are with each other | + +- **Sensitivity:** all runs (primary) vs loadavg1 < 0.5 × physical cores (secondary). Decisions read the primary column. +- **Sample size:** after PRM-05, if the Wilson half-width on R-recall > 0.12, grow R; hypotheses unchanged. + +## §4 Decision rules + +### §4.1 Hardware (lexicographic) + +1. **Admissible:** a parity receipt passes against its declared `parity.oracle` (`llama.cpp@d1d3c3396` once it is a declared input; until then `apr-parity-gpu-cpu` plus H1; `ds.yaml` min_cosine 0.98); H1 and H2 hold; the parse rate equals the reference cell's; the executor is forjar-declared; for tie-breaker or vote, the cell is **not** lambda-cuda. +2. Rank by replay p95 under natural load. +3. H6-rejected cells rank last. +4. Ties go to the larger 7-day idle fraction, then to a host outside the clean-room pool. +5. **Queue budget:** replay p95 ≤ the slowest voter's p95 on the same diffs (§2.4). A cell failing it is shadow-only. +6. **Output:** a primary cell and a failover cell. When both are down: `NotRun{NoExecutor}`. The quorum keeps width 3; a tie then goes to HRQ, as today. +7. Committed as `docs/audits/prm-001/hardware-ruling.md`. + +### §4.2 Admission (`trace-admit`, a separate binary) + +| Gate | Rule | Target | +|---|---|---| +| **G-PROV** | `train_eligible` only if provider = local (qwen, teacher-27b) **or** the row is a gold label (outcome, HRQ ruling, sealed-corpus label). Hosted outputs are never eligible | hosted rows in any training pool = **0** `[O]` | +| G-SEC | Rust scanner plus the gitleaks ruleset ported as data; a hit quarantines the row, with no redaction | 0 hit rows in a pool; 100% canary recall `[A]` | +| G-CON | 0 sealed hashes or near-dup clusters in a pool or replay set | 0 `[A]` | +| G-ID | provider, exact model id, access_channel, input/output sha, served_by, weights sha all present | 100% `[A]` | +| G-DUP | split by `repo#PR`, time-ordered, ≥ 14 d embargo, clusters closed to their earliest member | 0 cross-split clusters `[A]` | +| G-MAT | gold only after the 14 d outcome window; pending is never a negative | 0 immature gold `[A]` | +| G-HASH | blob re-hash equals its address | 0 mismatches `[A]` | + +- **Gold** = sealed-corpus labels, matured outcomes, and HRQ rulings (from tripwire onward). +- **Public release** (EXT-001) = base weights plus artifacts trained on G-PROV-eligible rows only. + +--- + +## §5 Improvement loop and promotion + +### §5.1 Speed loop (first priority; every aprender tag) + +| Lever | Owner | Expected effect | +|---|---|---| +| Keep the GPU path on gx10 (F2 guard) | #4313 | CPU → GPU; the largest single step | +| Honour `--context-length` | #4443 | right-sized KV | +| Prefill kernels and batched prefill | E2 → E6 | the dominant term (§0.2) | +| Resident `apr serve` plus a prefix cache for the fixed system prompt | E2 | TTFT | +| `--json-schema` | 0.70 | bounded output; parse rate → 1.0 | +| Knob sweep (threads; Q4_K_M vs Q8_0; KV) once per train | PRM-10 | per-cell optimum; a new quant goes through §5.4 | + +- **Ratchet:** replay p95 on the primary cell must never rise after 3 tags. A +10% planted regression turns RED (paired Harrell–Davis p95, landed). +- Every tag files one aprender issue per stratum where the apr/llama.cpp ratio worsened. + +### §5.2 Quality loop (no hosted-output targets) + +| Tier | Method | Available | +|---|---|---| +| Q1 | prompt versions | now | +| Q2 | retrieval few-shot from **gold** train-pool specimens (never the test split) | now | +| Q3 | constrained decoding (`--json-schema`) | 0.70 | +| Q4 | **local logit distillation:** Qwen3.5-27B on gx10 writes `sparse-logits-v1` over G-PROV-eligible inputs; the 4B student trains with forward-KL plus tail-mass correction; reverse-KL/GKD as challengers, using Qwen's own shadow outputs as on-policy samples | qwen35 `apr distill`, per D-7 | +| Q5 | QLoRA on gold labels only (`apr finetune`); merge (`apr merge`); re-quantize (`apr quantize`, now) | per D-7 / now | + +- Q4/Q5 are aprender acceptance tests: a qwen35 distill/finetune ticket is done only when its challenger passes §5.4. +- Until then the row reads `NotRun{VerbRefused(removed_by)}`. + +### §5.3 Champion/challenger (pre-registered) + +A challenger replaces the champion only if all of these hold on the current sealed version: +1. McNemar exact, one-sided, α 0.05: correctness improves. +2. The false-refute rate is not higher. +3. H1/H2 hold on the primary cell. +4. H7 holds. +5. G-CON and G-PROV hold on its training data. +6. Replay p95 meets the §4.1.5 queue budget. + +- Promotion is a forjar pin bump (weights and adapter sha); rollback is the previous sha. Each decision uses 1 of the 20. +- **Demotion:** 2 consecutive gold escapes where the champion's verdict decided the round → automatic HRQ and re-evaluation. + +### §5.4 Promotion ladder (`rex ladder`, contract `rex-promotion-ladder-v1`, extended by PRM-09) + +| Rung | Authority | Speed gate | Quality gate (sealed gold) | +|---|---|---|---| +| **shadow** (default; anything unreadable) | none | — | — | +| **tripwire** | a dissent against a 3–0 opens an HRQ, capped at D-8/day | replay p95 ≤ queue budget | H4 | +| **tie-breaker** | decides only a **1–1** split (a counted lane NotRun, or a 2-counted quorum) | queue budget **and** availability ≥ 95% over 7 d `[O cop crit. 4]`, on a non-lambda cell | H4 **and** H5 **and** H7, with H1/H2 on the primary cell | +| **vote** | counted voter | same | same, plus ≥ 20 decided tie-breaks `[A]` whose 14 d outcome agreement ≥ the voters' agreement on the same rounds | + +- The ladder reads only `prm-001-report-v1` and §5.3 reports carrying the `rex-prereg-v2` sha. +- It never skips a rung. +- A demotion drops exactly one rung. + +--- + +## §6 Hard rules (each has a §9 falsifier) + +- **R-1 Pre-registration is immutable.** `rex-prereg-v2` locks v3 §2–§5, `stats.rs`, analysis plan v2 and prompt v1. Any change is a new version, and earlier data is `exploratory`. +- **R-2 Sealed test.** Only the scorer reads it; it never appears in replay sets or pools. +- **R-3 Released artifacts only.** apr tag and weights by sha256. +- **R-4 Declared executors only.** No ad-hoc SSH or hand-installed apr. +- **R-5 Identity.** 0 `unknown` identity fields in receipts and ledger rows; `Coverage::holds()` requires `identity_gaps == 0`. +- **R-6 NotRun is never a pass,** and never in an accuracy numerator. +- **R-7 No invented thresholds.** `[A]` values are recalibrated after the first test version. +- **R-8 No Python.** +- **R-9 Lambda GPU.** + - 0 minutes for experiment cells, teacher, finetune and resident serve. + - Rung-2 shadow dispatch only (§2.1), with its minutes measured. + - 0 minutes once the lane holds tie-breaker or above. +- **R-10 train-active falls through, never blocks.** +- **R-11 No authority without a rung.** The lane decides nothing beyond its §5.4 rung. +- **R-12 Measurement states its tree.** +- **R-13 Capture is complete.** One `agent-trace-v1` row per lane call, NotRun included, with the full output and findings text. +- **R-14 Producer never labels.** +- **R-15 No secret in any pool or replay set;** no in-place redaction. +- **R-16 Blobs are write-once,** with a second copy within 24 h. +- **R-17 One spelling per lane state.** +- **R-18 Hosted outputs are never training targets.** G-PROV = 0 at every admission, in every pool manifest and in every release artifact. + +--- + +## §7 Tickets (EV-ordered) + +**Phases** (these replace APR-EPIC-001 §4d P0–P6): + +| Phase | Train | Content | +|---|---|---| +| **P0** | now | capture + shadow | +| **P1** | 0.71 (E2) | speed instrument + ratchet | +| **P2** | 0.71 | experiment + hardware ruling | +| **P3** | 0.72 | tripwire | +| **P4** | 0.73 (E6) | tie-breaker; prefill parity | +| **P5** | D-7 | local distill (Q4/Q5) | +| **P6** | 0.75 (E8) | vote | + +| EV | Row | Was | Phase | Work | Contract | Done when | K̂ | Status | +|---|---|---|---|---|---|---|---|---| +| 0 | PRM-00 | REX-00 | — | v1 prereg | `rex-prereg-v1` | — | — | **DONE** `5f0ae10ac` | +| 0 | **PRM-00b** | new | P0 | lock v3: §2–§5, `stats.rs` (min inside resample, R-class H5, H7 max-pair rule), analysis plan v2 | `rex-prereg-v2` | sha recorded; a planted §3 edit fails; 0 rows exist under v1 | 60 | todo, before any measurement | +| 0 | **PRM-L** | new | P0 | land `rex/001` plus this spec on `main` (own PR); banner on v1 file | — | merged; `rex prereg-check` green on `main` | 45 | todo | +| 0 | PRM-01 | REX-01 | P0 | split infra#1088: **gx10 slice** (4B weights by sha, `apr-review-serve` unit, cgroup, health, `exec_sha256`, train-active yield) as P0; corpus tar and replay sets to almacen | — | slice issue filed with checkable criteria | 20 | DONE → split todo | +| 0 | PRM-C1 | PRA T1 | P0 | capture at dispatch, incl. `latency_ms{queue,ttft,prefill,decode,total}` | `agent-trace-v1` | **transferred → pi#436**; F-R13 green there | — | foreign | +| 0 | **PRM-C2** | PRA T2 | P0 | `review-ledger-v2`: findings text, trace shas, weights sha, typed states | `review-ledger-v2` | FALSIFY: no weights sha → `holds()=false`; free-text reason → parse error; v1 fixtures parse | 120 | todo | +| 0 | PRM-C3–C5 | PRA T3–T5 | P0 | almacen store, second copy, transcript retention | `trace-blob-store-v1`, `trace-backup-v1`, `transcript-retention-v1` | **transferred → infra**; drill 0 mismatches; 31-day canary survives | — | foreign | +| 0 | **PRM-07** | REX-07 | P0 | shadow activation | `review-ledger-v2` | on gx10 slice + pi release: 100% of receipts carry an apr row; planted gx10-down → width 3, `NotRun{NoExecutor}` | 90 | active on its 2 deps | +| 1 | **PRM-S1** | new | P1 | replay benchmark `review-replay-v1` + runner (apr vs llama.cpp, same cell and GGUF) | `review-replay-v1` | 200 items, 4 strata, 0 sealed overlap; receipt per tag; queue budget derived from captured voter p95 | 150 | todo | +| 1 | **PRM-S2** | new | P1 | workload profile: input/output token distribution per lane | extends `agent-trace-v1` | §0.2 confirmed or falsified with receipt; strata weights set | 60 | todo | +| 1 | PRM-10 | REX-10 | P1 | perf ratchet wired to replay; per-tag issue per worsened stratum; knob sweep per train | `review-lane-perf-ratchet-v1` | 3 real tags recorded; planted +10% RED | 60 | gate landed `634916cbd` | +| 1 | PRM-02 / 03 | REX-02 / 03 | — | corpus v1; harness + scorer | — | — | — | **DONE** `8185ad1c3` / `0ab6ce731` | +| 2 | PRM-04 | REX-04 | P2 | cell admission re-run under the `parity.oracle` field | `rex-cell-admission-v1` | ≥ 1 cell `Admitted` or a receipted refusal | 30 | DONE (S-7) → re-run | +| 2 | PRM-C6 | PRA T6 | P2 | secret gate plus backfill | `trace-admission-secret-v1` | AWS-key literal plus 20 canaries quarantined | 180 | gate + `rex pool-admit` backfill on rex/001 (FALSIFY-TAS-001..009); the run over real rows waits on the almacen store (C3–C5, infra) | +| 2 | PRM-C7–C9 | PRA T7–T9 | P2 | contamination on clusters; MinHash dedup; `repo#PR` time split | `trace-dedup-v1`, `trace-split-guard-v1` | perturbed sealed item refused; 0 cross-split clusters | 360 | todo | +| 2 | PRM-C10 | PRA T10 | P2 | outcome join (14 d) plus HRQ rulings as gold | `trace-outcome-join-v1` | 3-day-old gold refused; weekly matured count | 150 | todo | +| 2 | **PRM-C13** | PRA T12–T13 | P2 | G-PROV gate plus κ_err probe (all pairs) | `lane-independence-v1`, `trace-admission-prov-v1` | a planted hosted row in a pool → RED; κ matrix `[V]` | 120 | todo | +| 2 | PRM-C14 | PRA T14 | P2 | datacard (Croissant+RAI) plus weekly yield/bytes receipt | `trace-datacard-v1` | G16 `[U]` → `[V]` | 120 | library + `rex datacard` on rex/001 (FALSIFY-TDC-001..009; exit 13 while any week is unmeasured); G16 `[V]` waits on real almacen rows (C3–C5, infra) | +| 3 | PRM-05 | REX-05 | P2 | pilot | extends receipt | sample-size rule on R-recall applied | 90 | parked (prep `469bf0c98`) | +| 3 | PRM-06 | REX-06 | P2 | full run plus comparators | extends receipt | 100% receipts or explicit NotRun; 0 unknown identity | 240 | parked | +| 4 | PRM-08 | REX-08 | P2 | analysis plus hardware ruling | `prm-001-report-v1` | H1–H7 verdicts; primary/failover or shadow-only with reason | 150 | parked | +| 5 | **PRM-09** | REX-09 | P3/P4 | ladder: add the **tie-breaker** rung; wire pi (#428) and arbiter (infra#1095) | `rex-promotion-ladder-v2` | planted reports: H4 only → tripwire; H4+H5+H7 → tie-breaker; lambda primary → refused | 60 | gate landed `4a4318f1c` → extend | +| 6 | PRM-11 | REX-11 | P3 | Q1–Q3 loop (prompt, gold few-shot, json-schema) | `review-champion-challenger-v1` | ≥ 1 challenger end-to-end; counter increments | 120 | gate landed `d8696650e` | +| 7 | PRM-C11 | PRA T11 | P5 | `sparse-logits-v1` in `apr serve` rows (local only) | `sparse-logits-v1` | header + residual mass round-trip | 180 | `capture` + `rex sparse-logits` on rex/001 (FALSIFY-SPL-005); the rail call in apr-advisory-ask.sh is outside aprender | +| 7 | PRM-12 | REX-12 | P5 | Q4/Q5: 27B teacher on gx10 → 4B student; gold QLoRA | `review-b2-loop-v1` | teacher dataset receipted (0 test hashes, 0 hosted rows); `NotRun{VerbRefused}` until D-7's train | 180 | gate landed `da7b6386f` | + +**Budget:** remaining aprender K̂ = **2,585 min `[A]`** `[C: sum of the rows above]` · **K = 2,840** (1.1 × K̂) · **andon at 2,275** (0.88 × K̂). Foreign rows are outside K. + +--- + +## §8 STOP conditions (exhaustive) + +- **S-1** A measurement before the `rex-prereg-v2` lock. +- **S-2** An undeclared `apr`, or a weights sha mismatch, on an executing host. +- **S-3** Any sealed hash or cluster in a pool or replay set. +- **S-4** Lambda GPU work outside rung-2 shadow, or any lambda work at tie-breaker or above. +- **S-5** A run overlaps train-active on a clean-room/CUDA pool without yielding. +- **S-6** **A hosted-model output appears in any training pool, manifest or release artifact (R-18).** +- **S-7** No admissible cell for the hardware ruling. Capture, shadow and P0/P1 rows continue. +- **S-8** A harness hook refuses writes for a reason other than a missing ticket. +- **S-9** K is reached, or the andon is crossed with more than one row incomplete. +- **S-10** Two consecutive non-passing quorums on the same PR. +- **S-11** A secret must be purged from all copies. +- **S-12** Almacen free < 20%, or the second-copy target is missing from forjar inventory. +- **S-13** Any change grants authority outside §5.4. +- **S-14** Transcripts contain third-party personal data beyond author identities. + +### Operator decisions + +| ID | Question | Status / default | +|---|---|---| +| R-9 | lambda GPU | **decided: option b**, shadow only (§2.1) | +| Training data | hosted outputs as targets | **decided: never** (§0.4, R-18) | +| D-7 | qwen35 finetune/distill: 0.71 (G5) or 0.75 (E8)? | default 0.71 under E4 as the PRM-12 acceptance test; E8 keeps multi-engine CRUX | +| D-8 | tripwire HRQ cap per day | none; blocks P3 only | +| D-10 | move the counted Claude lanes from Max OAuth to API keys (Commercial Terms; Max limits assume "ordinary, individual usage") | none; does not block Prometheus | + +--- + +## §9 Final report schema and scoreboard + +```yaml +spec: PRM-001 +version: 3 +prereg_sha: +session: {date_utc, host, tree: {head, origin_main, worktree}, model} +selected_row: PRM-NN +ticket: PMAT-NNNN +outcome: merged | stopped | already-done | premise-falsified | foreign-linked +stop: {id: S-n, evidence, resume_when} +baseline_remeasured: [{row: G#, value, mark, command}] +lane: {rung: shadow|tripwire|tie-breaker|vote, rows_7d, availability_7d, by_state: {}, by_served_by: {}, lambda_gpu_min} +speed: # PRM-S1 / PRM-10, per tag + replay_version: review-replay-vN + apr_tag: vX.Y.Z + cell: + p50_s: ; p95_s: ; p95_ci: [] + queue_budget_p95_s: ; within_budget: bool + ttft_ms: {apr: , llama_cpp: , ratio: } + prefill_tps: {"2k": {apr: , llama_cpp: , ratio: }, "8k": {}, "16k": {}, "32k": {}} + decode_tps: {apr: , llama_cpp: , ratio: } + parse_rate: ; verdict_identity_vs_prev_tag: +workload: {input_tokens_p50_p95: {}, output_tokens_p50_p95: {}} +corpus: {rows_7d, completeness_7d, capture_failed_open, gold_matured_7d, hosted_rows_in_pools: 0, quarantined_7d, restore_drill_mismatch} +cells: [{cell, status, reason, apr_tag, apr_sha, weights_sha, parity: {oracle, cosine, receipt}}] +results: {hypotheses: [{id: H1..H7, statistic, ci, p_holm, verdict}], + per_cell: [{cell, n, recall_P, recall_R, precision, false_refute, localization, parse_rate, p50_s, p95_s, ttft_ms, prefill_tps, decode_tps, peak_anon_gb, llama_cpp_ratio, interference}], + comparators: [{lane, model_id, recall_P, recall_R, precision, false_refute, p95_s}], + kappa_err: [{pair, kappa, n}]} +hardware_ruling: {primary, failover, rung_eligible, reason} +promotion: {champion: {weights_sha, adapter_sha, prompt_sha}, challengers: [{id, verdict, mcnemar_p, false_refute_delta, kappa_err}], test_version, evaluations_used, tie_breaks_decided, tie_break_outcome_agreement} +issues_filed: [{repo, url, purpose}] +budget: {k_hat_row, k_actual_row, cumulative, andon_crossed} +next_row: PRM-NN +``` + +| Scoreboard metric | Target | Rendered by | +|---|---|---| +| **Replay p95 on the primary cell** | ≤ slowest-voter p95 (measured); never up after 3 tags | PRM-S1 / PRM-10 | +| **TTFT ratio vs llama.cpp** | ≤ 2× by E2 (0.71) `[O]` | replay receipt | +| **Prefill ratio vs llama.cpp, per stratum** | Target: ≥ 1.0× by E6 (0.73) `[O]` | replay receipt | +| Decode ratio vs llama.cpp | Target: ≥ 1.0× by E6 `[O]` | replay receipt | +| Verdict availability, rolling 7 d | ≥ 95% before tie-breaker `[O]` | ledger | +| Receipts carrying an apr row | **100%** | receipt-lint | +| Closes decided by apr above its rung | **0** | receipt audit | +| Rows whose cell ≠ the backend that ran | **0** | `served_by` audit | +| Capture completeness; `capture_failed` open > 24 h | **100% / 0** | trace index | +| Hosted rows in any training pool or release | **0** | G-PROV | +| Secrets / sealed hashes / cross-split clusters in pools or replay | **0 / 0 / 0** | §4.2 | +| Restore mismatches; blobs without a second copy after 24 h | **0 / 0** | backup | +| Lambda GPU minutes outside rung-2 shadow | **0** | host receipts | +| Champion sealed-test correctness | never down across promotions | §5.3 | +| κ_err(qwen, voter) | ≤ max voter–voter κ_err | H7 | +| Python lines in files this spec touches | **0** | diff scan | + +**Five whys (why Qwen cannot break a tie today):** +- A review takes minutes, not seconds (G1: ≈ 10× Haiku). +- The GPU path falls back to CPU (#4313), and prefill runs at 0.26× (G7). +- Prefill was optimised against synthetic pp512, never against the real review workload, and the lane waited on a 4-host executor batch. + +Terminal causes, as mechanisms: +- **no real-workload speed gate** → fixed by PRM-S1 as the E2/E6 acceptance test; +- **batch-coupled executor dependency** → fixed by PRM-01, one slice per executor. diff --git a/docs/specifications/figures/prometheus-flow.png b/docs/specifications/figures/prometheus-flow.png new file mode 100644 index 0000000000..353732a8d3 Binary files /dev/null and b/docs/specifications/figures/prometheus-flow.png differ diff --git a/docs/specifications/research-design-pr-agent-qwen-3.5.md b/docs/specifications/research-design-pr-agent-qwen-3.5.md new file mode 100644 index 0000000000..b660cba1c4 --- /dev/null +++ b/docs/specifications/research-design-pr-agent-qwen-3.5.md @@ -0,0 +1,483 @@ +# research-design-pr-agent-qwen-3.5.md — PRA-001: Capture-First Trace Corpus for Distilling a Local Qwen 3.5 PR Reviewer + +> **SUPERSEDED** by [`PRM-001-prometheus.md`](PRM-001-prometheus.md) (PRM-001 v3, "PROMETHEUS"), which folds PRA-001 and REX-001 into one programme. This document is kept unchanged below as the design record. Where the two differ, PRM-001 wins. + +Every quorum is currently throwing away its training pairs. The fix is to capture every lane's exact input bytes, full raw output and findings text at dispatch, under a content-addressed and provenance-tagged `agent-trace-v1` contract, before any training happens. Silver and gold labels stay separable on every row, and a measured lane-independence guard (error-kappa) controls whether silver distillation is allowed at all. + +## TL;DR +- **Capture before training.** `review-ledger-v1` stores findings as a count, so each of the 50–200 quorums/day [A] loses its (input, output, findings) triples. PRA-001 moves capture to dispatch inside the quorum rail (paiml-implement #436). Each row stores the exact input bytes, the full raw reply, decoding params, top-k=20 [A] logits for local lanes, provider, exact model id, access channel and terms reference. Rows go into content-addressed zstd blobs on almacen, with a nightly second copy on separate media. +- **Silver is allowed but tagged and fenced.** The literature supports a conservative design. Top-k logit caches give biased teacher estimates (Anshumann et al., ACL 2025: top-K caching gives "biased estimates of teacher probability distribution"), and so does sequence-level KD alone. Code-review corpora carry heavy label noise: Liu et al. found only 64% of the original CodeReviewer comments were valid. They also carry near-duplicate/temporal leakage: Lee et al. (ACL 2022) found train-test overlap in over 4% of standard validation sets, and Allamanis found code-duplication inflates metrics by up to 100%. LLM errors are strongly correlated across providers: Kim et al. (ICML 2025) found model pairs agree about 60% of the time when both are wrong on HELM, against 1/3 by chance. So every row carries `label_tier`, `provider`, `model_id` and `terms_ref`. Splits are grouped by repo#PR and ordered by time, and outcome labels mature only after the 14-day [A] revert window. +- **The dissent lane has to stay independent.** Pairwise Cohen's κ on errors against gold is measured before and after any silver training. If κ rises above the pre-registered baseline + margin [A], the silver-trained candidate is refused as the Qwen shadow lane; it may still be promoted elsewhere via PRM-001 §5.4. Anthropic's and Google's current terms both contain competing-model/ML-development restrictions (summarised in §2.10 and Appendix A.7, not legal advice). The operator has ruled S-6, so the design's job is to make any provider's rows filterable later. + +--- + +## Header + +| Field | Value | +|---|---| +| Spec id | **PRA-001** (PR-Review Agent trace corpus) | +| Title | Research design: collecting PR-review agent data to fine-tune and distill a local Qwen 3.5 reviewer | +| Author | Noah Gift, Pragmatic AI Labs (paiml), Sovereign AI Stack | +| Date | 2026-09-25 | +| Target repo / path | `paiml/aprender` → `docs/specifications/research-design-pr-agent-qwen-3.5.md` | +| Runner | the aprender traffic cop | +| Launch line | `Implement docs/specifications/research-design-pr-agent-qwen-3.5.md autonomously.` | +| Related specs | REX-001 / PRM-001 "Prometheus" (H1–H6, champion/challenger §5.4, sealed-test rotation after 20 [A] promotion decisions); ARB-SELF-001 (dogfood loop, outcome ledger, escape join); ARB-APR-001 (arbiter on apr); FLOW-001 (fleet flow rules); EXT-001 (continuous HF release of the dogfooded model) | +| Contracts introduced | `agent-trace-v1`, `review-ledger-v2`, `trace-blob-store-v1`, `trace-backup-v1`, `transcript-retention-v1`, `trace-admission-secret-v1`, `trace-dedup-v1`, `trace-split-guard-v1`, `trace-outcome-join-v1`, `sparse-logits-v1`, `lane-independence-v1`, `trace-datacard-v1` | +| Contracts reused | `review-corpus-contamination-v1` (sealed test, corpus `review-corpus-v1`) | + +**Provenance marks (apply to every number in this document)** + +| Mark | Meaning | Obligation | +|---|---|---| +| **[V]** | Verified by a measurement command in this repo/fleet | The command is quoted beside the number | +| **[C]** | Computed from other marked numbers | The arithmetic is shown; it inherits the weakest input mark | +| **[A]** | Asserted (operator decision or design choice) | Changing it is a spec change, not a tuning knob | +| **[U]** | Unmeasured estimate | Must be replaced by [V] at the first scheduled probe (§3), or the row goes to andon | +| **[X]** | Third-party figure (paper, vendor, docs) | Cited in Appendix A; never used as a gate threshold without local re-measurement | + +Rule: there are no invented thresholds. A gate threshold is either [V] (a measured baseline plus a declared ratchet) or [A] (an explicit policy choice such as 0 or 100%). + +--- + +## §0 Operating assumptions + +1. **Capture-first.** Anything not captured at dispatch is lost forever. Capture comes before curation, curation before training, and training before promotion. No sampling and no TTL [A]. +2. **Silver and gold are separable per row, forever.** `label_tier ∈ {silver, gold, pending, quarantined}` together with `provider` and `model_id` is enough to filter any row out of any pool without re-deriving anything. +3. **The sealed test is inviolable.** The sealed test (105 items, 70 defects [A], corpus `review-corpus-v1`) is never in any training pool. Zero sealed hashes in any pool [A]. Rotation follows PRM-001 only. +4. **The shadow lane is zero-weight.** The Qwen row is mandatory in every receipt, but it never blocks, breaks a tie or escalates. PRA-001 changes what is *recorded*, not how votes are counted. +5. **The producer is never the gate.** The process that writes a trace never admits it to the training pool. Admission (§4) is a separate binary with its own contract. +6. **The operator has no pre-steps.** Anything only the operator can do (credentials, media purchase, ToS decisions, sealed-corpus changes) is a §8 STOP, not a hidden prerequisite. +7. **Pure Rust + bashrs-clean shell. No Python anywhere.** That includes training-data tooling, scanners and dedup. Provisioning uses only `forjar apply` or make targets, with released and sha-verified `apr` tags. Weights are pinned by sha256. +8. **Legal posture.** Provider terms are recorded as row metadata (§2.10). This document is **not legal advice**. The operator's S-6 ruling governs; PRA-001 only guarantees that rows can be filtered later. + +--- + +## §1 Ground truth (frozen baseline, 2026-09-25) + +Every operator-supplied fact is [A] until ticket T0 converts it to [V] by running the listed command. If a measured value disagrees with the [A] value, that is a §8 STOP (baseline drift). + +| # | Fact | Mark | Source of truth | Verify command (T0) | +|---|---|---|---|---| +| G1 | Quorums/day across paiml repos: 50–200 | [A] | quorum receipts | `make trace-yield WINDOW=7d` | +| G2 | Counted lanes: Claude Sonnet 5, Claude Haiku 4.5, agy (Antigravity / Gemini 3.1 Pro); shadow: Qwen3.5-4B-Q4_K_M via `apr serve` | [A] | paiml-implement rail config | `pmat quorum lanes --json` | +| G3 | Qwen row states: Verdict \| Refused \| NotRun{NoExecutor\|Busy\|Timeout\|ContextOverflow\|TrainActive} | [A] | receipt schema | receipt schema test | +| G4 | Dispatch ladder: gx10-cuda → lambda-cuda (shadow-only, dispatch-time, GPU-lock free, never resident) → intel-wgpu → mini-metal → intel-cpu → NotRun{NoExecutor}; row records the backend that actually ran | [A] | rail dispatcher | `make trace-served-by-audit` | +| G5 | `review-ledger-v1` keeps {family, model, verdict, findings: count} only; text/inputs/outputs discarded | [A] | `crates/aprender-review-experiment/src/ledger.rs` | `grep -n 'findings' crates/aprender-review-experiment/src/ledger.rs` | +| G6 | S-6: Claude and agy outputs MAY train the local model (silver) | [A] | operator ruling S-6 | n/a (ruling) | +| G7 | Gold = sealed corpus + outcome joins (merged / reverted ≤ 14 days / `regression`-labelled escape) | [A] | ARB-SELF-001 outcome ledger | `make trace-outcome-join --dry-run` | +| G8 | Sealed test: 105 items, 70 defects, corpus `review-corpus-v1` | [A] | `review-corpus-contamination-v1` | `pv check review-corpus-contamination-v1` | +| G9 | Storage: almacen UNAS Pro, 3×16 TB RAID 5, 32 TB usable, 10G | [A] | forjar inventory | `forjar inventory almacen --json` | +| G10 | Raw growth ≤ ~0.45 TB/yr | [U] | estimate | `make trace-bytes WINDOW=7d` (replace in week 1) | +| G11 | A real AWS-key literal has already landed in a diff | [A] | incident record | `make trace-secret-scan-backfill` | +| G12 | `apr finetune/distill/merge` refuse qwen35 until aprender 0.71; `apr quantize` available; `--json-schema` planned 0.70 | [A] | aprender release notes | `apr --version && apr finetune --model-arch qwen35 --dry-run` (expect refusal) | +| G13 | Qwen3.5-27B teacher on gx10 can write top-k logits | [A] | apr capability | `make teacher-logit-smoke` | +| G14 | Claude Code stores transcripts in plaintext under `~/.claude/projects/` "for 30 days by default" (Claude Code docs); Desktop/Cowork sessions exempt by default | [X] | Claude Code docs (App. A.8) | `make transcript-retention-audit` | + +--- + +### §1.1 Amendment (2026-09-25, T0): verify commands that exist + +T0 (`evidence/pra-001/t0-baseline.yaml`) found that none of the §1 verify commands exist as written: the seven `make` targets, `pmat quorum`, `pv check` and `apr --model-arch`. Cop ruling (2026-09-25): a missing instrument is not a contradiction. G1, G7, G10, G11 and G13 are **[U]-pending-instrument**, each tied to the ticket that builds it. G2 and G4 are paiml-implement#436 scope (T1, R-9). This table replaces the §1 "Verify command" column. A `make` target named in §1 lands with its ticket and replaces the interim command here. + +| # | Verify command (real, today) | Until then | +|---|---|---| +| G1 | lands with T1: `trace::weekly` row count over the index | [U]-pending-instrument (T1) | +| G2 | `jq '.quorum' ~/.claude/skills/paiml-implement/config.json` | STOP S-BASE → #436 scope | +| G3 | `cargo test -p aprender-review-experiment --lib every_wire_form_round_trips` | — | +| G4 | the #436 dispatcher's row `served_by` audit; no dispatcher exists yet | STOP S-BASE → #436 R-9 | +| G5 | `grep -n 'findings' crates/aprender-review-experiment/src/ledger.rs` | — | +| G6 | n/a (ruling) | — | +| G7 | lands with T10 (`trace-outcome-join-v1`) | [U]-pending-instrument (T10) | +| G8 | `pv validate contracts/review-corpus-contamination-v1.yaml && cargo test -p aprender-review-experiment --lib contamination` | — | +| G9 | `df -B1` on the almacen mount, on a host that mounts it | — | +| G10 | lands with T1: `trace::weekly` `bytes_raw`/`bytes_zstd` over week 1 | [U]-pending-instrument (T1) | +| G11 | lands with T6 (secret-scan backfill over the index) | [U]-pending-instrument (T6) | +| G12 | `. scripts/apr_bin.sh && "$APR" --version && "$APR" finetune --plan` (expect a refusal; `apr` has no `--model-arch`) | STOP S-BASE: `--plan` exits 0 with no refusal on 0.69.3 | +| G13 | lands with T16 (27B teacher on gx10, infra#1132) | [U]-pending-instrument (T16) | +| G14 | per host: `jq .cleanupPeriodDays ~/.claude/settings.json` and `find ~/.claude/projects -mtime +30 \| wc -l` | — | + +--- + +## §2 Design + +### §2.1 Capture points + +| Stream | Where | When | Why here | +|---|---|---|---| +| **S1: lane traces** | Inside the quorum rail (paiml-implement, issue #436), at the dispatch boundary of each lane call | Synchronously, before the verdict is parsed and before the receipt is written | This is the only point where the exact input bytes, the raw reply and the actual backend co-exist. Parsing is lossy and receipts are summaries. | +| **S2: session transcripts** | Claude Code `~/.claude/projects/**/*.jsonl` (including `subagents/`), agy logs | Harvested by a forjar-declared timer on each workstation into almacen; never edited in place | For later agentic (multi-turn, tool-use) distillation. Retention is set by config, not by hope (§2.8). | + +Mechanism for S1: the rail wraps every lane call in `TraceSpan::open(lane, round)`. The span hashes the request bytes *as serialised to the wire*, not a reconstruction. It streams the response into a blob writer while the lane client reads it. It closes with the parsed verdict and findings. If the blob write fails, the receipt row is written with `trace_status = CaptureFailed` and andon fires. The quorum itself is never blocked (jidoka on the capture line, not on the review line). + +### §2.2 `agent-trace-v1` row schema (one row per lane per round) + +| Field | Type | Required | Notes | +|---|---|---|---| +| `schema` | const `"agent-trace-v1"` | ✔ | versioned; readers reject unknown majors | +| `trace_id` | ULID | ✔ | time-sortable | +| `quorum_id` | string | ✔ | joins to receipt | +| `round` | u16 | ✔ | re-review rounds within a quorum | +| `lane` | enum {sonnet, haiku, agy, qwen-shadow, teacher-27b, …} | ✔ | | +| `counted` | bool | ✔ | false for qwen-shadow and teacher | +| `provider` | enum {anthropic, google, local} | ✔ | filter key for terms (§2.10) | +| `model_id` | string (exact, e.g. dated snapshot id or weights sha256) | ✔ | never a family alias | +| `access_channel` | enum {anthropic-api, claude-code-cli, antigravity, gemini-api, vertex, apr-serve} | ✔ | terms differ by channel (App. A.7) | +| `terms_ref` | {url, effective_date, fetched_at} | ✔ | snapshot of governing terms at capture | +| `repo`, `pr`, `head_sha`, `base_sha` | string/u64 | ✔ | | +| `diff_sha256` | hex | ✔ | sha of the exact diff bytes sent | +| `input_sha` | hex | ✔ | sha of the full wire request (system prompt + prompt version + diff + tools) | +| `input_parts` | [{role, blob_sha, bytes}] | ✔ | enables dedup of shared system prompts/diffs across lanes | +| `prompt_version` | semver | ✔ | | +| `output_sha` | hex | ✔ | full raw reply, including any returned reasoning; never truncated | +| `decoding` | {temperature, top_p, top_k, max_tokens, seed?, json_schema_sha?} | ✔ | provider-reported where available, otherwise as sent | +| `tokens` | {input, output, reasoning?, cached?} | ✔ | provider-reported; `null` is not the same as 0 | +| `latency_ms` | {ttft?, total} | ✔ | | +| `served_by` | string (cell that actually ran, e.g. `gx10-cpu`) | ✔ | records parity-guard fallback | +| `lane_state` | enum Verdict \| Refused \| NotRun{…} | ✔ | NotRun rows carry no output blob but are still rows | +| `verdict` | enum {approve, request_changes, comment, abstain} | if Verdict | | +| `findings` | [{file, line_start, line_end, severity, category, text}] | if Verdict | **text, not count** | +| `parse_status` | enum {ok, repaired, failed} | ✔ | failed parses keep raw output | +| `logits_sha` | hex \| null | local lanes | `sparse-logits-v1` blob (§2.7) | +| `label_tier` | enum {silver, gold, pending, quarantined} | ✔ | set by admission, never by producer | +| `outcome` | enum {merged, reverted_le14d, regression_escape, closed_unmerged, pending} | ✔ | via `trace-outcome-join-v1` | +| `split_guard` | {group_key: "repo#pr", split: train\|val\|test\|sealed, assigned_at, dedup_cluster} | ✔ | §2.9 | +| `secret_scan` | {scanner_versions, hits, status} | ✔ | §4 | +| `trace_status` | enum {ok, capture_failed, partial} | ✔ | | +| `producer` | {binary, version, git_sha} | ✔ | provenance of the writer | + +### §2.3 Blob layout on almacen + +``` +/almacen/traces/ + blobs/sha256/ab/cd/.zst # content-addressed, zstd, immutable, write-once + index/agent-trace-v1/YYYY/MM/DD.jsonl # uncompressed JSONL index, append-only + transcripts/claude-code//.zst + transcripts/agy//.zst + quarantine/sha256/… # secret-hit blobs, mode 0400, excluded from pool + manifests/YYYY-MM-DD.sha256 # daily Merkle-style manifest of new blobs + datacard/ # Croissant + RAI metadata (§2.11) +``` + +- The address is the sha256 of the **uncompressed** bytes. The compression level is recorded in the index, never implied by the path. This allows recompression, or later dictionary training, without re-addressing. +- Blobs are write-once (`O_EXCL`), and existing addresses are skipped, which gives free dedup of shared system prompts and diffs across the 4 lanes. The zstd level defaults to 19 [A] for archival blobs and 3 [A] for hot index rotation; replace both with the measured ratio from §3 M8. +- The index is uncompressed so that `grep`/`jq`-class Rust tools and the mining index (§2.9) can read it without decompression. + +### §2.4 Second copy (RAID 5 is not a backup) + +- Nightly `trace-backup-v1`: every blob named in `manifests/.sha256` is copied to separate media on a separate host. This is 3-2-1 in shape: primary almacen, second media, and a third offsite copy as a §8 STOP pending operator media. +- Weekly restore drill: sample N=100 [A] random blobs from the second copy, decompress them, re-hash them, and compare. Target mismatch = 0 [A]. +- Rationale (computed, worst case): a degraded 3×16 TB RAID 5 rebuild reads the two surviving 16 TB members, 32 TB ≈ 2.56×10¹⁴ bits [C]. At a consumer URE spec of 1 per 10¹⁴ bits [X], the expected UREs are ≈ 2.56 [C] and P(≥1) ≈ 1 − e^−2.56 ≈ 0.92 [C]. At an enterprise spec of 1 per 10¹⁵ [X], P ≈ 0.23 [C]. Spec rates are warranty floors, and field rates are lower [X]. But "kept forever" plus a single-parity array is not a durability story. The second copy is mandatory. + +### §2.5 Index schema (JSONL, one object per row) + +This is the full `agent-trace-v1` row minus blob contents, plus the mining columns: + +| Column | Definition | +|---|---| +| `agreed` | all counted lanes returned the same verdict in this round | +| `lane_disagreement` | bitmask of lanes that differ from the counted majority (Qwen included, as a measurement only) | +| `outcome` | as §2.2 | +| `label_tier` | as §2.2 | +| `unique_diff` | true if this `diff_sha256` is not in any earlier row and not in a near-dup cluster already seen (§2.9) | +| `dedup_cluster` | MinHash-LSH cluster id over diff shingles | +| `outcome_matured_at` | timestamp when the 14-day [A] revert window closed; null means pending | + +### §2.6 Retention + +Keep forever. No sampling, no TTL [A]. A row is never deleted. The only operation is to *quarantine* it (move the blob, flip `label_tier`). Deletion requests (for example, a leaked secret that must be purged from all copies) are a §8 STOP handled by the operator. The index records the tombstone. + +### §2.7 Logit storage format (`sparse-logits-v1`) + +The design choice is to store top-k **plus the residual mass**, and to record the sampling policy. Future distillation objectives (forward KL, reverse KL, on-policy) need more than the top-k ids. + +Per output token t: + +``` +struct TokenLogits { + token_id_sampled: u32, + k: u8, // 20 [A] + ids: [u32; k], // u32 because Qwen vocab > 2^16 [A: verify via `apr inspect`] + logprobs: [f16; k], // log-softmax at temperature 1.0, pre-sampling-filters + logsumexp_full: f32, // normaliser over full vocab + topk_mass: f16, // Σ exp(logprobs) = retained mass M ≤ 1 +} +``` + +- Header per blob: `{model_sha256, tokenizer_sha256, apr_tag, backend(served_by), temperature_of_record, k, n_tokens}`. +- Encoding: columnar (all ids, then all logprobs), then zstd. Size ≈ 20 × (4+2) + 8 ≈ 128 B/token raw [C]. That is ≈ 256 KB [C] for a 2,000-token [U] review. +- Why residual mass: naive top-k caching gives biased estimates of the teacher distribution and hurts calibration [X, App. A.1]. A fused forward-KL objective over a sparse top-K teacher explicitly uses retained mass M ≤ 1 [X]. Storing `topk_mass` and `logsumexp_full` keeps the "tail bucket" correction available. +- Option reserved: `sparse-logits-v1` has a `mode` field {topk, rs-sample}. It allows importance-sampled (RS-KD-style) token sets to be written by the 27B teacher later without a schema change [X, App. A.1]. +- Frontier lanes (Claude, agy) provide no full logits. Their rows carry `logits_sha = null`; sequence-level KD only. + +### §2.8 Session transcripts (S2) + +- `transcript-retention-v1`: `forjar apply` writes Claude Code **managed settings** fleet-wide with `cleanupPeriodDays` set to a large value (36500 [A]). The value must be ≥1 [X]; 0 fails validation in current versions [X], and in an earlier version it silently disabled persistence [X]. Managed settings take precedence over user settings [X]. +- The harvester copies `projects/**/*.jsonl` **and** `projects/**/subagents/*.jsonl` to almacen hourly [A]. These are content-addressed, so re-harvesting an appended file stores a new blob, and the index links versions by `session_id`. +- Transcripts are plaintext and include tool outputs, file contents and anything a command printed [X]. They are the highest secret-risk stream, so every transcript blob passes §4 G-SEC before it is eligible for any pool. +- Falsifier: plant a canary transcript older than 31 days [A] on a test host. Run a Claude Code session start. Assert the canary survives. + +### §2.9 Split policy (`trace-split-guard-v1`) + +1. **Group key = `repo#pr`.** Every row, round and re-review of a PR goes to one split [A]. Rows are never split individually. Re-reviews are near-duplicates. +2. **Time-ordered.** Splits are assigned by PR *open time* with forward-chaining cutoffs: train < val < test in time [A]. There is an embargo gap ≥ 14 days [A] between train cutoff and val start, so that outcome labels in val cannot leak through revert chains. +3. **Cross-PR near-dup closure.** Cherry-picks, backports and re-opened PRs produce near-identical diffs under different keys. Diff shingles are MinHash-LSH clustered. A union-find closure assigns every cluster to the split of its **earliest** member [A]. Target cross-split cluster count = 0 [A]. +4. **Sealed firewall.** Any row whose `diff_sha256` or cluster matches `review-corpus-v1` sealed items gets `split = sealed` and is never in a pool (G-CON, §4). +5. **Outcome maturity.** A row is eligible for gold only after `outcome_matured_at` is set. Pending is never treated as negative [A]. + +Mining at training time: oversample `lane_disagreement ≠ 0` and `outcome ∈ {reverted_le14d, regression_escape}`. The oversampling factor is chosen per run and logged; it is not a spec constant. + +### §2.10 Provider-terms provenance (tagging requirement, not advice) + +Every row carries `provider`, `model_id`, `access_channel` and `terms_ref`. The governing terms differ by channel (App. A.7). The pool builder takes a `--exclude-provider` / `--exclude-channel` filter as a first-class argument. Rebuilding any pool without a given source is a single command, and it is falsified in §9 (P-TERMS). **Not legal advice; S-6 governs.** + +### §2.11 Dataset documentation (`trace-datacard-v1`) + +- Machine-readable: a Croissant JSON-LD file (dataset metadata, resources, record sets, semantics) with the Croissant-RAI extension fields: collection process, labelling (silver/gold provenance), limitations, intended use, PII/secret handling. It is generated by a Rust binary from the index, never hand-edited. +- Human-readable: a Datasheet-style card (motivation, composition, collection, preprocessing, uses, distribution, maintenance) under `datacard/`. It is regenerated per snapshot. +- Publication to HF (EXT-001) is gold-and-local-only by default [A]. Rows with `provider ∈ {anthropic, google}` are excluded from any public release unless the operator overrides this at a §8 STOP. + +--- + +## §3 Research questions and pre-registered measurements + +All measurements run from make targets delivered by the tickets in §7. The first run sets the [V] baseline, and later gates ratchet from it. + +| ID | Question | Metric (definition) | Command | Baseline mark now | +|---|---|---|---|---| +| M1 | Yield | admitted rows/week by `label_tier` × lane | `make trace-yield WINDOW=7d` | [U] | +| M2 | Capture completeness | rows with all ✔ fields ÷ lane calls dispatched | `make trace-completeness` | target 100% [A] | +| M3 | Disagreement rate | fraction of rounds with `lane_disagreement ≠ 0` among counted lanes | `make trace-disagreement` | [U] | +| M4 | Error overlap | Cohen's κ on binary error indicators vs gold, per lane pair (incl. Qwen); CAPA as secondary | `make lane-kappa SPLIT=val` | [U] | +| M5 | Near-dup rate | fraction of rows in a cluster of size > 1; cross-split clusters | `make trace-dedup-report` | cross-split = 0 [A]; rate [U] | +| M6 | Bytes/round | raw and compressed bytes per lane-round, split by blob kind (input, output, logits) | `make trace-bytes WINDOW=7d` | [U] (G10 ≤ ~0.45 TB/yr) | +| M7 | Secret hit rate | hits ÷ rows scanned, by stream (S1 diff, S1 output, S2 transcript) | `make trace-secret-report` | [U] | +| M8 | Compression ratio | raw ÷ zstd bytes by blob kind and level | `make trace-zstd-bench` | [U]; literature suggests 6–7× for JSONL at level 3 [X] | +| M9 | Outcome label yield | gold rows matured/week; revert and escape counts | `make trace-outcome-join` | [U] | +| M10 | Shadow availability | Qwen `lane_state` distribution by `served_by` | `make trace-served-by-audit` | [U] | +| M11 | Token lengths | p50/p95 input/output/reasoning tokens by lane | `make trace-tokens` | [U] | +| M12 | Silver noise | fraction of silver findings contradicted by gold outcome (on matured rows) | `make silver-audit` | [U] | + +Pre-registered hypotheses (they extend REX-001 H1–H6 and do not replace them): +- **PRA-H1:** Silver-trained Qwen increases κ_err(Qwen, counted lane) by more than the pre-registered margin [A] relative to base Qwen. +- **PRA-H2:** Gold-only fine-tuning does not increase κ_err beyond the margin [A]. +- **PRA-H3:** Disagreement-oversampled training improves sealed-test defect recall over uniform sampling at an equal token budget. +- **PRA-H4:** Grouped time-ordered splits lower val score relative to random row splits. If they do, the size of the drop is the leakage measure, and it is reported. + +--- + +## §4 Admission gates to the training pool + +Admission is a separate binary (`trace-admit`). It is never the producer. A row enters the pool only if **all** gates pass. Failure is recorded; there is no silent drop. + +| Gate | Rule | Target | Falsifier (planted RED → GREEN) | +|---|---|---|---| +| **G-SEC** secret scan | Union of two independent scanners over input blobs, output blobs and transcripts: (a) an in-process Rust entropy+pattern scanner (ripsecrets-class); (b) the gitleaks public ruleset ported **as data** (TOML rules → Rust `regex` set), with no Go/Python runtime. Any hit sends the row to `quarantined`, blobs to `quarantine/`. **No in-place redaction** [A]. | quarantined-row leakage into pool = 0 [A] | Plant the known AWS-key-literal diff plus 20 [A] synthetic canaries (AWS, GitHub PAT, private key, JWT, high-entropy generic) in a fixture PR. RED: the pool contains the canary. GREEN: all canaries quarantined, pool count 0. | +| **G-SEC-FN** scanner recall | Measured recall on the canary set plus a held-out labelled fixture | 100% on canaries [A]; real-world recall [U] | Remove one rule; the canary for it must go RED. | +| **G-CON** contamination | 0 sealed-test hashes (exact `diff_sha256`) and 0 sealed near-dup clusters in any pool (`review-corpus-contamination-v1` extended to MinHash clusters) | 0 [A] | Insert one sealed item with whitespace/rename perturbation into a candidate pool. RED on exact-hash-only; GREEN once cluster check lands. | +| **G-ID** identity completeness | `provider`, `model_id` (exact), `access_channel`, `terms_ref`, `input_sha`, `output_sha`, `served_by`, `label_tier`, `split_guard` all non-null | 100% [A] | Row with `model_id = "claude-sonnet"` (alias) must be refused. | +| **G-DUP** dedup and split | Row's cluster is not in a different split; exact duplicates of admitted rows are linked, not re-admitted | cross-split = 0 [A] | Backport PR with identical diff under new PR number assigned to val while original in train: RED → GREEN after union-find. | +| **G-MAT** outcome maturity | `label_tier = gold` requires `outcome_matured_at` ≠ null | 0 immature gold rows [A] | Gold row 3 days old must be refused. | +| **G-HASH** blob integrity | Re-hash of every referenced blob equals its address | mismatch = 0 [A] | Flip one byte in a test blob. | + +--- + +## §5 How the data feeds the improvement loop + +| Phase | When | What | Data used | +|---|---|---|---| +| **B1: prompt / few-shot** | now (apr 0.69+) | Mine disagreement and escape rows into retrieval few-shot exemplars for the Qwen prompt, then version the prompt (`prompt_version`). After 0.70, use `--json-schema` constrained decoding to drive `parse_status = failed` toward 0. | gold first; silver exemplars only when tagged in the prompt manifest | +| **B2a: sequence-level distill / SFT** | apr 0.71 | `apr finetune` on (input, output) pairs. Silver and gold are pooled per run manifest, and every manifest lists its tiers and providers. | admitted pool | +| **B2b: logit distill from 27B teacher** | apr 0.71 | The 27B teacher re-scores diffs, writing `sparse-logits-v1`. The student is trained with a forward-KL objective that uses the tail-mass correction, with reverse-KL / on-policy (GKD-style) as challengers. On-policy needs the student's own samples, which the Qwen shadow rows already are. | local rows only (teacher and student both local) | +| **Promotion** | per PRM-001 §5.4 | Champion/challenger on the sealed test. Rotation after 20 [A] promotion decisions. | sealed (evaluation only) | + +**Preserving dissent independence (`lane-independence-v1`).** +1. **Measure before training.** κ_err for every (Qwen, counted-lane) pair on matured gold val rows goes into the scoreboard as the [V] baseline. +2. **Two heads, two pools.** The *dissent* checkpoint that fills the shadow lane is trained on **gold only**. A *general* checkpoint may use silver. Only the dissent checkpoint may occupy the Qwen shadow lane. +3. **Gate.** A candidate for the shadow lane is refused if κ_err(candidate, any counted lane) > baseline + δ, with δ [A] pre-registered in the run manifest before training. Refusal is an andon, not a warning. +4. **Diversity option (research, not default).** Down-weight silver examples where all three counted lanes agree and the outcome is still pending. These are exactly the examples that teach "agree with the voters", with no evidence behind them. + +--- + +## §6 Hard rules (each falsified in §9) + +| # | Rule | Falsifier id | +|---|---|---| +| R1 | Every lane call produces exactly one `agent-trace-v1` row, NotRun included | F-R1 | +| R2 | `output_sha` covers the full raw reply; truncation anywhere is a defect | F-R2 | +| R3 | Findings are stored as text; a count-only findings field fails schema validation | F-R3 | +| R4 | The producer never sets `label_tier` other than `pending` | F-R4 | +| R5 | No row with a secret hit ever enters a pool; no in-place redaction | F-R5 | +| R6 | 0 sealed hashes or sealed clusters in any pool | F-R6 | +| R7 | Splits are by `repo#pr` group, time-ordered, cluster-closed | F-R7 | +| R8 | Every row has exact `model_id`, `provider`, `access_channel`, `terms_ref` | F-R8 | +| R9 | Blobs are write-once and re-hash equal to their address | F-R9 | +| R10 | Nightly second copy exists and the weekly restore drill mismatch is 0 | F-R10 | +| R11 | Transcript retention is declared fleet-wide by forjar; no host runs the default 30 | F-R11 | +| R12 | Qwen shadow lane weight stays 0 regardless of trace content | F-R12 | +| R13 | A shadow-lane candidate failing the κ_err gate is not deployed | F-R13 | +| R14 | No Python in any PRA-001 artifact; shell is bashrs-clean | F-R14 | +| R15 | `apr` is a released, sha-verified tag; weights are pinned by sha256; `served_by` reflects the actual backend | F-R15 | + +--- + +## §7 EV-ordered tickets + +Each ticket: `pmat work add`, feature branch, PR, protected main, `ci / gate`, TDD, one `pv` contract with its falsifier planted RED then GREEN. **K̂** = estimated minutes [A]; **K** = measured minutes (recorded at close); **andon** if K > 2·K̂ [A] or any falsifier fails to go RED first. + +| EV rank | Ticket | Contract | Done when | K̂ [A] | +|---|---|---|---|---| +| 1 | T1 Capture at dispatch in quorum rail (paiml-implement #436) | `agent-trace-v1` | F-R1, F-R2, F-R3 GREEN; 7 days of 100% completeness (M2) | 240 | +| 2 | T2 Ledger v2: findings text + trace links; v1 readers keep working | `review-ledger-v2` | v1 fixtures parse; v2 row carries findings text; migration test | 120 | +| 3 | T3 Content-addressed blob store on almacen | `trace-blob-store-v1` | F-R9 GREEN; O_EXCL write-once test; forjar-declared mount | 120 | +| 4 | T4 Transcript retention + harvester | `transcript-retention-v1` | F-R11 GREEN on every host in inventory; subagents/ harvested | 90 | +| 5 | T5 Nightly second copy + weekly restore drill | `trace-backup-v1` | F-R10 GREEN; first drill mismatch = 0 | 120 | +| 6 | T6 Secret admission gate (Rust scanner + ported ruleset) + backfill | `trace-admission-secret-v1` | F-R5 GREEN; AWS-key literal quarantined in backfill | 180 | +| 7 | T0 Baseline freeze: run §1 verify commands, convert [A]/[U]→[V] | (report only) | §1 table all [V] or STOP raised | 60 | +| 8 | T7 Contamination gate extended to clusters | `review-corpus-contamination-v1` (extend) | F-R6 GREEN incl. perturbed sealed item | 90 | +| 9 | T8 MinHash-LSH dedup index (pure Rust) | `trace-dedup-v1` | M5 report produced; cluster ids in index | 150 | +| 10 | T9 Group/time split assigner with union-find closure | `trace-split-guard-v1` | F-R7 GREEN; cross-split clusters = 0 | 120 | +| 11 | T10 Outcome join (merged/revert ≤14d/regression escape) | `trace-outcome-join-v1` | G-MAT falsifier GREEN; M9 produced | 150 | +| 12 | T11 Qwen top-k logits capture in `apr serve` rows | `sparse-logits-v1` | header + residual mass round-trip test; M6 logits bytes measured | 180 | +| 13 | T12 Terms/provider tagging + exclusion filter | `agent-trace-v1` (terms fields) | P-TERMS GREEN (pool rebuild excluding a provider) | 60 | +| 14 | T13 Lane κ_err probe + scoreboard | `lane-independence-v1` | M4 baseline [V] recorded for every pair | 120 | +| 15 | T14 Datacard: Croissant + RAI + datasheet generator | `trace-datacard-v1` | generated file passes Croissant validation; regenerated per snapshot | 90 | +| 16 | T15 B1 few-shot miner from disagreement/escape rows | (uses `agent-trace-v1`) | prompt_version bump with manifest of exemplar tiers | 120 | +| 17 | T16 27B teacher re-score job (gx10) | `sparse-logits-v1` (mode topk) | teacher blobs for all admitted local rows of week N | 180 | + +EV rationale: T1–T5 stop irreversible loss, which is ongoing and compounds daily. T6–T10 make the data *admissible*. T11–T17 make it *useful*. Nothing training-related (B2) is ticketed until aprender 0.71 ships; that is a §8 STOP. + +--- + +## §8 STOP conditions (exhaustive) + +The runner halts, writes the §9 report with `status: STOPPED`, and waits for the operator when: + +1. **S-BASE:** any §1 verify command disagrees with its [A] value (baseline drift). +2. **S-CRED:** a step needs credentials, API keys, or account settings the runner does not hold. +3. **S-MEDIA:** the second-copy media or an offsite target does not exist in forjar inventory. +4. **S-CAP:** almacen free space < 20% [A], or the M6 projection exceeds 50% of usable capacity within 5 years [A]. +5. **S-SECRET-PURGE:** a secret hit requires purging from *all* copies (rotation is the operator's job; the runner only quarantines). +6. **S-SEALED:** any change to `review-corpus-v1`, sealed rotation outside PRM-001, or any sealed hash found in a pool. +7. **S-TERMS:** any change in provider terms detected by `terms_ref` refresh, any publication including non-local rows, or any request to change S-6. +8. **S-APR:** a required verb is missing from the released `apr` tag (for example B2 before 0.71), or a tag's sha fails verification. HEAD is never a fallback. +9. **S-PY:** a dependency would introduce Python, or a non-forjar install would be needed. +10. **S-KAPPA:** the κ_err gate refuses a shadow-lane candidate (the operator decides whether δ was right; the runner never widens δ). +11. **S-WEIGHT:** any change would give the Qwen lane non-zero weight, tie-break or escalation power. +12. **S-ANDON:** K > 2·K̂ [A] on any ticket, or a falsifier that cannot be made RED first. +13. **S-PII:** a transcript contains third-party personal data beyond author identities (scrubbing policy is an operator decision). +14. **S-LOSS:** any blob mismatch in the restore drill, or `capture_failed` rows > 0 for 24 h [A]. + +--- + +## §9 Final report schema and scoreboard + +```yaml +report: PRA-001 +version: 1 +generated_at: +status: COMPLETE | STOPPED | IN_PROGRESS +stop: {code: |null, evidence: } +apr_tag: # sha-verified +baseline: # §1 after T0 + - {id: G1, value: , mark: V, command: "make trace-yield WINDOW=7d"} +tickets: + - {id: T1, contract: agent-trace-v1, pr: , khat_min: 240, k_min: , andon: false, + falsifier: {id: F-R1, red_commit: , green_commit: }} +measurements: # §3 + - {id: M4, pair: [qwen-shadow, sonnet], kappa_err: , n: , split: val, mark: V} +gates: # §4 + - {id: G-SEC, pool_leaks: 0, canaries_quarantined: /} +falsifiers: # §6 + - {id: F-R5, state: GREEN, evidence: } +storage: + bytes_raw_per_week: + bytes_zstd_per_week: + projected_tb_per_year: # replaces G10 [U] + restore_drill_mismatch: 0 +``` + +**Scoreboard (targets and probes only)** + +| Probe | Target | Mark | +|---|---|---| +| Capture completeness (M2) | 100% | [A] | +| Rows lost to `capture_failed` | 0 | [A] | +| Secret-hit rows in any pool | 0 | [A] | +| Sealed hashes/clusters in any pool | 0 | [A] | +| Cross-split near-dup clusters | 0 | [A] | +| Rows missing identity fields | 0 | [A] | +| Immature gold rows | 0 | [A] | +| Restore-drill mismatches | 0 | [A] | +| Hosts with default transcript retention | 0 | [A] | +| Qwen lane weight | 0 | [A] | +| κ_err(shadow candidate, counted lane) | ≤ baseline + δ | [V] baseline, [A] δ | +| Bytes/year | ≤ measured M6 projection; ratchet | [U]→[V] | +| Yield/week by tier | ratchet up from week-1 baseline | [U]→[V] | +| P-TERMS: pool rebuild excluding one provider | succeeds, 0 rows of that provider | [A] | + +Falsifier catalogue (F-R1…F-R15): each is a `pv` contract test that is committed RED (failing on a planted defect) before the implementing commit turns it GREEN. Examples: F-R1 kills a lane mid-call and asserts a NotRun row. F-R2 sends a 1 MB reply and asserts the byte-exact `output_sha`. F-R12 injects a Qwen verdict that disagrees with all counted lanes and asserts the quorum outcome is unchanged. F-R14 runs a repo-wide scan for `*.py`, `python`, `pip` in PRA-001 paths and expects 0 hits. + +--- + +## Appendix A: Research findings and how each changed the design + +**A.1 Distillation (sequence vs logit, sparse top-k).** +- Anshumann et al., *Sparse Logit Sampling* (ACL 2025 oral, arXiv 2503.16870), prove that caching top-K probabilities gives "biased estimates of teacher probability distribution to the student, resulting in suboptimal performance and calibration". Their importance-sampling RS-KD gives unbiased estimates while using only 0.01% of precomputed teacher logits. A secondary summary reports parity with full KD using about 12 tokens/step and ≈3.6 TB per 100B tokens versus ≈90 TB for top-300 [X, secondary]. +- A 2026 preprint (arXiv 2608.03796) frames sparse top-K KD as forward KL against a teacher with retained mass M ≤ 1. +- MiniLLM (Gu et al., ICLR 2024) argues reverse KL suits generative students. GKD (Agarwal et al., ICLR 2024) trains on student-generated sequences with teacher feedback. +- **Design changes:** store `topk_mass` + `logsumexp_full`, reserve `mode: rs-sample`, and keep Qwen's own outputs as on-policy samples to be re-scored by the local 27B teacher. Frontier lanes are sequence-level only, because no logits are available. + +**A.2 Code-review datasets and label quality.** +- CodeReviewer (Li et al., ESEC/FSE 2022) built a multilingual corpus (nine languages) of real diffs and review comments with three tasks: quality estimation, comment generation, refinement. Its schema is diff-in / label-or-comment-out, with human reviewer comments as labels. +- *Too Noisy To Learn* (Liu, Lin & Thongtanunam, arXiv 2502.02757) shows substantial vague or non-actionable noise remains after heuristic and SVM cleaning. Only 64% of the original CodeReviewer comments were valid, and LLM cleaning raised that "to up to 85%", with 66–85% precision in detecting valid comments. +- **Design change:** findings text alone is not a label. Gold comes from outcomes and the sealed corpus, silver is tagged, and M12 measures silver noise against matured outcomes. + +**A.3 Outcome signals as labels.** +- SZZ-derived bug-inducing labels are notoriously noisy. Herbold et al. (arXiv 1911.08938) report that "about one quarter of the links to defects detected by SZZ is wrong", and that for every correctly labelled bug issue there are 0.74 mislabelled ones. Afric et al. report 14.3% mislabelled even after refinement (via ReDef, arXiv 2509.09192). +- **Design change:** gold outcomes are restricted to direct, observable events (merge, revert ≤14 days [A], explicit `regression` label). No blame-tracing heuristics in v1. Outcomes mature before use (G-MAT), and pending is never negative. + +**A.4 Near-duplicates and leakage.** +- Allamanis (Onward! 2019) showed code duplication inflates reported metrics by up to 100%. +- Lee et al. (ACL 2022) found near-duplicates via MinHash (e.g. 3.04% of C4) and train-test overlap affecting over 4% of validation sets. +- **Design change:** group-by-PR and time-ordered splits, MinHash-LSH cluster closure across PRs, contamination extended from exact hash to cluster (G-CON), and PRA-H4 to quantify leakage. + +**A.5 Ensemble diversity and error correlation.** +- Kim et al. (*Correlated Errors in LLMs*, ICML 2025) evaluated 350+ LLMs. On HELM, model pairs agree about 60% of the time when both are wrong, against 1/3 for uniformly random wrong answers. Shared provider and architecture raise correlation, and larger, more accurate models are highly correlated even across providers. +- Goel et al. (*Great Models Think Alike*, ICML 2025) introduce CAPA and show judges favour similar models. They find weak-to-strong gains are larger when supervisor and student are functionally different. +- **Design change:** κ_err is measured before and after training (M4, CAPA as secondary). The dissent checkpoint is gold-only. A pre-registered δ gates the shadow lane (§5). + +**A.6 Secret detection.** +- Basak et al. (arXiv 2307.00714) measured tools on SecretBench (15,084 true secrets among 97,479 reported). Gitleaks had the best recall (88%) at 46% precision and the top F1 (60%), and TruffleHog reached 52% recall. GitHub's scanner had the best precision (75%) but only 6% recall. "No current tool has the coveted high precision and high recall." +- On a different benchmark (AssetHarvester, arXiv 2403.19072), Gitleaks recall fell to 0.02 and TruffleHog reached 0.40. Recall is dataset-dependent. +- ripsecrets is a Rust, local-only scanner that claims to be ≥95× faster [X, vendor claim]. +- **Design change:** union of two independent rule sources, implemented in Rust (the gitleaks ruleset ported as data). Quarantine, not redaction. Recall is measured locally with planted canaries rather than trusting published numbers. + +**A.7 Provider terms (factual summary; not legal advice; S-6 governs).** +- **Anthropic Commercial Terms of Service** (effective June 17, 2025), §D.4: "Customer may not and must not attempt to (a) access the Services to build a competing product or service, including to train competing AI models or resell the Services except as expressly approved by Anthropic…". §B also states that the customer owns its Outputs and that Anthropic may not train on Customer Content. +- **Gemini API Additional Terms** (effective March 23, 2026): "You may not use the Services to develop models that compete with the Services (e.g., Gemini API or Google AI Studio)." +- **Google Antigravity Additional Terms** (undated page) incorporate the Google Terms of Service (effective July 30, 2026). Those forbid "using AI-generated content from our services to develop machine learning models or related AI technology." Antigravity §8 binds users who select an Anthropic model to Anthropic's commercial terms. +- Google Cloud service terms exempt Gemini Enterprise Agent Platform (formerly Vertex AI) when no Google pre-trained model is used. +- Consumer terms for subscription-based Claude Code were not verified here. +- **Design change:** `access_channel` and `terms_ref` are mandatory. Pools can be rebuilt excluding any provider or channel (P-TERMS). Public releases default to local/gold rows. Terms changes are S-TERMS. + +**A.8 Claude Code transcripts.** +- Official docs: transcripts under `~/.claude/projects/` are plaintext, include everything that passed through tools, and are deleted after `cleanupPeriodDays`. The default is 30 and the minimum is 1; 0 fails validation. Managed settings govern. +- A February 2026 issue (#23710) reported that 0 had silently disabled persistence in an earlier version. Another issue (#58154) reports that `subagents/` transcripts are not swept. +- **Design change:** forjar-managed settings with a large explicit value, harvesting that includes `subagents/`, a survival canary, and a secret scan on every transcript. + +**A.9 Dataset documentation.** +- Croissant (Akhtar et al., NeurIPS 2024 D&B) defines four layers (metadata, resources, structure, semantics). Croissant-RAI builds on Datasheets for Datasets and Data Cards. +- NeurIPS 2026's Evaluations & Datasets track now requires RAI metadata in the Croissant file. Hugging Face auto-generates core Croissant. +- **Design change:** `trace-datacard-v1` generates Croissant + RAI + a datasheet from the index on each snapshot. + +**A.10 Storage sizing.** +- A consumer drive URE spec of 1 per 10¹⁴ bits (≈12.5 TB read) is a worst-case warranty floor. Field studies show lower rates [X]. +- zstd at level 3 is reported at 6–7× on JSONL [X, blog], and dictionaries help small similar files [X]. +- **Design change:** compression level is recorded per blob and not implied, M8 measures our own ratio, the second copy plus restore drill is mandatory, and G10 stays [U] until week 1. + +## Caveats +- Some literature figures come from secondary summaries (the RS-KD storage numbers, and the Afric et al. 14.3% figure, which comes via ReDef) and are marked [X, secondary] where applicable. None is used as a gate threshold. +- Secret-scanner recall differs by an order of magnitude between benchmarks. Local canary recall is necessary, not sufficient, and a zero-hit scan is not proof of absence. +- Provider terms change: Anthropic updates take effect 30 days after posting. `terms_ref` must be refreshed on a schedule, and nothing here is legal advice. +- κ_err needs matured gold rows. Until M9 produces enough of them, the independence gate cannot be evaluated, and B2 shadow-lane deployment must wait (S-KAPPA). \ No newline at end of file diff --git a/docs/specifications/review-experiment-protocol.md b/docs/specifications/review-experiment-protocol.md index f2447ba639..818b240386 100644 --- a/docs/specifications/review-experiment-protocol.md +++ b/docs/specifications/review-experiment-protocol.md @@ -11,6 +11,10 @@ --- +> **SUPERSEDED** by [`PRM-001-prometheus.md`](PRM-001-prometheus.md) (PRM-001 v3, 2026-09-25). This file is kept +> byte-frozen in §2–§5 so its `rex-prereg-v1` lock (`docs/audits/rex-001/prereg.lock`, +> prereg_sha `ef51087d…`) still verifies. Do not edit it; amend PRM-001 instead. + ## §0 Operating assumptions 1. **This is a scientific experiment, not a benchmark run.** @@ -49,7 +53,7 @@ | G4 | Train themes | 0.69 = verbs that are true; 0.70 = fast (TTFT, resident serve, `--json-schema`); finetune/distill on qwen35 refuse until 0.71; WGPU/Metal refuse until 0.71 | [A] operator ruling 2026-09-20; [U] at HEAD | `docs/specifications/06x-release-schedule.md`, milestones | | G5 | Parity oracle | llama.cpp pin `d1d3c3396` (`scripts/llama_pin.toml` `build_commit`); `ds.yaml` min_cosine 0.98 | [V] 2026-09-20; [U] at HEAD | `scripts/llama_pin.toml`, `ds.yaml` | | G6 | TTFT gap | apr 0.68.2 was several times slower than llama.cpp on Qwen3.5-4B, 4090, for both TTFT and end-to-end; the measured ratios are in #3596 (no evidence/ receipt in-tree) | [V] 2026-09-20 | #3596 | -| G7 | Host roles | lambda-labs = agent host, primary x86 CUDA (scarce); **its GPU is excluded from this experiment by construction.** intel = clean-room runner (contended), dual AMD GPU (Vulkan). gx10 = GB10, 120 GB unified, clean-room pool. mini = M4, macOS, `rust-neutral` pool | [A] | `infra/machines/*/forjar.yaml` | +| G7 | Host roles | lambda-labs = agent host, primary x86 CUDA (scarce); **its GPU is excluded from this experiment by construction** (shadow rung 2 only, R-9). intel = clean-room runner (contended), dual AMD GPU (Vulkan). gx10 = GB10, 120 GB unified, clean-room pool. mini = M4, macOS, `rust-neutral` pool | [A] | `infra/machines/*/forjar.yaml` | | G8 | Undeclared apr | lambda-labs `~/.local/bin/apr` was a hand-installed 0.64.0 | [V] 2026-09-20; [U] at HEAD | `command -v apr`; `fleet-bins.tsv` | | G9 | Reviewer identity | 301/301 historical quorum receipts record the reviewer model as `unknown` | [V] 2026-09-10 | PV-LEAN-AUDIT | | G10 | Decode ceilings (bandwidth ÷ bytes/token) | 4B Q4_K_M ≈ 2.5 GB/token → ceiling: 4090 <= ~400, GB10 <= ~110 tok/s | [C] theoretical, [U] for apr | — | @@ -263,7 +267,12 @@ A challenger (any §5.3 candidate) replaces the champion only if all of these ho - Every numeric gate cites its measurement command or is marked `[A]`/`[U]`. - The `[A]` values in this spec (corpus sizes, 14-day revert window, 20-evaluation budget) are recalibrated after the first test version and logged. - **R-8 No Python** in any file this spec creates, harness or analysis. Shell passes `bashrs`; everything else is Rust (an aprender workspace crate or xtask). -- **R-9 Lambda GPU is excluded.** No cell, teacher job or serve in this spec may place work on the lambda-labs 4090. +- **R-9 Lambda GPU is excluded, except as shadow rung 2 at dispatch time** (operator ruling 2026-09-25, option b). + - No REX experiment cell, teacher or finetune job, resident `apr serve` or VRAM-preloaded weights may place work on the lambda-labs 4090. Every other R-9 exclusion stands. + - The single exception: the apr SHADOW lane may run on `lambda-cuda` as rung 2, only when rung 1 (`gx10-cuda`) returned `NotRun` and the lambda GPU lock is free at dispatch time. + - `lambda-cuda` yields to agent sessions and `train-active`. A lock taken before or during the round makes the row `NotRun{Busy}`, never a Verdict, and the ladder continues. The round is not retried on the same cell. + - Fallback order: `gx10-cuda` → `lambda-cuda` → `intel-wgpu` → `mini-metal` → `intel-cpu` → `NotRun{NoExecutor}`. Each row records the backend that actually ran; the `gx10-cpu` labelling rule applies to lambda too. + - The rung-2 routing lives in paiml-implement#436. Falsifier: a fixture holding the lambda GPU lock produces a `lambda-cuda` row of `NotRun{Busy}`. - **R-10 The release train wins.** When `train-active` is set, the lane and the experiment jobs yield on clean-room/CUDA pools. - **R-11 The asymmetric vote holds** until H4 and H5 pass through the §5.4 machinery. Before that, apr never casts a deciding vote. - **R-12 Measurement states its tree:** HEAD vs `origin/main`, the host, and a per-session worktree. @@ -299,7 +308,7 @@ A challenger (any §5.3 candidate) replaces the champion only if all of these ho - **S-1** REX-00 cannot be committed before any measurement (pre-registration out of order). - **S-2** An executing host resolves `apr` to an undeclared binary, or its weights sha mismatches the declaration. - **S-3** The sealed-test contamination check fails anywhere. -- **S-4** Any change would place work on the lambda-labs GPU (R-9). +- **S-4** Any change would place work on the lambda-labs GPU outside the R-9 shadow rung-2 exception. - **S-5** A run overlaps `train-active` on a clean-room/CUDA pool without yielding. - **S-6** **Training on hosted-model outputs.** Using Claude or agy (Gemini) outputs as training data for B2 may conflict with those providers' terms. It is an operator decision, and until Noah rules, silver labels stay evaluation-irrelevant and unused for training. B2 proceeds with the local 27B teacher and gold labels only. - **S-7** Every cell is `Refused` or `NotRun`, so there is no admissible cell for (A). @@ -349,5 +358,5 @@ next_row: REX-NN | Gap to llama.cpp on the primary cell | measured every tag; filed when it grows | perf ratchet | | Champion correctness on the sealed test | never-down across promotions | §5.4 reports | | Lane precision / recall | measured, then ratcheted never-down | §5.4 reports | -| Lambda GPU minutes consumed by this spec | **0** | host receipts | +| Lambda GPU minutes consumed by this spec outside the R-9 shadow rung 2 | **0** | host receipts | | Python lines in files this spec touches | **0** | `grep -rl python3` over the diff | diff --git a/evidence/pra-001/report.yaml b/evidence/pra-001/report.yaml new file mode 100644 index 0000000000..dbcf898242 --- /dev/null +++ b/evidence/pra-001/report.yaml @@ -0,0 +1,42 @@ +# PRA-001 §9 final report. Lead: aprender-a2. Updated on every ticket close. +# The cop reads this file; it does not run measurements. +report: PRA-001 +version: 1 +generated_at: 2026-09-25T17:40:00Z +status: IN_PROGRESS +stop: {code: null, evidence: null} +apr_tag: null # set when a ticket measures against a sha-verified tag +spec: {path: docs/specifications/research-design-pr-agent-qwen-3.5.md, handoff_sha256_prefix: e8764b72adbe, amended: "§1.1 (T0 verify commands)"} +baseline: # §1 after T0; full rows in evidence/pra-001/t0-baseline.yaml + source: evidence/pra-001/t0-baseline.yaml + verified: [G3, G5, G8] + partial: [G9, G14] + ruling: [G6] + pending_instrument: {G1: T1, G7: T10, G10: T1, G11: T6, G13: T16} + stop_s_base: {G2: "paiml-implement#436 (T1)", G4: "paiml-implement#436 R-9", G12: "apr finetune --plan exits 0 with no refusal on 0.69.3"} +tickets: + - {id: T0, owner: aprender-60, contract: null, state: DONE, evidence: evidence/pra-001/t0-baseline.yaml} + - {id: T1, owner: paiml-implement-21, contract: agent-trace-v1, state: IN_PROGRESS} + - {id: T2, owner: "aprender-a2 + aprender-84", contract: review-ledger-v2, state: IN_PROGRESS, khat_min: 120} + - {id: T3, owner: infra-27, contract: trace-blob-store-v1, state: IN_PROGRESS} + - {id: T4, owner: infra-27, contract: transcript-retention-v1, state: IN_PROGRESS, note: "retention done 5/5 hosts, infra#1135"} + - {id: T5, owner: infra-27, contract: trace-backup-v1, state: IN_PROGRESS} + - {id: T6, owner: aprender-cb, contract: trace-admission-secret-v1, state: IN_PROGRESS} + - {id: T7, owner: aprender-cb, contract: review-corpus-contamination-v1, state: BLOCKED_ON, on: T6} + - {id: T8, owner: aprender-a2, contract: trace-dedup-v1, state: BLOCKED_ON, on: T2, khat_min: 150} + - {id: T9, owner: aprender-a2, contract: trace-split-guard-v1, state: BLOCKED_ON, on: T8, khat_min: 120} + - {id: T10, owner: aprender-a2, contract: trace-outcome-join-v1, state: BLOCKED_ON, on: T9, khat_min: 150} + - {id: T11, owner: aprender-84, contract: sparse-logits-v1, state: OPEN} + - {id: T12, owner: aprender-fb, contract: agent-trace-v1 (terms fields), state: OPEN} + - {id: T13, owner: aprender-36, contract: lane-independence-v1, state: OPEN, note: "idle until rc.2"} + - {id: T14, owner: aprender-88, contract: trace-datacard-v1, state: IN_PROGRESS} + - {id: T15, owner: null, state: NOT_STARTED, note: "not before T1-T10 GREEN"} + - {id: T16, owner: null, state: NOT_STARTED, note: "needs gx10 (infra#1132)"} +measurements: [] +gates: [] +falsifiers: [] +storage: + bytes_raw_per_week: null # trace::weekly over week 1 (T1) + bytes_zstd_per_week: null + projected_tb_per_year: null + restore_drill_mismatch: null diff --git a/evidence/pra-001/t0-baseline.yaml b/evidence/pra-001/t0-baseline.yaml new file mode 100644 index 0000000000..4e9c28f615 --- /dev/null +++ b/evidence/pra-001/t0-baseline.yaml @@ -0,0 +1,43 @@ +# PRA-001 T0 — baseline freeze (§1 G1–G14), report-only. Owner aprender-60, 2026-09-25 17:06Z, host lambda-labs. +# For aprender-a2's §9 report (evidence/pra-001/report.yaml `baseline:`). +# Finding that applies to all rows: NONE of the §1 "Verify command (T0)" commands exists as written. +# - make targets trace-yield, trace-served-by-audit, trace-outcome-join, trace-bytes, trace-secret-scan-backfill, +# teacher-logit-smoke, transcript-retention-audit: 0 of 7 in aprender origin/main Makefile +# - `pmat quorum lanes`: no `quorum` subcommand; `forjar inventory almacen`: no such argument form; +# `pv check`: no `check` subcommand; ledger.rs is on origin/rex/001 only, not main; +# `apr finetune --model-arch`: no such flag in apr 0.69.3. +# Where a fact could be measured another way it is [V] with that command; otherwise STOP S-BASE, not guessed. +apr_tag: v0.69.3 # x86_64 cpu asset, sha256 of tarball verified; apr binary a83d8c8e0829c9fa…580f6 +baseline: + - {id: G1, mark: U-pending-instrument, ticket: T1, ruling: "cop 2026-09-25: missing instrument, not a contradiction", value: null, + evidence: "quorum artifacts carry no timestamp (keys: agreed,author,…,head,lanes,ticket,width); file mtimes are checkout-contaminated (581 of 640 dated today); make trace-yield absent"} + - {id: G2, mark: STOP, code: S-BASE, scope: "paiml-implement#436 (T1)", value: "installed ~/.claude/skills/paiml-implement/config.json quorum.lane_models = [gemini-3.1-pro-high, gemini-3.8-flash-high, gemini-3.7-flash-high]; quorum.advisory_lane = null; quorum.local_lane = null", + command: "jq '.quorum' ~/.claude/skills/paiml-implement/config.json", + evidence: "DRIFT vs [A]: counted lanes sonnet-5+haiku-4-5+agy are passed per-round via --lane-model, not the rail config; the Qwen shadow is not wired in the installed rail at all"} + - {id: G3, mark: V, value: "NotRun{NoExecutor|Busy|Timeout|ContextOverflow|TrainActive} + Verdict + Refused", + command: "git grep -n 'enum NotRun' origin/rex/001 -- crates/aprender-review-experiment/src/ledger.rs", + note: "matches [A]; lives on origin/rex/001 (not main). receipt.rs has a different NotRun (NoDeclaredExecutor|ContextOverflow|Refused|ServeError|Inadmissible)"} + - {id: G4, mark: STOP, code: S-BASE, scope: "paiml-implement#436 R-9", value: "ledger row schema records cell+backend (gx10-cuda, intel-wgpu in tests); no ladder dispatcher exists", + evidence: "no gx10-cuda→lambda-cuda→intel-wgpu→mini-metal→intel-cpu dispatcher in paiml-implement origin/main or aprender origin/main; the only deployed executor is the gx10 watcher apr_dogfood_lane_4252.py (availability ≤60.8%, aprender evidence/4252-promotion.md)"} + - {id: G5, mark: V, value: "LaneRow {family, model, verdict, findings: usize}", + command: "git show origin/rex/001:crates/aprender-review-experiment/src/ledger.rs | grep -n findings", + note: "matches [A]; file is on origin/rex/001, not main"} + - {id: G6, mark: A, value: "operator ruling S-6", note: "a ruling, not measurable (spec: n/a)"} + - {id: G7, mark: U-pending-instrument, ticket: T10, ruling: "cop 2026-09-25: missing instrument, not a contradiction", value: null, + evidence: "make trace-outcome-join absent; no merged/reverted≤14d/regression outcome join found in infra origin/main arbiter sources"} + - {id: G8, mark: V, value: "Test split 105 items, 70 defects (35 P + 35 R + 35 G); Dev 45; corpus 150", + command: "git show origin/rex/001:docs/audits/review-corpus/corpus-v1.jsonl | jq -r '[.split,.class,((.defect|length)>0)]|@tsv' | sort | uniq -c", + note: "matches [A]; pv check absent, measured by count; corpus on origin/rex/001 only"} + - {id: G9, mark: V-partial, value: "32 TB usable (df: 31.98e12 B, 4.3T used, 15%)", + command: "df -B1 # the cifs media share", + evidence: "S-BASE for the unverifiable sub-claims: RAID 5, 3×16 TB, 10G link are not visible from a client and no forjar inventory entry states them (stack-architecture.yaml: 'UNAS Pro storage' only)"} + - {id: G10, mark: U-pending-instrument, ticket: T1, ruling: "cop 2026-09-25: missing instrument, not a contradiction", value: null, evidence: "make trace-bytes absent; no capture exists yet to measure — stays [U] until T1 captures a week"} + - {id: G11, mark: U-pending-instrument, ticket: T6, ruling: "cop 2026-09-25: missing instrument, not a contradiction", value: null, + evidence: "make trace-secret-scan-backfill absent; no incident record found (gh search 'AWS key' → only pmat#1129, an AWS-key-SHAPED literal the scanner missed, not a landed real key). Not re-scanned here: T6 owns the scanner"} + - {id: G12, mark: STOP, code: S-BASE, note_cop: "verify command wrong: apr has no --model-arch; see spec §1.1", value: "apr 0.69.3: finetune/distill/merge reject --model-arch (usage, rc=2); `apr finetune Qwen3.5-4B-Q4_K_M.gguf --plan` rc=0 (no refusal in plan mode); quantize present; `apr run --help` has no --json-schema", + evidence: "the refusal on a real qwen35 finetune was NOT probed: running one is training work (§8 S-APR before 0.71). Verify command needs rewriting"} + - {id: G13, mark: U-pending-instrument, ticket: T16, ruling: "cop 2026-09-25: missing instrument, not a contradiction", value: null, evidence: "make teacher-logit-smoke absent; 27B teacher on gx10 needs infra#1132"} + - {id: G14, mark: V-host, value: "this host: cleanupPeriodDays = 36500 (default 30 overridden); oldest transcript 2026-08-13", + command: "jq .cleanupPeriodDays ~/.claude/settings.json; find ~/.claude/projects -name '*.jsonl' -printf '%T@\\n' | sort -n | head -1", + note: "the docs claim stays [X]; other hosts not audited (infra-27 T4 reports retention done 5/5, infra#1135)"} +summary: {V: [G3, G5, G8], V_partial: [G9, G14], A_ruling: [G6], STOP_S_BASE: [G1, G2, G4, G7, G10, G11, G12, G13]} diff --git a/evidence/prm-001/s1-v2/33b5e5b24/rows-apr.jsonl b/evidence/prm-001/s1-v2/33b5e5b24/rows-apr.jsonl new file mode 100644 index 0000000000..c6eb8b9248 --- /dev/null +++ b/evidence/prm-001/s1-v2/33b5e5b24/rows-apr.jsonl @@ -0,0 +1,200 @@ +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"f1ca076d355345060d79de6ab291b725d78bfd3596f2cfc01444d40d11e2ea79","stratum":"2k","engine":"apr","engine_version":"apr 0.70.0 (33b5e5b24 srv-tim-001)","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":29791.826384,"prompt_ms":917.944433,"prompt_tps":1069.7815300111963,"decode_tps":29.604283217479693,"input_tokens":982,"output_tokens":187,"peak_rss_mb":8357.9296875,"verdict":{"state":"Fail"},"prompt_ids_sha256":"27e16faa8fbdff4c4d6e0ff52b176400b21425330e38ee63f75e165299d93d1b"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"7788e95d4d3ecd6288d7d51fa0ed8459158b9b7ff7b94bbbffd244bbb7e1dad1","stratum":"2k","engine":"apr","engine_version":"apr 0.70.0 (33b5e5b24 srv-tim-001)","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":4985.651980000001,"prompt_ms":733.382894,"prompt_tps":942.2090502154527,"decode_tps":31.0941566948802,"input_tokens":691,"output_tokens":131,"peak_rss_mb":8376.609375,"verdict":{"state":"Fail"},"prompt_ids_sha256":"10e34de14fde098ef400a5795d6d775f2af564296e76433ce573ff85c38f81a0"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"366eed3dd5e0055ebfdd837af58bd800debdd62e4c0a7f7da2dffa84d06ada9f","stratum":"2k","engine":"apr","engine_version":"apr 0.70.0 (33b5e5b24 srv-tim-001)","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":6913.986322999999,"prompt_ms":1536.35251,"prompt_tps":909.9474182523384,"decode_tps":27.883326935772196,"input_tokens":1398,"output_tokens":149,"peak_rss_mb":8377.5625,"verdict":{"state":"Pass"},"prompt_ids_sha256":"513376357643c1fb6950a0630d0a17b9cd1678c7d4961730e6cdd82a5e6755c4"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"f9d7c31e37b0ed40fbfd87af8984c8860ec6c047dc68a8b55bcb88ee67ee6452","stratum":"2k","engine":"apr","engine_version":"apr 0.70.0 (33b5e5b24 srv-tim-001)","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":6102.472102,"prompt_ms":753.266433,"prompt_tps":946.544235563992,"decode_tps":30.849735196369508,"input_tokens":713,"output_tokens":164,"peak_rss_mb":8377.609375,"verdict":{"state":"Fail"},"prompt_ids_sha256":"13634b8349a0b7acd08e20bc05d86f438f3a09e1be22555abc534924c2845212"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"f8879c54b81ffd2aaeaf6398391a818b5f56d1c37ffd4a71100bf19864c1b902","stratum":"2k","engine":"apr","engine_version":"apr 0.70.0 (33b5e5b24 srv-tim-001)","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":5188.224902,"prompt_ms":927.520064,"prompt_tps":1024.2366034682352,"decode_tps":29.805423686656003,"input_tokens":950,"output_tokens":126,"peak_rss_mb":8377.609375,"verdict":{"state":"Pass"},"prompt_ids_sha256":"6526b0ed8df35ece7bfb8f728415890c02540d9548fa313c47f20bb0514294f2"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"296c5544001c3a5e623d3bc13a3b59728bad8d086d6f5baaa8ec44409d3e5cea","stratum":"2k","engine":"apr","engine_version":"apr 0.70.0 (33b5e5b24 srv-tim-001)","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":4210.466455,"prompt_ms":458.122564,"prompt_tps":702.8686759903841,"decode_tps":33.069972025462704,"input_tokens":322,"output_tokens":123,"peak_rss_mb":8377.6953125,"verdict":{"state":"Fail"},"prompt_ids_sha256":"1f540bb4461f766244ab8dbc1d7a1c48162b6cb270d8ae0eb239e075d14fcb7f"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"275cbe4c206c2eb59aca3ca92f0bbd7f171bc949a205afe8e44882e4e56fdc5c","stratum":"2k","engine":"apr","engine_version":"apr 0.70.0 (33b5e5b24 srv-tim-001)","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":12115.443689,"prompt_ms":1856.159411,"prompt_tps":1025.773965703854,"decode_tps":25.71843116329161,"input_tokens":1904,"output_tokens":263,"peak_rss_mb":8377.6953125,"verdict":{"state":"Fail"},"prompt_ids_sha256":"e482de36d4b6c8e06ae0882d309b33305eabf0479dedcbcc690ff717f0dc969c"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"2d012d3f8e232bfdbed32db1d29426b65b9c7725d2a001308ac4fb59f8cdafa9","stratum":"2k","engine":"apr","engine_version":"apr 0.70.0 (33b5e5b24 srv-tim-001)","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":15653.418716999999,"prompt_ms":2008.317401,"prompt_tps":909.716760453444,"decode_tps":25.787349673329253,"input_tokens":1827,"output_tokens":351,"peak_rss_mb":8377.78125,"verdict":{"state":"Pass"},"prompt_ids_sha256":"aee8112d87fb3291454282615ad62f23da8c7bfa3b2e162d7e909ec867ad11b3"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"e777a873cb9229ece5b23335a8dd1bb421f1b5e298f76024a7af842e2cfcadfb","stratum":"2k","engine":"apr","engine_version":"apr 0.70.0 (33b5e5b24 srv-tim-001)","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":8061.9946979999995,"prompt_ms":1972.704207,"prompt_tps":957.568799872286,"decode_tps":25.925013634542616,"input_tokens":1889,"output_tokens":157,"peak_rss_mb":8377.78125,"verdict":{"state":"Pass"},"prompt_ids_sha256":"dd8abf75e8df1fb4ba891ef3ac0728486da2bed010880eb742d6d79e669fe432"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"a7a62e12ad053746008e769e95d9c06f39e8b741a52ad4f650d05d1ca720c754","stratum":"2k","engine":"apr","engine_version":"apr 0.70.0 (33b5e5b24 srv-tim-001)","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":10417.306201000001,"prompt_ms":2002.574432,"prompt_tps":911.8262826197912,"decode_tps":26.014346962707982,"input_tokens":1826,"output_tokens":218,"peak_rss_mb":8377.8046875,"verdict":{"state":"Pass"},"prompt_ids_sha256":"7081fe352adf097f8a152f710ba69d4a79e4044130c1c3982d8f715f175e3c32"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"66ec2c8042874218dcf97952a6a557fe1223cb59db4a28bf7615cd2a6a4f269b","stratum":"2k","engine":"apr","engine_version":"apr 0.70.0 (33b5e5b24 srv-tim-001)","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":3656.4179660000004,"prompt_ms":477.688129,"prompt_tps":759.9100290809195,"decode_tps":32.7431342170391,"input_tokens":363,"output_tokens":103,"peak_rss_mb":8377.8046875,"verdict":{"state":"Pass"},"prompt_ids_sha256":"7f8167b3b624a3f6f0dd6b3c48781ebe5c7aa79d9a20f2016a592cbdb64df348"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"5f449a13b53a4663c67c7b0cea0588720e0a2e63e79561ba6adcd58d9bd87d77","stratum":"2k","engine":"apr","engine_version":"apr 0.70.0 (33b5e5b24 srv-tim-001)","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":14394.34874,"prompt_ms":1883.005295,"prompt_tps":926.17902064901,"decode_tps":26.046633893322603,"input_tokens":1744,"output_tokens":325,"peak_rss_mb":8377.8046875,"verdict":{"state":"Fail"},"prompt_ids_sha256":"4be9a7ed9292e82aa712abd073d04ef245be3607a90208a80f7195e73fbdfd62"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"3b86db746737d9338f2ac5c3c0aa7184acd7f05314b46fb8ae1a8bbdd83ac4ad","stratum":"2k","engine":"apr","engine_version":"apr 0.70.0 (33b5e5b24 srv-tim-001)","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":4940.2985739999995,"prompt_ms":724.988025,"prompt_tps":906.2218648370088,"decode_tps":31.085391025631303,"input_tokens":657,"output_tokens":130,"peak_rss_mb":8377.80859375,"verdict":{"state":"Pass"},"prompt_ids_sha256":"8278ed707fc9d1928b2ec8f690a552e0a16913c5847057eb444ca2509a21b839"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"a5f04841ad4bdf65c809d00511d1b4669ed6f52b6ec0be2eac5ee0417b29297f","stratum":"2k","engine":"apr","engine_version":"apr 0.70.0 (33b5e5b24 srv-tim-001)","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":5865.681105000001,"prompt_ms":1740.686276,"prompt_tps":936.4122774298244,"decode_tps":26.887150064032173,"input_tokens":1630,"output_tokens":110,"peak_rss_mb":8377.80859375,"verdict":{"state":"Pass"},"prompt_ids_sha256":"f02afc68f2366821fc07f38c65009e232df057bd81d998875a7600a85ff58c1c"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"f1206fea7a5282d2607d846f5d4f0a9fe7b6982f8d0ab75a42632d7a8ca4e487","stratum":"2k","engine":"apr","engine_version":"apr 0.70.0 (33b5e5b24 srv-tim-001)","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":7215.449715,"prompt_ms":961.961235,"prompt_tps":1044.740643836859,"decode_tps":29.260357764841338,"input_tokens":1005,"output_tokens":182,"peak_rss_mb":8377.80859375,"verdict":{"state":"Pass"},"prompt_ids_sha256":"f76934c956f6886a06b4e394e263a24c39df6734ad36fed9be55913886901237"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"993b00de597f00fefd6e145197fc5659223b89690cc0a671c636d955b4c961c5","stratum":"2k","engine":"apr","engine_version":"apr 0.70.0 (33b5e5b24 srv-tim-001)","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":10845.966950999999,"prompt_ms":1696.5569120000002,"prompt_tps":934.834539756365,"decode_tps":26.769128146452974,"input_tokens":1586,"output_tokens":244,"peak_rss_mb":8377.80859375,"verdict":{"state":"Pass"},"prompt_ids_sha256":"79f32211f992adecbdc1e0848ba2e99956661eb798fbac366156478f0670d2d2"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"47015b6bbe7f1a4b0b27cd8396c3430142487ab219124adf8c7f331e365c7ec3","stratum":"2k","engine":"apr","engine_version":"apr 0.70.0 (33b5e5b24 srv-tim-001)","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":10464.630604,"prompt_ms":1786.93517,"prompt_tps":928.964871176608,"decode_tps":26.492912740349993,"input_tokens":1660,"output_tokens":229,"peak_rss_mb":8377.80859375,"verdict":{"state":"Pass"},"prompt_ids_sha256":"c8ceab8a8499f6119d42fc491d582972ddcbb9d0eeb513d16f0a8bc73f03a953"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"9af203f89adb6dc0348f46161c0da6bdf7ebea3752a638220571ad3a65854634","stratum":"2k","engine":"apr","engine_version":"apr 0.70.0 (33b5e5b24 srv-tim-001)","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":8127.020953,"prompt_ms":1886.414086,"prompt_tps":907.0118871027132,"decode_tps":26.42708532007474,"input_tokens":1711,"output_tokens":164,"peak_rss_mb":8377.80859375,"verdict":{"state":"Pass"},"prompt_ids_sha256":"40d56ab8f83a8a2a5627f8c21de53e8cc3086149ca9a2d3f7bf948f43294f577"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"34d4c5a7a0e46bbd1b00935ef3000cc8ddba8dd64cbdffa9e01f1207863f945c","stratum":"2k","engine":"apr","engine_version":"apr 0.70.0 (33b5e5b24 srv-tim-001)","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":11457.418155,"prompt_ms":1576.610402,"prompt_tps":921.5973700013684,"decode_tps":27.21739406408738,"input_tokens":1453,"output_tokens":268,"peak_rss_mb":8377.80859375,"verdict":{"state":"Fail"},"prompt_ids_sha256":"6ea38a67c9dcd29895861eb72204a5cb6730ab44d894a6ab601f1753ff7c9df4"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"167d286e0f8a0bae32d26b9d8abc056df02e030188cf395ff255acc3cb1d640d","stratum":"2k","engine":"apr","engine_version":"apr 0.70.0 (33b5e5b24 srv-tim-001)","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":8497.093073,"prompt_ms":895.024811,"prompt_tps":1054.7193646456356,"decode_tps":28.71537043205392,"input_tokens":944,"output_tokens":217,"peak_rss_mb":8377.80859375,"verdict":{"state":"Fail"},"prompt_ids_sha256":"2fcc6f03b34f1893afd08fe8ec1dcf30568ee3397ef74662c46ef4085107c1d6"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"fab75467c60ebd40cc94c13296129957a26a7eb9865a58175dd09db9610f1631","stratum":"2k","engine":"apr","engine_version":"apr 0.70.0 (33b5e5b24 srv-tim-001)","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":8450.357757999998,"prompt_ms":988.811728,"prompt_tps":1064.9145536833682,"decode_tps":28.449177766327075,"input_tokens":1053,"output_tokens":211,"peak_rss_mb":8377.80859375,"verdict":{"state":"Pass"},"prompt_ids_sha256":"e857ce0443d4fb4bf8d286c247650ecf93b401aff929197f55c9aa7777881371"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"c274c3708f67fd97e467895e6c780665af6a13a28c8a727d25ea1863e3328dc8","stratum":"2k","engine":"apr","engine_version":"apr 0.70.0 (33b5e5b24 srv-tim-001)","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":9207.837065,"prompt_ms":1637.387991,"prompt_tps":942.965264486296,"decode_tps":26.977711741229975,"input_tokens":1544,"output_tokens":203,"peak_rss_mb":8377.82421875,"verdict":{"state":"Pass"},"prompt_ids_sha256":"6624743530f806ebfeef29164bc538770adccaf112ec5c254583759b9530f2af"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"11ca5b1e87d2bbf217c1fd81b2c2ea63a96390dea93e9d9258aa7dff541f8de8","stratum":"2k","engine":"apr","engine_version":"apr 0.70.0 (33b5e5b24 srv-tim-001)","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":10906.896439,"prompt_ms":1547.771698,"prompt_tps":989.1639716492606,"decode_tps":26.95210295814403,"input_tokens":1531,"output_tokens":251,"peak_rss_mb":8377.82421875,"verdict":{"state":"Pass"},"prompt_ids_sha256":"7f03bbbdd56a316389c60433ff343e431f7bf8984965b1d7c775d0a2b663d42c"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"7ba9cfea210f85f585654545dc13dc487fe37b56924c0996577c87a463100bac","stratum":"2k","engine":"apr","engine_version":"apr 0.70.0 (33b5e5b24 srv-tim-001)","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":3600.8057989999998,"prompt_ms":418.70338,"prompt_tps":654.4012135751088,"decode_tps":33.15190429863316,"input_tokens":274,"output_tokens":104,"peak_rss_mb":8377.82421875,"verdict":{"state":"Pass"},"prompt_ids_sha256":"4a21bcf8590e6ea0af00d401671ef07b3715194ed9f4846feab7f8ce28239f10"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"b1c9fb8c5a26aa63dcdb57e6ec6ce35a2c5a73287516f57082f77944c007add1","stratum":"2k","engine":"apr","engine_version":"apr 0.70.0 (33b5e5b24 srv-tim-001)","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":4189.183352999999,"prompt_ms":790.151149,"prompt_tps":1002.3398700392196,"decode_tps":30.419138392023104,"input_tokens":792,"output_tokens":102,"peak_rss_mb":8377.82421875,"verdict":{"state":"Pass"},"prompt_ids_sha256":"2f02b9baa0667bc81604dd82b8f81abd390a46442f75a67fed1f9ca6ef54d929"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"9d82e15d5df69ff6b6a2f1d4dd81ba144dca2978ee4f352de1b134cf86a4bfdd","stratum":"2k","engine":"apr","engine_version":"apr 0.70.0 (33b5e5b24 srv-tim-001)","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":13124.780472999999,"prompt_ms":1666.771633,"prompt_tps":968.938976417053,"decode_tps":26.549779145254732,"input_tokens":1615,"output_tokens":303,"peak_rss_mb":8377.82421875,"verdict":{"state":"Fail"},"prompt_ids_sha256":"7e2fd107b311a51bce7d7d6826cbde8412778be86ccc57cd60587b5bf24a54f4"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"4dca9a17db342792f25605430639922e342855e3341653071266173c29d9a859","stratum":"2k","engine":"apr","engine_version":"apr 0.70.0 (33b5e5b24 srv-tim-001)","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":9354.420521,"prompt_ms":1066.947199,"prompt_tps":1107.833640791066,"decode_tps":28.392007577940387,"input_tokens":1182,"output_tokens":234,"peak_rss_mb":8377.82421875,"verdict":{"state":"Fail"},"prompt_ids_sha256":"cb47cabf619ed11eda1601028c79ced363020fe66316b9895d5602b2f527c0d8"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"e8e5551522f3e72418df3e0fa2227dded865de9949b65a4b85becc6cd3acbc35","stratum":"2k","engine":"apr","engine_version":"apr 0.70.0 (33b5e5b24 srv-tim-001)","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":3391.826576,"prompt_ms":466.966774,"prompt_tps":822.3283140911435,"decode_tps":32.639592895441346,"input_tokens":384,"output_tokens":94,"peak_rss_mb":8377.82421875,"verdict":{"state":"Pass"},"prompt_ids_sha256":"0655ead49567576a1c526b6f710c1807619174567ee1f131159a9ae03c68b8ed"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"e0cf54bdaa7fbbbd164db0ebf846b77b0b78b65b8bfd4a392fcc3171c64a2504","stratum":"2k","engine":"apr","engine_version":"apr 0.70.0 (33b5e5b24 srv-tim-001)","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":3380.256746,"prompt_ms":667.500577,"prompt_tps":906.3662577178566,"decode_tps":31.491043600798456,"input_tokens":605,"output_tokens":84,"peak_rss_mb":8377.82421875,"verdict":{"state":"Pass"},"prompt_ids_sha256":"a7da9728eaf7dc43519a3329e56fbddffb918e60cd11e3b60c2fdd066013112c"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"7492d83b3314d90dae86dcb3530baaa502e8d425427daa63a8e9fc03de96b069","stratum":"2k","engine":"apr","engine_version":"apr 0.70.0 (33b5e5b24 srv-tim-001)","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":5955.61306,"prompt_ms":1359.542642,"prompt_tps":952.5262099134658,"decode_tps":28.13134550711303,"input_tokens":1295,"output_tokens":128,"peak_rss_mb":8377.82421875,"verdict":{"state":"Pass"},"prompt_ids_sha256":"3710b42517c7bd8dcd003e4e02e46f2547485a103ea66db61273afabb7a3c2bf"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"0d9bcac1ca95dc9fb8293713b22add78150aaaacfa6a269e8065ea5605feee17","stratum":"2k","engine":"apr","engine_version":"apr 0.70.0 (33b5e5b24 srv-tim-001)","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":10199.421675,"prompt_ms":856.637168,"prompt_tps":1125.3305786984015,"decode_tps":29.256455058437705,"input_tokens":964,"output_tokens":272,"peak_rss_mb":8377.82421875,"verdict":{"state":"Fail"},"prompt_ids_sha256":"5ddc4f9e03e618df9e68b2df8ad867702dff23019b64b9d2ef32f96ad0949717"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"3f51712000a76356352961b38d5b137cdf5179679655ed3fbfef2acad4eeb0e8","stratum":"2k","engine":"apr","engine_version":"apr 0.70.0 (33b5e5b24 srv-tim-001)","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":4971.072349,"prompt_ms":693.098059,"prompt_tps":992.6445342995833,"decode_tps":30.86741683575437,"input_tokens":688,"output_tokens":131,"peak_rss_mb":8377.82421875,"verdict":{"state":"Pass"},"prompt_ids_sha256":"d7e97aaa3ac6609d18fe9c3ba27bfcd91550e16482bf475e2d4f19eb26acd77a"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"b7e5b398d98a7f3dc848eee0e437e07104d372e9365e6f3ce0c101e6667ed890","stratum":"2k","engine":"apr","engine_version":"apr 0.70.0 (33b5e5b24 srv-tim-001)","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":2875.8521880000003,"prompt_ms":440.676141,"prompt_tps":698.9259715787517,"decode_tps":33.053495163145236,"input_tokens":308,"output_tokens":79,"peak_rss_mb":8377.87890625,"verdict":{"state":"Pass"},"prompt_ids_sha256":"b05019b1e8f3ab2be02e8b81733a49d390f98ae5a16b0e8f13d09593ea04b9e6"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"903ef35fb09bf1cb357547ce61b9b7828b90c24845aa5f9fecd8923e022be75f","stratum":"2k","engine":"apr","engine_version":"apr 0.70.0 (33b5e5b24 srv-tim-001)","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":7212.154421,"prompt_ms":850.766113,"prompt_tps":1127.2193207347457,"decode_tps":29.44517455865528,"input_tokens":959,"output_tokens":186,"peak_rss_mb":8377.87890625,"verdict":{"state":"Fail"},"prompt_ids_sha256":"ec9c7f9e8c70ed80adfbcb818a9db16b1472d69cc4b7d5152bd8e139204963b5"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"34186bc542a9bc6191d88846aadf532b79f67209e4b17474d1d4b0a1cf87cc8c","stratum":"2k","engine":"apr","engine_version":"apr 0.70.0 (33b5e5b24 srv-tim-001)","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":4453.949460999999,"prompt_ms":624.492764,"prompt_tps":842.2835784851464,"decode_tps":31.709050143551693,"input_tokens":526,"output_tokens":120,"peak_rss_mb":8377.87890625,"verdict":{"state":"Pass"},"prompt_ids_sha256":"717745ed7fb623d700dffd2b1f7f0d7ae241a0fc248f74ed4b935313a1a249fc"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"c9a02d51ce74d1c65af0fa596da511b8d2a571aa8f65f8d380ed9838b4c5243e","stratum":"2k","engine":"apr","engine_version":"apr 0.70.0 (33b5e5b24 srv-tim-001)","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":11134.561870000001,"prompt_ms":915.61852,"prompt_tps":1117.2775316951868,"decode_tps":28.89875301602561,"input_tokens":1023,"output_tokens":294,"peak_rss_mb":8377.87890625,"verdict":{"state":"Fail"},"prompt_ids_sha256":"0c2f2e99acdaace2521543f52e497a9e01039e659d7c95ffb47efa75d6a0e742"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"d09cb9713ee02abec5dce5dd4cd0b58237e449a0032fdaf96dcc6ac95a0a6101","stratum":"2k","engine":"apr","engine_version":"apr 0.70.0 (33b5e5b24 srv-tim-001)","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":4286.006193,"prompt_ms":543.618198,"prompt_tps":890.3307537912849,"decode_tps":31.919910542340745,"input_tokens":484,"output_tokens":118,"peak_rss_mb":8377.87890625,"verdict":{"state":"Pass"},"prompt_ids_sha256":"6ab504c554d5d36c516e432f36c59eac000d1d9af5d19121bcc80d31a4dd1ace"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"a889b6ddfa18809d32510029c8f832ddf1dc03f3209feee377d47043c757104e","stratum":"2k","engine":"apr","engine_version":"apr 0.70.0 (33b5e5b24 srv-tim-001)","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":3554.99833,"prompt_ms":648.959319,"prompt_tps":930.7208977763364,"decode_tps":31.462870440061018,"input_tokens":604,"output_tokens":90,"peak_rss_mb":8377.87890625,"verdict":{"state":"Pass"},"prompt_ids_sha256":"909814ad448169a742c52875b82d6ef865c02703afda46a7450ffbe8394e794a"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"c1d14d2cd0dbbdd90638158ee715c79fc76baa8ff2c6322160dac0bb5bd27771","stratum":"2k","engine":"apr","engine_version":"apr 0.70.0 (33b5e5b24 srv-tim-001)","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":6652.212727,"prompt_ms":434.303419,"prompt_tps":764.4425198503906,"decode_tps":32.56152624027801,"input_tokens":332,"output_tokens":201,"peak_rss_mb":8377.87890625,"verdict":{"state":"Fail"},"prompt_ids_sha256":"82eb2fb253fce96176b306689444000ece53e63e156d9fa4863a938a3e705b71"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"955705510e7439ac4b8e27505f87bcb7d3c71b677cb11bddc4a91d5c44e16a04","stratum":"2k","engine":"apr","engine_version":"apr 0.70.0 (33b5e5b24 srv-tim-001)","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":12053.597645,"prompt_ms":1837.119178,"prompt_tps":975.440255297362,"decode_tps":25.95730293564044,"input_tokens":1792,"output_tokens":264,"peak_rss_mb":8377.921875,"verdict":{"state":"Fail"},"prompt_ids_sha256":"51eeb6fcedffb81fd02fc9c5f05805cb72211d244fb56aad4117e964b187a2d0"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"bf94290e1f98fc502cb4f962bce2883c3eaa9ca2fe6cb67f6b87c7083d3ec0e1","stratum":"2k","engine":"apr","engine_version":"apr 0.70.0 (33b5e5b24 srv-tim-001)","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":10289.948218,"prompt_ms":1318.932296,"prompt_tps":965.9328260167192,"decode_tps":27.898807316777244,"input_tokens":1274,"output_tokens":249,"peak_rss_mb":8377.921875,"verdict":{"state":"Fail"},"prompt_ids_sha256":"05a06916a4b3576cd25ee421c0588937e59d599607237eeab47bec5ef1db27eb"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"258be4b71904b9f67bdda4ea10d76b9be53c374b04a9d486facd87a97fab0756","stratum":"2k","engine":"apr","engine_version":"apr 0.70.0 (33b5e5b24 srv-tim-001)","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":5964.523484,"prompt_ms":1276.8722750000002,"prompt_tps":949.9775535497472,"decode_tps":28.216909450225607,"input_tokens":1213,"output_tokens":131,"peak_rss_mb":8377.921875,"verdict":{"state":"Pass"},"prompt_ids_sha256":"d1fbddde370dc12c4aec16fbd2fc2fb32418489c008dd55dd3d29e40a5fbd855"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"1bf59f2a1774a0efe188bbfa1ac4a7af14b2059017de009fc98df2a6b8382017","stratum":"2k","engine":"apr","engine_version":"apr 0.70.0 (33b5e5b24 srv-tim-001)","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":14351.487605999999,"prompt_ms":1856.854463,"prompt_tps":964.5343971149988,"decode_tps":25.705464294593934,"input_tokens":1791,"output_tokens":320,"peak_rss_mb":8377.921875,"verdict":{"state":"Fail"},"prompt_ids_sha256":"6c5a3de9322024e61926b8105522954447661224770b9881c91a10be3aa69a7c"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"305cdbe50000c485ff10ec6127fbe0744109a126be589ccbb6d243834c3c13ef","stratum":"2k","engine":"apr","engine_version":"apr 0.70.0 (33b5e5b24 srv-tim-001)","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":7910.693473,"prompt_ms":991.912516,"prompt_tps":1046.4632548300258,"decode_tps":28.949476771019196,"input_tokens":1038,"output_tokens":199,"peak_rss_mb":8377.921875,"verdict":{"state":"Fail"},"prompt_ids_sha256":"0d2fc4877c6ceccc5bcecc9e02a8dc8468a8265069ac9e54235fb1f170a59c3e"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"7680438d94dac7e6c61561a71575435fcd22d1543b508ec9434d00f686511896","stratum":"2k","engine":"apr","engine_version":"apr 0.70.0 (33b5e5b24 srv-tim-001)","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":7389.679997,"prompt_ms":848.3471089999999,"prompt_tps":1083.282998492543,"decode_tps":29.555201964063357,"input_tokens":919,"output_tokens":192,"peak_rss_mb":8377.921875,"verdict":{"state":"Fail"},"prompt_ids_sha256":"0ab5ac2510b12bcacb54e31107686f33fa3f624fb80ca84c81fbced4c8e24f46"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"d928b82fd55ca6c509650e822de0f3946d5ccb3d5fbbabba30784e0f6483e2ff","stratum":"2k","engine":"apr","engine_version":"apr 0.70.0 (33b5e5b24 srv-tim-001)","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":8032.185837,"prompt_ms":1453.445117,"prompt_tps":982.4932385114614,"decode_tps":26.186957541385883,"input_tokens":1428,"output_tokens":171,"peak_rss_mb":8377.921875,"verdict":{"state":"Fail"},"prompt_ids_sha256":"ef93f1ea44b6a4051e70202045ed0b6322e8987557045343e51ca8cdc8074df0"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"64cce1452c38a191c5f249f069acb21eb4098bd2620cddec04b346c3075849dc","stratum":"2k","engine":"apr","engine_version":"apr 0.70.0 (33b5e5b24 srv-tim-001)","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":7295.269584,"prompt_ms":702.454152,"prompt_tps":872.6548177623982,"decode_tps":29.210051175550863,"input_tokens":613,"output_tokens":191,"peak_rss_mb":8379.921875,"verdict":{"state":"Fail"},"prompt_ids_sha256":"337e04b1920ed7b30ceedf4ff49f4bf841d2ce7657bafdb2a9165eea27d39ea6"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"69b831cc2059acf0f511c8d788337f36e8ef890f6b86b91f6ffa159b8369d1e5","stratum":"2k","engine":"apr","engine_version":"apr 0.70.0 (33b5e5b24 srv-tim-001)","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":8346.050830999999,"prompt_ms":2160.284564,"prompt_tps":897.566937389828,"decode_tps":24.737658216882508,"input_tokens":1939,"output_tokens":152,"peak_rss_mb":8379.96875,"verdict":{"state":"Pass"},"prompt_ids_sha256":"d93d068e263876cd1cb37558b0015f0fdcf11be3da9eefe2c4aff454b8a9e616"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"14d125f8a319774d2ddc89c330d806c3371695a96ec1cc8757cb63f20c9afe37","stratum":"2k","engine":"apr","engine_version":"apr 0.70.0 (33b5e5b24 srv-tim-001)","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":6113.531546,"prompt_ms":663.319294,"prompt_tps":872.8827960188959,"decode_tps":30.895538601769967,"input_tokens":579,"output_tokens":167,"peak_rss_mb":8379.96875,"verdict":{"state":"Pass"},"prompt_ids_sha256":"04222af6320ccf0f9b9bc386e01fc9ed98736d6df5ee61bd98894f8facc87347"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"0b54a849b9e7541ec803e57fab1147aee8d209935102b9230c0a024229ccdac7","stratum":"2k","engine":"apr","engine_version":"apr 0.70.0 (33b5e5b24 srv-tim-001)","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":4977.429227,"prompt_ms":675.3228409999999,"prompt_tps":836.6368878673838,"decode_tps":31.469218251128016,"input_tokens":565,"output_tokens":134,"peak_rss_mb":8379.96875,"verdict":{"state":"Pass"},"prompt_ids_sha256":"6ea89d378295c73f8cfc8d17f5f2fcab76f6abb4e8c59463e8a4a4025885fed7"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"ae0f31c8b5b43715e0f5f31727de5648fe149c06faef682adb8898f4dddc0d59","stratum":"8k","engine":"apr","engine_version":"apr 0.70.0 (33b5e5b24 srv-tim-001)","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":11612.340682,"prompt_ms":2413.867093,"prompt_tps":894.4154407924564,"decode_tps":24.69453573093115,"input_tokens":2159,"output_tokens":226,"peak_rss_mb":8379.96875,"verdict":{"state":"Pass"},"prompt_ids_sha256":"a81f7ea1e1114b3b88a0717effd69745ef46e82f2efdcdbd465afae931ebf1f0"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"6324ab9b418859326df39301f2fc9ca30815280479d95ad375d91e6252a4b82b","stratum":"8k","engine":"apr","engine_version":"apr 0.70.0 (33b5e5b24 srv-tim-001)","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":17138.478285,"prompt_ms":3660.833985,"prompt_tps":976.0071105764716,"decode_tps":20.846927585340634,"input_tokens":3573,"output_tokens":280,"peak_rss_mb":8380.0859375,"verdict":{"state":"Pass"},"prompt_ids_sha256":"3ae7c4e045aeaa9df4ea1cf4961254d625990b616146ceda3ec66666473b7cff"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"187723585ce31ae0a9dfb2bd1c94ee372724ba456ade6c3ef28816fb1e369e4d","stratum":"8k","engine":"apr","engine_version":"apr 0.70.0 (33b5e5b24 srv-tim-001)","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":7745.897397000001,"prompt_ms":2448.28185,"prompt_tps":925.5470320951814,"decode_tps":24.564394451801324,"input_tokens":2266,"output_tokens":129,"peak_rss_mb":8380.0859375,"verdict":{"state":"Pass"},"prompt_ids_sha256":"eb266f1f82f6cd93fd1590494e67349a0bed128c01781f180d09e71b40998f5d"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"dd16771d607cda7d13b2528bec93be7261e9d739a6d40c855b0794500da92522","stratum":"8k","engine":"apr","engine_version":"apr 0.70.0 (33b5e5b24 srv-tim-001)","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":22535.415436,"prompt_ms":5051.472294,"prompt_tps":981.4960295613174,"decode_tps":18.10437639624086,"input_tokens":4958,"output_tokens":314,"peak_rss_mb":8380.125,"verdict":{"state":"Fail"},"prompt_ids_sha256":"51ee3f91d5eaa6d044852fa83079495964304a501fbc737ecc01becf473718b6"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"40ce4b133ac399054d67393d53e91be97d3cafaa33622094f90a35fdea53c703","stratum":"8k","engine":"apr","engine_version":"apr 0.70.0 (33b5e5b24 srv-tim-001)","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":29944.102564,"prompt_ms":7487.316136,"prompt_tps":927.03445051916,"decode_tps":14.727009389844794,"input_tokens":6941,"output_tokens":330,"peak_rss_mb":8378.09765625,"verdict":{"state":"Pass"},"prompt_ids_sha256":"212c625213d37b2c9f816b691908099bd9f9a58e680ca2d15863dcdabb735b51"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"bc9a159757fe4720bfdc9073b5c2d8879c8adec829ab9308b1b1031768b5c3f0","stratum":"8k","engine":"apr","engine_version":"apr 0.70.0 (33b5e5b24 srv-tim-001)","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":30221.190556,"prompt_ms":7161.160583,"prompt_tps":910.46694518734,"decode_tps":15.682991600271546,"input_tokens":6520,"output_tokens":361,"peak_rss_mb":8378.09765625,"verdict":{"state":"Pass"},"prompt_ids_sha256":"966d327b207b9dda0ee074cf93126d6091ceef81459965ec9b5b1a03824d5f5f"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"33b63a5283685481a92f5226d573249f59330892462fc75a3c5ff14e9c61ba82","stratum":"8k","engine":"apr","engine_version":"apr 0.70.0 (33b5e5b24 srv-tim-001)","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":30326.520684,"prompt_ms":7355.60702,"prompt_tps":923.2412745182248,"decode_tps":15.303889545629366,"input_tokens":6791,"output_tokens":349,"peak_rss_mb":8388.87890625,"verdict":{"state":"Pass"},"prompt_ids_sha256":"8d3561cb81010d0287b8d36013ce4022187e868e05fb5a2acba13d93ddd0ba75"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"df48ffe5f40ed18ea677fa829b51ee4eaaee5c8caa5a7462bbe212d83396c540","stratum":"8k","engine":"apr","engine_version":"apr 0.70.0 (33b5e5b24 srv-tim-001)","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":18360.230726,"prompt_ms":3707.221216,"prompt_tps":928.727960754096,"decode_tps":21.061255318956363,"input_tokens":3443,"output_tokens":306,"peak_rss_mb":8388.87890625,"verdict":{"state":"Pass"},"prompt_ids_sha256":"513d5ac32c370db50606bba64bf9369a630e11ff0935f649326b6b1082750baf"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"cbc1450d830a7bda671630905c4b86039bf483e305a0d8fecf6971de02ad0406","stratum":"8k","engine":"apr","engine_version":"apr 0.70.0 (33b5e5b24 srv-tim-001)","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":22316.924506,"prompt_ms":7279.77222,"prompt_tps":931.2104548238186,"decode_tps":15.14699570129997,"input_tokens":6779,"output_tokens":227,"peak_rss_mb":8388.87890625,"verdict":{"state":"Pass"},"prompt_ids_sha256":"c9b299004f9b7cc8d7af8cf6262f62e552c1752be74d07b00d51c812552f258f"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"62b4bd91655d82632f2ad6754eac98e25881dcdfeb3eff15172402b9da179800","stratum":"8k","engine":"apr","engine_version":"apr 0.70.0 (33b5e5b24 srv-tim-001)","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":20140.593671000002,"prompt_ms":4234.218469,"prompt_tps":924.3736544667177,"decode_tps":19.86582770945326,"input_tokens":3914,"output_tokens":315,"peak_rss_mb":8388.87890625,"verdict":{"state":"Pass"},"prompt_ids_sha256":"1d6f64f5e542ef9606b341f0aa9e816aad85f379d21ff095d0138038b8b4ba4a"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"a6c9bb4b1ea341e343c764aed2b694ca7f4c6ab69461291cf6e20a2d1fe53a47","stratum":"8k","engine":"apr","engine_version":"apr 0.70.0 (33b5e5b24 srv-tim-001)","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":15647.535002999999,"prompt_ms":3085.890464,"prompt_tps":985.4529950030008,"decode_tps":22.131733403213573,"input_tokens":3041,"output_tokens":277,"peak_rss_mb":8388.87890625,"verdict":{"state":"Fail"},"prompt_ids_sha256":"a5e089a434b2f460a61b44fe52bddb2e49e8dc5668c0e6de4a0f0bad9f8fe848"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"1239ceb996f617d8cda28ed536075072f460a875c5a4a25a2ef422a9d923043e","stratum":"8k","engine":"apr","engine_version":"apr 0.70.0 (33b5e5b24 srv-tim-001)","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":12390.281664,"prompt_ms":4770.414135,"prompt_tps":901.3892459464632,"decode_tps":18.755226483137612,"input_tokens":4300,"output_tokens":142,"peak_rss_mb":8388.87890625,"verdict":{"state":"Pass"},"prompt_ids_sha256":"f276cbcdd3b7b2349836fbdb1d397d0d986373840f6ddaddc46c581e5f7288df"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"90134b8a8a941ee22156f752e986d85a39c92f11186fff94174ce25e7fc2c65d","stratum":"8k","engine":"apr","engine_version":"apr 0.70.0 (33b5e5b24 srv-tim-001)","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":17215.590626,"prompt_ms":3296.892671,"prompt_tps":953.0185885750964,"decode_tps":21.400259043348164,"input_tokens":3142,"output_tokens":297,"peak_rss_mb":8388.87890625,"verdict":{"state":"Fail"},"prompt_ids_sha256":"1fb374c347b9be341a38517cceab0992299ad2aaea74befbaf3906626c37d34b"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"b3c917322e600c6a75b1fdabd254bbb864628227545b9f51c05fedcb33835e55","stratum":"8k","engine":"apr","engine_version":"apr 0.70.0 (33b5e5b24 srv-tim-001)","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":15124.292465,"prompt_ms":2818.518129,"prompt_tps":973.916034726346,"decode_tps":22.920853614306186,"input_tokens":2745,"output_tokens":281,"peak_rss_mb":8388.87890625,"verdict":{"state":"Fail"},"prompt_ids_sha256":"a24ccc6f60289e1cfdec88d1bedae95fd3e07227a8da2a84b31e646d817ee93c"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"38f0a202313d9e2120e55b92747ced425cac793e09a4db44c212449b1cd8566e","stratum":"8k","engine":"apr","engine_version":"apr 0.70.0 (33b5e5b24 srv-tim-001)","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":32134.260489,"prompt_ms":4703.574707,"prompt_tps":975.4283254334202,"decode_tps":18.69750013090688,"input_tokens":4588,"output_tokens":512,"peak_rss_mb":8388.87890625,"verdict":{"state":"Fail"},"prompt_ids_sha256":"95b34da10969f03fa2337931389f1835b5efc1eb5f79585e13cb239ef9503d00"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"e12a747f914c805b9dcbc4349d5ed7fe706e6d0334848a677837cef3d8b95400","stratum":"8k","engine":"apr","engine_version":"apr 0.70.0 (33b5e5b24 srv-tim-001)","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":9330.948619,"prompt_ms":2569.615231,"prompt_tps":960.065915798582,"decode_tps":23.97464039139119,"input_tokens":2467,"output_tokens":161,"peak_rss_mb":8388.87890625,"verdict":{"state":"Pass"},"prompt_ids_sha256":"0b4a3d5a626c23d2a0a7f227b0d4edfddd53b4da5a6bb9963171c2d1ab882b65"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"a74db9b8ff37392fa217b05b5e70a2564d3c8a6960f02eebe3db8016184e114a","stratum":"8k","engine":"apr","engine_version":"apr 0.70.0 (33b5e5b24 srv-tim-001)","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":13059.246925,"prompt_ms":6151.38781,"prompt_tps":969.0496167888332,"decode_tps":16.764606787233863,"input_tokens":5961,"output_tokens":115,"peak_rss_mb":8388.87890625,"verdict":{"state":"Pass"},"prompt_ids_sha256":"ad6c508530582a081bce003291bb7ee3c17effe3ed0527e73172c39694fbc89b"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"27565a444276a49af4557cade6d92398cbccd183c6366e78a5453683923bfe6b","stratum":"8k","engine":"apr","engine_version":"apr 0.70.0 (33b5e5b24 srv-tim-001)","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":23075.344959,"prompt_ms":8470.244637,"prompt_tps":929.7252130830044,"decode_tps":14.31411902900302,"input_tokens":7875,"output_tokens":205,"peak_rss_mb":8388.87890625,"verdict":{"state":"Pass"},"prompt_ids_sha256":"72815a6948e2f27d022cec776bb31fd4a99af1123e5c14a25c6fcb175a28e00a"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"e0d0c14ea617d00ecec166c8e598898b3d1b757d639df1729ddf1e3af37872cd","stratum":"8k","engine":"apr","engine_version":"apr 0.70.0 (33b5e5b24 srv-tim-001)","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":9295.348246,"prompt_ms":2986.703985,"prompt_tps":946.5283517208017,"decode_tps":22.95252564786544,"input_tokens":2827,"output_tokens":144,"peak_rss_mb":8388.87890625,"verdict":{"state":"Pass"},"prompt_ids_sha256":"5019b3cebd26be737bb7fe45f6ddb10faa0e2c3bb6fd157fa1eee1deea81d19d"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"7269a7b0e6abf53a633d344677435b768b4342ad507d8833da2aa1b5b90747ee","stratum":"8k","engine":"apr","engine_version":"apr 0.70.0 (33b5e5b24 srv-tim-001)","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":18627.702452,"prompt_ms":4833.306061,"prompt_tps":959.798519161065,"decode_tps":18.76694678890109,"input_tokens":4639,"output_tokens":258,"peak_rss_mb":8388.87890625,"verdict":{"state":"Pass"},"prompt_ids_sha256":"b76967bf52a3d1d50f5eeb855e43df852e8ff8be2ed8bb4909ea519ca09fa086"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"f87c2ffe3aafeec5f8308352f5190b3c8ec03ed0eafd6d94b000ef39bb3d922f","stratum":"8k","engine":"apr","engine_version":"apr 0.70.0 (33b5e5b24 srv-tim-001)","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":33466.536628999995,"prompt_ms":4959.992188,"prompt_tps":1003.02577331398,"decode_tps":17.983819074804284,"input_tokens":4975,"output_tokens":512,"peak_rss_mb":8388.87890625,"verdict":{"state":"Fail"},"prompt_ids_sha256":"805c7c9887566084d332a0ff4e3decc7ee4a100a69194e545f3460c67c1fb92a"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"c7323e9dc8cc06122998a5fb83ccb487d2538b7444f9852de80f63983e93dc44","stratum":"8k","engine":"apr","engine_version":"apr 0.70.0 (33b5e5b24 srv-tim-001)","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":26687.22925,"prompt_ms":3855.980054,"prompt_tps":987.5569755735048,"decode_tps":20.18989685259704,"input_tokens":3808,"output_tokens":460,"peak_rss_mb":8388.87890625,"verdict":{"state":"Fail"},"prompt_ids_sha256":"b5cb29cd4d1d2b02437000ebf0115b5483e5b7bf00187bc0d72bb47ba8be7019"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"3bed1317707a01f00daf828a5434d3681278c66100d991520153baa4dc7ebd61","stratum":"8k","engine":"apr","engine_version":"apr 0.70.0 (33b5e5b24 srv-tim-001)","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":14387.783919,"prompt_ms":2293.6487159999997,"prompt_tps":900.3122342122421,"decode_tps":24.98249575912652,"input_tokens":2065,"output_tokens":301,"peak_rss_mb":8388.87890625,"verdict":{"state":"Fail"},"prompt_ids_sha256":"92f0703d894d6fb99af6cf992d9c3d76abd4fad16e00adf056499d5bf79df42a"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"a974a820da6ee897219e5353ad22358cd5cec02a3534baa60388a3f554850d96","stratum":"8k","engine":"apr","engine_version":"apr 0.70.0 (33b5e5b24 srv-tim-001)","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":13124.918631,"prompt_ms":4466.532641,"prompt_tps":971.894834071582,"decode_tps":19.39315934905969,"input_tokens":4341,"output_tokens":167,"peak_rss_mb":8388.87890625,"verdict":{"state":"Fail"},"prompt_ids_sha256":"4017ef5b163960754cdb6c87147d51d9110da0dcbb8d7f37fc5b1b1af7c2f8d4"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"1c50664db77045b861287a6f03ded90d514b8d0a32f04336caa5c9c90777b18c","stratum":"8k","engine":"apr","engine_version":"apr 0.70.0 (33b5e5b24 srv-tim-001)","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":11743.645926,"prompt_ms":2419.837566,"prompt_tps":916.1771976557536,"decode_tps":24.57569569597515,"input_tokens":2217,"output_tokens":228,"peak_rss_mb":8388.87890625,"verdict":{"state":"Pass"},"prompt_ids_sha256":"ad1b4023aa37427996cc325caba661c60a9cf8343640671dcb1983cfbc5c89b8"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"7e09255545a53208daf6239daa699089bbd30c3ec111d44a59cbe007a8004a57","stratum":"8k","engine":"apr","engine_version":"apr 0.70.0 (33b5e5b24 srv-tim-001)","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":11757.30303,"prompt_ms":3972.386014,"prompt_tps":986.5607184669742,"decode_tps":20.291119173412365,"input_tokens":3919,"output_tokens":157,"peak_rss_mb":8388.87890625,"verdict":{"state":"Pass"},"prompt_ids_sha256":"7979c0667fd9abec64e255baa8d0377e7856b52e4e008e1f1b7c35c4471ccdf2"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"1aee819c753ccc3dd6976511c879d33fd99478a25bfe8c5639d006aebe6d00ad","stratum":"8k","engine":"apr","engine_version":"apr 0.70.0 (33b5e5b24 srv-tim-001)","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":13659.34933,"prompt_ms":4866.247489,"prompt_tps":987.6193125943169,"decode_tps":18.523128666495744,"input_tokens":4806,"output_tokens":162,"peak_rss_mb":8388.87890625,"verdict":{"state":"Pass"},"prompt_ids_sha256":"93d0c8edbd6a7cca000f9f23b65b7bd49a90c3e64f4d169cf4705ff14a6bbaf9"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"e92967237c1b1abeb4a548c343928a0afda2c3df6c49e94cec87b7146276b045","stratum":"8k","engine":"apr","engine_version":"apr 0.70.0 (33b5e5b24 srv-tim-001)","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":19764.072438,"prompt_ms":4087.519588,"prompt_tps":984.9494083941256,"decode_tps":19.833735121577885,"input_tokens":4026,"output_tokens":310,"peak_rss_mb":8388.87890625,"verdict":{"state":"Fail"},"prompt_ids_sha256":"96c30189e094a2ed3603508cd4222ab22c468239354545620eebf61674389fb1"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"5ef3cfe455ce3ddedb5fd853a62ccbda685b16f06e7ec8818c46f639474fd302","stratum":"8k","engine":"apr","engine_version":"apr 0.70.0 (33b5e5b24 srv-tim-001)","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":9678.957584,"prompt_ms":2740.561171,"prompt_tps":1027.1618928954022,"decode_tps":22.92365275563612,"input_tokens":2815,"output_tokens":158,"peak_rss_mb":8388.87890625,"verdict":{"state":"Pass"},"prompt_ids_sha256":"bc74840b53b9b199026a3b937a88491510f44071bb5e597f5a05fc460493e8e5"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"08db5d324448f2faa678eca074285b6275f42db2c667f594ede9fa4adfc9c566","stratum":"8k","engine":"apr","engine_version":"apr 0.70.0 (33b5e5b24 srv-tim-001)","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":20824.348449999998,"prompt_ms":5888.398872,"prompt_tps":967.665425502106,"decode_tps":16.992760817313695,"input_tokens":5698,"output_tokens":253,"peak_rss_mb":8388.87890625,"verdict":{"state":"Pass"},"prompt_ids_sha256":"759d47a6fcd52490cc43e3f9f91db6587e8b59bf2ed44a06dae3647f1cab6df2"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"bc995711883be2b01bb079a4433bb34b0eb50a9cda6440856b8c0665d7a4afb5","stratum":"8k","engine":"apr","engine_version":"apr 0.70.0 (33b5e5b24 srv-tim-001)","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":8678.970206,"prompt_ms":2723.2999680000003,"prompt_tps":1026.3283636920307,"decode_tps":23.013342620958877,"input_tokens":2795,"output_tokens":136,"peak_rss_mb":8388.87890625,"verdict":{"state":"Pass"},"prompt_ids_sha256":"6a64782f6b63fcb21b6814ed2634f945270c9fb500f9ec47abffff02026b3b33"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"9f29c99424409b984aeea5524ec870e016e2e16c617351a938ed700a057ce9ba","stratum":"8k","engine":"apr","engine_version":"apr 0.70.0 (33b5e5b24 srv-tim-001)","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":25781.306262000002,"prompt_ms":7884.598138,"prompt_tps":949.0654905968526,"decode_tps":14.66990494299252,"input_tokens":7483,"output_tokens":262,"peak_rss_mb":8388.87890625,"verdict":{"state":"Pass"},"prompt_ids_sha256":"f79d101859efe06916d34257165ec054aa373990f5d969fc6c2513c9ee1dd354"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"3d765c54312ef5c279e895fb042a9c08ef9bcd6b44e1098b9db17f68d379f845","stratum":"8k","engine":"apr","engine_version":"apr 0.70.0 (33b5e5b24 srv-tim-001)","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":20938.742274,"prompt_ms":3510.937415,"prompt_tps":1000.8722983744784,"decode_tps":20.94148351184443,"input_tokens":3514,"output_tokens":364,"peak_rss_mb":8388.87890625,"verdict":{"state":"Pass"},"prompt_ids_sha256":"afa4aed5cdc97dd82a2f3c6f1c3cf83de33f5a873bda726cf8cbedecc68e184a"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"b66cef24141dfcf0cd4776c23e13ac2a297f86fb966ea2f474d3ba2b22ac4689","stratum":"8k","engine":"apr","engine_version":"apr 0.70.0 (33b5e5b24 srv-tim-001)","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":16523.652713,"prompt_ms":2382.510195,"prompt_tps":928.852268772768,"decode_tps":24.47445745838344,"input_tokens":2213,"output_tokens":345,"peak_rss_mb":8388.87890625,"verdict":{"state":"Pass"},"prompt_ids_sha256":"a7ed93b0333e5ba437c439c1d28332f7c2c21e02f641095bc38eadd1a02cd167"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"bf69f8045076d4b114aef30013614158379bdcec1dddb96497b724fd4bd16df2","stratum":"8k","engine":"apr","engine_version":"apr 0.70.0 (33b5e5b24 srv-tim-001)","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":16236.201189,"prompt_ms":4820.691847,"prompt_tps":986.7878202918868,"decode_tps":18.560835637124335,"input_tokens":4757,"output_tokens":211,"peak_rss_mb":8388.87890625,"verdict":{"state":"Pass"},"prompt_ids_sha256":"ef2a1c7ad19234288174165afe4ff5dc308ed4a5d5448495732bf50390f28fde"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"0ba74445847c37901a423c32a811b0f15026d75d3dd2225fd67ab1a474f97ae2","stratum":"8k","engine":"apr","engine_version":"apr 0.70.0 (33b5e5b24 srv-tim-001)","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":18710.470708,"prompt_ms":5691.455492,"prompt_tps":972.1590562866164,"decode_tps":17.266349825190503,"input_tokens":5533,"output_tokens":224,"peak_rss_mb":8388.87890625,"verdict":{"state":"Pass"},"prompt_ids_sha256":"5fc6cf167f33d8897f20f6457b0677a778f646264bdb552be4e00fb52b0e6b9f"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"a08ed5ccc3331e0a7179c7d49737607bccb9a9cfbc16c6533afacfa6072b133f","stratum":"8k","engine":"apr","engine_version":"apr 0.70.0 (33b5e5b24 srv-tim-001)","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":6745.6151310000005,"prompt_ms":2445.905549,"prompt_tps":952.6124183137866,"decode_tps":24.44355625022262,"input_tokens":2330,"output_tokens":104,"peak_rss_mb":8388.87890625,"verdict":{"state":"Pass"},"prompt_ids_sha256":"baf2d30022d1dbc62e43697b26bd1999ee610fcd6b4275293cd352e091390a04"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"b0bdb087e1d95680866dad2b47061fa38e925929fe937f39b834992f44838399","stratum":"8k","engine":"apr","engine_version":"apr 0.70.0 (33b5e5b24 srv-tim-001)","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":21331.286413,"prompt_ms":6544.192777,"prompt_tps":939.1532629662936,"decode_tps":16.35095924387899,"input_tokens":6146,"output_tokens":241,"peak_rss_mb":8388.87890625,"verdict":{"state":"Pass"},"prompt_ids_sha256":"62861c84c3f9994a8f69de8848af2f7ea98e8e6de26bf95f24f760b6c5975b69"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"6e56cee326693c4915a7feef43eee6a597f75ff2e54f970e8f3197135869aed1","stratum":"8k","engine":"apr","engine_version":"apr 0.70.0 (33b5e5b24 srv-tim-001)","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":26978.345699999998,"prompt_ms":7276.132129,"prompt_tps":967.8218970121728,"decode_tps":15.162276464682964,"input_tokens":7042,"output_tokens":298,"peak_rss_mb":8388.87890625,"verdict":{"state":"Pass"},"prompt_ids_sha256":"6383d5bff555ad33b2339b6120920cf9522aa807a1285f41b3efcface1525924"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"c81288783ae98e2b5a143cdfc4738ce6c817ae09e2b01a0979e18a0de5f90bb6","stratum":"8k","engine":"apr","engine_version":"apr 0.70.0 (33b5e5b24 srv-tim-001)","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":19246.056531000002,"prompt_ms":7781.529791,"prompt_tps":951.8693880176138,"decode_tps":14.788777755574154,"input_tokens":7407,"output_tokens":169,"peak_rss_mb":8388.87890625,"verdict":{"state":"Pass"},"prompt_ids_sha256":"3c18df6c64b92c1d238f0b099ce92e3e85ec2f7c4a8a113995df1d4036e878bd"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"0354fce0d010d6d69a1539589a7d6046495172ee6865f418b0a39c852edae583","stratum":"8k","engine":"apr","engine_version":"apr 0.70.0 (33b5e5b24 srv-tim-001)","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":12195.691931000001,"prompt_ms":4096.679061,"prompt_tps":986.8969328080836,"decode_tps":19.965627255483422,"input_tokens":4043,"output_tokens":161,"peak_rss_mb":8388.87890625,"verdict":{"state":"Pass"},"prompt_ids_sha256":"04371fb2a9cfff6184614b3943d1f0afebf2446e3ae21acce139cb914a34632b"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"b45b72bc1a202cb0504dfd27dd16f60af1ec09570fe31e7ecd927168837a02cf","stratum":"8k","engine":"apr","engine_version":"apr 0.70.0 (33b5e5b24 srv-tim-001)","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":18603.742293,"prompt_ms":6194.863018,"prompt_tps":968.0601463785264,"decode_tps":16.583902834891287,"input_tokens":5997,"output_tokens":205,"peak_rss_mb":8388.87890625,"verdict":{"state":"Pass"},"prompt_ids_sha256":"b4ac6d8b05c4c10cf4f5c3388011ec14c3513f17aefa6dc582e2dff4d2968405"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"b6dabc525c8486ab7bab78b2968c241327371b5c234ca1b8a87adb93bb8768ab","stratum":"8k","engine":"apr","engine_version":"apr 0.70.0 (33b5e5b24 srv-tim-001)","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":18122.027135,"prompt_ms":4868.857599,"prompt_tps":993.4568636785468,"decode_tps":18.399767991979612,"input_tokens":4837,"output_tokens":243,"peak_rss_mb":8388.87890625,"verdict":{"state":"Pass"},"prompt_ids_sha256":"f6a335774a9b2c8389c1c66b5157283cfdff4ebf3dead387979335582189e2cd"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"fd684a94b71725b926c4fb5be284b7c83518a537c466c552d7dbc8c9c1f64bec","stratum":"8k","engine":"apr","engine_version":"apr 0.70.0 (33b5e5b24 srv-tim-001)","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":10789.293516000002,"prompt_ms":2350.97039,"prompt_tps":972.3644371378068,"decode_tps":24.395954645663988,"input_tokens":2286,"output_tokens":205,"peak_rss_mb":8388.87890625,"verdict":{"state":"Pass"},"prompt_ids_sha256":"15033c21d4c89645a7f23d359d8c331c1cdae370c57dc97d3aa3ba5c4b935347"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"c5a7b5a62a110bae58ca259b02ea467d3dcdbde7ad3308a657e8028e63322e39","stratum":"8k","engine":"apr","engine_version":"apr 0.70.0 (33b5e5b24 srv-tim-001)","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":15810.514044,"prompt_ms":4072.060087,"prompt_tps":983.0405039403832,"decode_tps":20.01294170914728,"input_tokens":4003,"output_tokens":234,"peak_rss_mb":8388.87890625,"verdict":{"state":"Pass"},"prompt_ids_sha256":"6906e62e5d443f9ed949a4eb821a1ea194d8a3d072af1e52d8d0df23e1fe9f62"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"b71b016e3bf4e4aa82e2c46d8b1c2f137099cf683949b4c65e5900bad8de95ce","stratum":"8k","engine":"apr","engine_version":"apr 0.70.0 (33b5e5b24 srv-tim-001)","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":8807.547918999999,"prompt_ms":2276.182843,"prompt_tps":902.8272954080956,"decode_tps":25.24501771684491,"input_tokens":2055,"output_tokens":164,"peak_rss_mb":8388.87890625,"verdict":{"state":"Pass"},"prompt_ids_sha256":"24e56a79ebb675cd95f21cf12c693fbac4af6e2b009309f49d8f85d8b61164e5"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"611ff958e756f16b6143ffe425e3351e94a71034d0306f0e00ebd25696e66a31","stratum":"8k","engine":"apr","engine_version":"apr 0.70.0 (33b5e5b24 srv-tim-001)","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":12818.718582000001,"prompt_ms":3588.8421780000003,"prompt_tps":993.3565822018712,"decode_tps":21.014518723998847,"input_tokens":3565,"output_tokens":193,"peak_rss_mb":8388.87890625,"verdict":{"state":"Pass"},"prompt_ids_sha256":"ac1e1de733551d5421552bcc206f001ac1b3efa35aa1a05a6ab785fd99ff12d8"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"1f97a6ca5af77396cc4febaacda124d2f993c18d3e3b96e3e0b28ff92f20610d","stratum":"8k","engine":"apr","engine_version":"apr 0.70.0 (33b5e5b24 srv-tim-001)","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":30183.604671,"prompt_ms":8398.092494,"prompt_tps":937.9510889678468,"decode_tps":14.20758682488044,"input_tokens":7877,"output_tokens":309,"peak_rss_mb":8388.87890625,"verdict":{"state":"Pass"},"prompt_ids_sha256":"c42007b3ae5fcfb14a8d8a6f5d25753226fc0910d9bc95d65e8b75cc5e4b05f1"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"3811dbd8d38716269bb87eca6a6039f05923f1c129d3d2b09659f4cff15dfc2e","stratum":"8k","engine":"apr","engine_version":"apr 0.70.0 (33b5e5b24 srv-tim-001)","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":17858.781064,"prompt_ms":4623.565088,"prompt_tps":988.4147650177948,"decode_tps":18.88004016209457,"input_tokens":4570,"output_tokens":249,"peak_rss_mb":8388.87890625,"verdict":{"state":"Pass"},"prompt_ids_sha256":"ea94bd384af8194fc5c38a4dbcdc751afcf318dc8834c3f4560913718900fb35"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"f65d88532cd6ca202557458343fb035dbe140810881382b7ce5d8c668afda27d","stratum":"8k","engine":"apr","engine_version":"apr 0.70.0 (33b5e5b24 srv-tim-001)","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":11691.564667999999,"prompt_ms":3974.262092,"prompt_tps":996.4113861466992,"decode_tps":20.205350799639827,"input_tokens":3960,"output_tokens":155,"peak_rss_mb":8388.87890625,"verdict":{"state":"Pass"},"prompt_ids_sha256":"d01aacddaa6f22bdfb64eca2e82ca55373391a16f47162da338cdbbad3508144"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"6bcf0a0fb95dfe5e90aa366595a006bc443f7a3d9de3c86ba176541bdbbc01cc","stratum":"16k","engine":"apr","engine_version":"apr 0.70.0 (33b5e5b24 srv-tim-001)","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":45078.929466,"prompt_ms":18469.936747,"prompt_tps":853.061936043673,"decode_tps":8.999769323523433,"input_tokens":15756,"output_tokens":239,"peak_rss_mb":8388.87890625,"verdict":{"state":"Pass"},"prompt_ids_sha256":"e898c162c9392f2d3bd457479d776fac6602c99eba6b04a3d80e8cecc0c2cc92"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"c70e109e3c965254163267ebeee325ed9dfebe4479eb874c38a592a3488af34e","stratum":"16k","engine":"apr","engine_version":"apr 0.70.0 (33b5e5b24 srv-tim-001)","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":33712.61027,"prompt_ms":12157.838314,"prompt_tps":914.0605190647382,"decode_tps":11.485449446260189,"input_tokens":11113,"output_tokens":247,"peak_rss_mb":8388.87890625,"verdict":{"state":"Pass"},"prompt_ids_sha256":"6c1223de3236d024d3bf9a1a600bf90f1709debccf3ee61c2bd92011cb7c1968"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"9ec99261cd227d8f5e528bab7391b88aaafa4483233ccb0731a46b4f64bee8dc","stratum":"16k","engine":"apr","engine_version":"apr 0.70.0 (33b5e5b24 srv-tim-001)","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":34241.143807,"prompt_ms":15545.422813,"prompt_tps":873.1830689512427,"decode_tps":10.031054680327172,"input_tokens":13574,"output_tokens":187,"peak_rss_mb":8388.87890625,"verdict":{"state":"Pass"},"prompt_ids_sha256":"404ea4c6921bceb338fc1491d0dbe4aa8a2711122ca4646724bc17b6bcaaa069"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"9444f47e49f1779a103b611560b50ba89a8f20df19b976c9d4e06c2b01e103a5","stratum":"16k","engine":"apr","engine_version":"apr 0.70.0 (33b5e5b24 srv-tim-001)","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":44907.166333,"prompt_ms":9351.069223,"prompt_tps":949.8379049695972,"decode_tps":13.176071374762422,"input_tokens":8882,"output_tokens":468,"peak_rss_mb":8388.87890625,"verdict":{"state":"Fail"},"prompt_ids_sha256":"8f31bc51da4630c662e6ec7e0c67c440f57a03802d09a5f5f286399b72e3c666"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"5e5b0935bd95c58dc95d3caebb22a4f41d0573018dff4ce1633bb7668afdd1a2","stratum":"16k","engine":"apr","engine_version":"apr 0.70.0 (33b5e5b24 srv-tim-001)","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":30749.393268,"prompt_ms":16891.545245,"prompt_tps":860.1936524605982,"decode_tps":9.56094784404278,"input_tokens":14530,"output_tokens":132,"peak_rss_mb":8388.87890625,"verdict":{"state":"Pass"},"prompt_ids_sha256":"b191956bebc847f0567b83979f08d07982cecd6a68b6d78eba5b6a707c96236e"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"df963cc72999db8bd18665c41caca7b3cd058ec66923e7f5f3c161415d0cd4a2","stratum":"16k","engine":"apr","engine_version":"apr 0.70.0 (33b5e5b24 srv-tim-001)","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":35261.722639,"prompt_ms":12952.604207,"prompt_tps":897.5029124812969,"decode_tps":11.13602691660759,"input_tokens":11625,"output_tokens":248,"peak_rss_mb":8388.87890625,"verdict":{"state":"Pass"},"prompt_ids_sha256":"9b005fd461f84a7451b7d950c5dac4626187c3e983e8ef186cb0ac942803b1d9"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"675cb632f8d8a2bb1d48f14a4d927d717d45d10dd3c0cacbe19f6e2cf94455a7","stratum":"16k","engine":"apr","engine_version":"apr 0.70.0 (33b5e5b24 srv-tim-001)","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":28779.116291000002,"prompt_ms":9508.989692,"prompt_tps":938.5855163465668,"decode_tps":13.214707699256811,"input_tokens":8925,"output_tokens":254,"peak_rss_mb":8388.87890625,"verdict":{"state":"Pass"},"prompt_ids_sha256":"d017282343b77dda0b7ab50f3cb163775efea0802dee5ceeed7c6495c76cd8bf"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"c4cf0db8bffc8083c5691e577702ca73a932cb4cde56dd1c9dac360353ee38b1","stratum":"16k","engine":"apr","engine_version":"apr 0.70.0 (33b5e5b24 srv-tim-001)","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":46558.215613,"prompt_ms":12211.752607,"prompt_tps":916.0847226455774,"decode_tps":11.39685112124366,"input_tokens":11187,"output_tokens":391,"peak_rss_mb":8388.87890625,"verdict":{"state":"Fail"},"prompt_ids_sha256":"dc38252b026854e36f129b97bd87960e52b6fcb44f4e545c4860459f962308d2"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"aa3e63ab8090ebf9be4f8d697c9f96d692a2763af91a2ad7bf96d1a26c384025","stratum":"16k","engine":"apr","engine_version":"apr 0.70.0 (33b5e5b24 srv-tim-001)","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":40228.423579999995,"prompt_ms":10596.959391,"prompt_tps":923.4724451535836,"decode_tps":12.401064650877046,"input_tokens":9786,"output_tokens":367,"peak_rss_mb":8388.87890625,"verdict":{"state":"Pass"},"prompt_ids_sha256":"a2e6135129fe41dfbb6cefed737ccaecbb1acd94d176936ba19a8d4d00969e76"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"ea0520ed2859350cd1372043b3a06ff52b3e2f75fb0195b3d0521d1f23b3e7d6","stratum":"16k","engine":"apr","engine_version":"apr 0.70.0 (33b5e5b24 srv-tim-001)","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":35114.299179,"prompt_ms":9868.475296,"prompt_tps":886.4591274343906,"decode_tps":13.32894828547994,"input_tokens":8748,"output_tokens":336,"peak_rss_mb":8388.87890625,"verdict":{"state":"Pass"},"prompt_ids_sha256":"c132e4a7a5ce9599d799aadfe7b680427fb4469c2031225692c05d86ceefb87c"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"54246f96c1844cc9afe81e68e68bd564d60308d58851d0af96c22aeb7059b962","stratum":"16k","engine":"apr","engine_version":"apr 0.70.0 (33b5e5b24 srv-tim-001)","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":48163.307894,"prompt_ms":12016.087912,"prompt_tps":915.4393743253764,"decode_tps":11.54788625540133,"input_tokens":11000,"output_tokens":417,"peak_rss_mb":8388.87890625,"verdict":{"state":"Pass"},"prompt_ids_sha256":"51ccafe7aeaa0580788c3daf75e8b77c84449cb6f48369075c3a4e317d97ec7a"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"014e7869f90798986ecf651ca3a959bdf86f6369b6ce3d5ff71ecf0fd7cf7417","stratum":"16k","engine":"apr","engine_version":"apr 0.70.0 (33b5e5b24 srv-tim-001)","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":34686.612187,"prompt_ms":13321.051938,"prompt_tps":901.5804499447931,"decode_tps":10.931190323494292,"input_tokens":12010,"output_tokens":233,"peak_rss_mb":8388.87890625,"verdict":{"state":"Pass"},"prompt_ids_sha256":"9bf4cad8197210b509c2cc01811950c93a723c31ef9acd1dd2d4eb9ced1a48e1"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"2cdf5aadfe826822cbeef1cfad68e8d2d21d6bb99bc84ba02b50ad8224102206","stratum":"16k","engine":"apr","engine_version":"apr 0.70.0 (33b5e5b24 srv-tim-001)","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":41674.078233,"prompt_ms":10339.746109,"prompt_tps":927.1987821495164,"decode_tps":12.59412286567807,"input_tokens":9587,"output_tokens":394,"peak_rss_mb":8388.87890625,"verdict":{"state":"Pass"},"prompt_ids_sha256":"3163837c3c6ce35fc9144f2561444c6a631999d70734909762d7308610035870"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"6aa76be5cbd8ad7332c2acf93046bc20e866b9847dd276b88d4b66abe11949c8","stratum":"16k","engine":"apr","engine_version":"apr 0.70.0 (33b5e5b24 srv-tim-001)","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":26047.232455999998,"prompt_ms":9698.111705,"prompt_tps":945.0293292945732,"decode_tps":13.006280360389232,"input_tokens":9165,"output_tokens":212,"peak_rss_mb":8388.87890625,"verdict":{"state":"Pass"},"prompt_ids_sha256":"5ef3f3955e3349beecc7b3ead0a588b47ee06838c7a297707417f2fd7c975fc4"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"720dc2dbf89b8f4f37b09af101e95d2b2c11697ff7a5d59d50f131286980e9ab","stratum":"16k","engine":"apr","engine_version":"apr 0.70.0 (33b5e5b24 srv-tim-001)","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":23207.33306,"prompt_ms":14741.899206,"prompt_tps":879.5338930768701,"decode_tps":10.338638961747948,"input_tokens":12966,"output_tokens":87,"peak_rss_mb":8388.87890625,"verdict":{"state":"Pass"},"prompt_ids_sha256":"34405e806691b42d660adc2a2cfea20841a3de09c5c209d1aa96f85455b602c6"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"ebb26bc757178bfd19484adb04721a053df335e8059aed67cf8e11eb4d53cca1","stratum":"16k","engine":"apr","engine_version":"apr 0.70.0 (33b5e5b24 srv-tim-001)","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":32672.907633000003,"prompt_ms":15591.878782,"prompt_tps":877.7646485938413,"decode_tps":9.975596726481198,"input_tokens":13686,"output_tokens":170,"peak_rss_mb":8388.87890625,"verdict":{"state":"Fail"},"prompt_ids_sha256":"2a4712db6c149f45728ae2b1ed0ed2762692034e6140466b328757593cef7e3d"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"1ee0546fe5e55ded3962edb4c5d8eb0632d49a0d3fcbcb6de9ec2d9936a567f2","stratum":"16k","engine":"apr","engine_version":"apr 0.70.0 (33b5e5b24 srv-tim-001)","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":29897.229697,"prompt_ms":9725.534781,"prompt_tps":941.1307661848564,"decode_tps":13.012902165546842,"input_tokens":9153,"output_tokens":262,"peak_rss_mb":8388.87890625,"verdict":{"state":"Pass"},"prompt_ids_sha256":"4125241c2b414624ad34f56859fa91c135d8d1ffcc58e335cf7051aadd0439c5"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"1c2d4d2a2013dc86953f03eddb0ce4b67ac53e05931b7b18ff6d527d2c880640","stratum":"16k","engine":"apr","engine_version":"apr 0.70.0 (33b5e5b24 srv-tim-001)","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":41767.692926,"prompt_ms":12180.046079,"prompt_tps":916.5810972791148,"decode_tps":11.438406475612252,"input_tokens":11164,"output_tokens":338,"peak_rss_mb":8388.87890625,"verdict":{"state":"Pass"},"prompt_ids_sha256":"e4d7241584af87981056a070fd2ec4e3eee56ef2016c62d5f7f71ab382b92dc9"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"f51557a52249a78f34a126d0646c5108cec49aa8bf9829fbfa487f75f82c7984","stratum":"16k","engine":"apr","engine_version":"apr 0.70.0 (33b5e5b24 srv-tim-001)","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":28436.229803,"prompt_ms":9037.486812,"prompt_tps":933.887946458007,"decode_tps":13.69316477765186,"input_tokens":8440,"output_tokens":265,"peak_rss_mb":8388.87890625,"verdict":{"state":"Pass"},"prompt_ids_sha256":"44aebbb6cf4173813f2621076bab1a2a912ba2f0a714ee96b98f86cfe61ab844"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"69a9d1096c603ec6d3706769831302fbf6a198a02c189f771a730704506c2941","stratum":"16k","engine":"apr","engine_version":"apr 0.70.0 (33b5e5b24 srv-tim-001)","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":45567.496341,"prompt_ms":16718.998351,"prompt_tps":860.3386218492965,"decode_tps":9.614844001689775,"input_tokens":14384,"output_tokens":277,"peak_rss_mb":8388.87890625,"verdict":{"state":"Pass"},"prompt_ids_sha256":"dbd2749149ab437d0166e1de2f12680147b380a10ddb9b038ba1e6759f59d1a0"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"36528c9baa7523120f3683b655e9bd996e7ec070c288fa94b92aa17f033e18c4","stratum":"16k","engine":"apr","engine_version":"apr 0.70.0 (33b5e5b24 srv-tim-001)","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":24743.902844,"prompt_ms":9093.550949,"prompt_tps":942.0962227018804,"decode_tps":13.578256382233016,"input_tokens":8567,"output_tokens":212,"peak_rss_mb":8388.87890625,"verdict":{"state":"Pass"},"prompt_ids_sha256":"036ef65a29008fd891473edcdf5366c0e021f70e465cd8fd70aee4f55c392c39"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"7d6b1ea94587f39adc61be730808d638df69dc46fb23c0292d839974de14c4aa","stratum":"16k","engine":"apr","engine_version":"apr 0.70.0 (33b5e5b24 srv-tim-001)","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":30242.180668,"prompt_ms":9514.809916,"prompt_tps":940.8490636209574,"decode_tps":13.19447696031174,"input_tokens":8952,"output_tokens":273,"peak_rss_mb":8388.87890625,"verdict":{"state":"Pass"},"prompt_ids_sha256":"5640f5c62747622fad6026aa1b84af1b301adde703da627336351517c166ff00"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"25c07e0a68726a522357ec60ea143c91cea4227fe4139c3db6f0c52ac274bb8d","stratum":"16k","engine":"apr","engine_version":"apr 0.70.0 (33b5e5b24 srv-tim-001)","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":37123.013339000005,"prompt_ms":16931.707895,"prompt_tps":856.0270523140867,"decode_tps":9.577258752129763,"input_tokens":14494,"output_tokens":193,"peak_rss_mb":8388.87890625,"verdict":{"state":"Pass"},"prompt_ids_sha256":"3e4d4daca07a12047b37ab6fa0e74f224e28c54cd87dad01f94457face129e0c"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"cc6f12cb2a5dc9e2cd0cff0eda9338245b06a7b29be77caf7c7ad415dd948bb4","stratum":"16k","engine":"apr","engine_version":"apr 0.70.0 (33b5e5b24 srv-tim-001)","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":33556.881197,"prompt_ms":12410.530926,"prompt_tps":912.6120443624284,"decode_tps":11.369788246509756,"input_tokens":11326,"output_tokens":240,"peak_rss_mb":8388.87890625,"verdict":{"state":"Pass"},"prompt_ids_sha256":"7ef7e31a465dc80e0aa2ce7fa43085eba91d4ec00561981cadf511afc94587fc"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"88ef36ab5267cf425c39c87c20a1e697a4b612b9d36b17c39ca3b51cd24f9a9e","stratum":"16k","engine":"apr","engine_version":"apr 0.70.0 (33b5e5b24 srv-tim-001)","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":34052.515761,"prompt_ms":13145.951937,"prompt_tps":900.2771390552361,"decode_tps":11.027706355477749,"input_tokens":11835,"output_tokens":230,"peak_rss_mb":8388.87890625,"verdict":{"state":"Pass"},"prompt_ids_sha256":"537725b34a695cc189b0d1338ef38a956b5d0250fa0c9c4bd1b5740c766399ce"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"43d42945260ed2fee02c5b9fda7974cf8f66fd65dff63de4e17ea8faf547e878","stratum":"16k","engine":"apr","engine_version":"apr 0.70.0 (33b5e5b24 srv-tim-001)","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":31263.515961,"prompt_ms":10529.318244,"prompt_tps":923.89647407071,"decode_tps":12.52060711554712,"input_tokens":9728,"output_tokens":259,"peak_rss_mb":8388.87890625,"verdict":{"state":"Pass"},"prompt_ids_sha256":"d9b5939d45f4b6cadbb25f9ce529b2543a1f3da2d3ab3decf3c7abe8c21fed68"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"059c0fd951a748e2ae74924c143d7c8e8d3c0c277645ba032d7f3fe28458b002","stratum":"16k","engine":"apr","engine_version":"apr 0.70.0 (33b5e5b24 srv-tim-001)","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":50651.697723,"prompt_ms":17534.985074,"prompt_tps":872.8835488257685,"decode_tps":9.194391971726011,"input_tokens":15306,"output_tokens":304,"peak_rss_mb":8388.87890625,"verdict":{"state":"Fail"},"prompt_ids_sha256":"698b52cf76abfa36ca28d8f350ef8107c942ea080b91406c1ac66d4b38c385b0"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"0882349afbe288f0c5a3e2112311c022c01875743aabcc1d12afe9ce23093012","stratum":"16k","engine":"apr","engine_version":"apr 0.70.0 (33b5e5b24 srv-tim-001)","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":36722.172381,"prompt_ms":15464.104293,"prompt_tps":890.255247840755,"decode_tps":9.950375447953098,"input_tokens":13767,"output_tokens":211,"peak_rss_mb":8388.87890625,"verdict":{"state":"Pass"},"prompt_ids_sha256":"ff993db7ae7f149d4e3a1d6d7840460f0cd8a26c0eea77c35f6196552a40d4e5"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"bcef4d3206e948203a70d22840d2a5bbbd067648e038d4939ac7913c2d0d977d","stratum":"16k","engine":"apr","engine_version":"apr 0.70.0 (33b5e5b24 srv-tim-001)","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":35514.838047,"prompt_ms":13428.777523,"prompt_tps":899.2627943472111,"decode_tps":10.885541665601464,"input_tokens":12076,"output_tokens":240,"peak_rss_mb":8388.87890625,"verdict":{"state":"Pass"},"prompt_ids_sha256":"39836f7486ad00c5fda9e448cf60e59f2860a0307cbf1871f34c46333a1c992c"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"f44f0554c5d06d6d9bd8a87e7c9adef7b5acc16f8dd84ee8990ad95cdb0d7a86","stratum":"16k","engine":"apr","engine_version":"apr 0.70.0 (33b5e5b24 srv-tim-001)","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":39535.842824,"prompt_ms":12124.925088,"prompt_tps":922.3149766976936,"decode_tps":11.434599340599922,"input_tokens":11183,"output_tokens":313,"peak_rss_mb":8388.87890625,"verdict":{"state":"Pass"},"prompt_ids_sha256":"1f020914b5d4e264ef245a02a23947d4b1fd4c90b001bd77a260391e94464e4f"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"d49cd0c3fad3e8f5c978cbcb184e89f2cd360cdf37c1a97d192e6ca1e88aa974","stratum":"16k","engine":"apr","engine_version":"apr 0.70.0 (33b5e5b24 srv-tim-001)","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":40888.801118,"prompt_ms":15594.337541,"prompt_tps":882.62806700267,"decode_tps":9.939051142362391,"input_tokens":13764,"output_tokens":251,"peak_rss_mb":8388.87890625,"verdict":{"state":"Pass"},"prompt_ids_sha256":"2ad00e24d8988d50c0203eb5a7be9e633ca84a757788f60888e7586e09d1d1c8"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"6472e6c3bf01349b337af3cf2ddd2fc277b3af86eeb21bf5ca250923f3760dbb","stratum":"16k","engine":"apr","engine_version":"apr 0.70.0 (33b5e5b24 srv-tim-001)","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":27421.73297,"prompt_ms":9400.17224,"prompt_tps":937.4296316085374,"decode_tps":13.345183733416356,"input_tokens":8812,"output_tokens":240,"peak_rss_mb":8388.87890625,"verdict":{"state":"Pass"},"prompt_ids_sha256":"27a05690c12bac021035e63d41f559f99635ca5e4f2488ee991d136a21e34989"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"5a466ceed2939ee8936626418aadce90a670d794ce8d1289af467fe01430718d","stratum":"16k","engine":"apr","engine_version":"apr 0.70.0 (33b5e5b24 srv-tim-001)","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":38270.551845,"prompt_ms":16884.915546,"prompt_tps":861.064419326886,"decode_tps":9.55679488836836,"input_tokens":14539,"output_tokens":204,"peak_rss_mb":8388.87890625,"verdict":{"state":"Pass"},"prompt_ids_sha256":"3f65be738f0bac2a6d1a61af3c99be128d0245394c8f562f5629e4378a22d17b"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"c8cd063621db62fd4dde1e4e2fe07d67562b8d073e89e6bd35c00b88b2c6d546","stratum":"16k","engine":"apr","engine_version":"apr 0.70.0 (33b5e5b24 srv-tim-001)","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":31360.497290000003,"prompt_ms":14751.930986,"prompt_tps":894.1880227421503,"decode_tps":10.25927054970496,"input_tokens":13191,"output_tokens":170,"peak_rss_mb":8388.87890625,"verdict":{"state":"Pass"},"prompt_ids_sha256":"7d874c65f58ba21b178b10b378237380b9c35c43a22721cead3d56854f952312"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"1c7345312237ba7892931e444d0b663807feae59083cbc849d13f6e4748d2bd2","stratum":"16k","engine":"apr","engine_version":"apr 0.70.0 (33b5e5b24 srv-tim-001)","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":31268.768051,"prompt_ms":16866.941440000002,"prompt_tps":866.9028734115293,"decode_tps":9.5391467509588,"input_tokens":14622,"output_tokens":137,"peak_rss_mb":8388.87890625,"verdict":{"state":"Pass"},"prompt_ids_sha256":"6dc5aeb4272351770a57d1673d2f6fceb604770bb1e63363de5dde21a07ecbae"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"0dc43a5ae0a19646ed73c5f0aa0495eda3a3b460efd42e35ab5a0d1fe2002192","stratum":"16k","engine":"apr","engine_version":"apr 0.70.0 (33b5e5b24 srv-tim-001)","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":31556.864131000002,"prompt_ms":9268.143207,"prompt_tps":943.2310015880402,"decode_tps":13.391964488736646,"input_tokens":8742,"output_tokens":298,"peak_rss_mb":8388.87890625,"verdict":{"state":"Pass"},"prompt_ids_sha256":"5a5c9a9d4cc57194ef9db7c284d1261f917f9c29a19a53489aa80dac05004407"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"5189d6142784fae84f7d1634633cf93f49b3d2528950f68906f1c5ca9e8dfc5a","stratum":"16k","engine":"apr","engine_version":"apr 0.70.0 (33b5e5b24 srv-tim-001)","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":51008.720091,"prompt_ms":18927.315703,"prompt_tps":855.8001701970131,"decode_tps":8.826808239185787,"input_tokens":16198,"output_tokens":280,"peak_rss_mb":8388.87890625,"verdict":{"state":"Pass"},"prompt_ids_sha256":"490004b669067449f5c7e62e6e309d0ee30ed55781d9c2b9d140e4ac5dbb38c8"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"28c93295ce9295f699d1ab17eecef2d2ef643f6ec8434694ee70fa20ad70c0b9","stratum":"16k","engine":"apr","engine_version":"apr 0.70.0 (33b5e5b24 srv-tim-001)","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":34208.734355,"prompt_ms":15364.121595,"prompt_tps":884.0075832529233,"decode_tps":10.050531124999573,"input_tokens":13582,"output_tokens":189,"peak_rss_mb":8388.87890625,"verdict":{"state":"Pass"},"prompt_ids_sha256":"8c0ae16658161f3d1698b96183289aed7fd75bcda9877263efda1062664d7497"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"0f1bc13a198826247cef2fe67851805b3a7046ac9d291b02b47581866c6d896c","stratum":"16k","engine":"apr","engine_version":"apr 0.70.0 (33b5e5b24 srv-tim-001)","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":26495.014259,"prompt_ms":10910.101975,"prompt_tps":916.7650332617536,"decode_tps":12.35094663311022,"input_tokens":10002,"output_tokens":192,"peak_rss_mb":8388.87890625,"verdict":{"state":"Pass"},"prompt_ids_sha256":"6a1e197e1c7efd9196871bec225a755aca2bd8ddfe47e0d46af2f3e0b14f1419"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"4e7c51716440d0055d37c6d118eca15cfb84381cb2984b437ddbf7920cee4651","stratum":"16k","engine":"apr","engine_version":"apr 0.70.0 (33b5e5b24 srv-tim-001)","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":20387.29621,"prompt_ms":11075.155249,"prompt_tps":924.1405442983872,"decode_tps":12.184277440673496,"input_tokens":10235,"output_tokens":113,"peak_rss_mb":8388.87890625,"verdict":{"state":"Pass"},"prompt_ids_sha256":"f45b53c01e8955c24c49d0f24260bd34e4258513c7701ab5e462911006f37e99"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"d2107848512e9694b1e838662ca844b67f962cdc8d3826265b75f6b002d2982d","stratum":"16k","engine":"apr","engine_version":"apr 0.70.0 (33b5e5b24 srv-tim-001)","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":34933.544076,"prompt_ms":9160.244833,"prompt_tps":939.0578698301916,"decode_tps":13.483442517160547,"input_tokens":8602,"output_tokens":347,"peak_rss_mb":8388.87890625,"verdict":{"state":"Pass"},"prompt_ids_sha256":"5b324a8a6e5df50a926268dd114f6374ee8d731ed147592868a109e4fa8cbc33"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"f0466d3c53c99fb39ebfe065673d3b9bdd70fdf30ee02898eb6e886201e8d4a3","stratum":"16k","engine":"apr","engine_version":"apr 0.70.0 (33b5e5b24 srv-tim-001)","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":40976.511631,"prompt_ms":16276.611836,"prompt_tps":870.5128648863848,"decode_tps":9.691268230115163,"input_tokens":14169,"output_tokens":239,"peak_rss_mb":8388.87890625,"verdict":{"state":"Pass"},"prompt_ids_sha256":"8600d8498d868371fc947edc7db322d590a85502079ad419abb0df4a4727df72"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"f570aee8b67fbb453024af27fe758441e49aa16bc6b09147772a0d43a92115ec","stratum":"16k","engine":"apr","engine_version":"apr 0.70.0 (33b5e5b24 srv-tim-001)","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":30651.661434,"prompt_ms":14244.470652,"prompt_tps":863.6333564471722,"decode_tps":10.752779757241662,"input_tokens":12302,"output_tokens":176,"peak_rss_mb":8388.87890625,"verdict":{"state":"Pass"},"prompt_ids_sha256":"8b2497a464c07d512e5a9659dcb4f07a47d97fc06c111ae926ed313355ce34d3"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"409826d3610e79d93d92ba45ad55d943307796f6b4218ad327e872390681ef26","stratum":"16k","engine":"apr","engine_version":"apr 0.70.0 (33b5e5b24 srv-tim-001)","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":51509.65479,"prompt_ms":19153.826514999997,"prompt_tps":849.2819952848989,"decode_tps":8.792376962116053,"input_tokens":16267,"output_tokens":284,"peak_rss_mb":8388.87890625,"verdict":{"state":"Pass"},"prompt_ids_sha256":"cc546fdfce8b6b26c4668049fff094491f2f8bee3cf437f37a5eb735bdb27740"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"b53f7c2d4a456e341e0f1e013eede2c53dbe5230bb2ef59c8437a9dbd9b1126f","stratum":"16k","engine":"apr","engine_version":"apr 0.70.0 (33b5e5b24 srv-tim-001)","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":37966.178728000006,"prompt_ms":15010.402365999998,"prompt_tps":889.7163230114575,"decode_tps":10.129674634691366,"input_tokens":13355,"output_tokens":232,"peak_rss_mb":8388.87890625,"verdict":{"state":"Pass"},"prompt_ids_sha256":"58c397b59aee26d82ba43edafd279603a863fc48170cd0fb176fdf91ffc3a98f"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"fd326fe7d4bacc97fcf64369c18bd23e1a718b755ec9d720b6010b7bea96f970","stratum":"16k","engine":"apr","engine_version":"apr 0.70.0 (33b5e5b24 srv-tim-001)","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":26477.42465,"prompt_ms":9261.508955,"prompt_tps":907.4115288160406,"decode_tps":13.630956964150151,"input_tokens":8404,"output_tokens":234,"peak_rss_mb":8388.87890625,"verdict":{"state":"Pass"},"prompt_ids_sha256":"450e403e1bc170e1470b9b1d2a97c64aa904b0167319be75d1fa671f5a99f4dd"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"fa7720559e61059c92e06f6156fc0da7ae0a6367cd814f36465e85f7d04800e7","stratum":"16k","engine":"apr","engine_version":"apr 0.70.0 (33b5e5b24 srv-tim-001)","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":50680.036106,"prompt_ms":19624.293213,"prompt_tps":806.8061268831593,"decode_tps":8.934857339460278,"input_tokens":15833,"output_tokens":277,"peak_rss_mb":8388.87890625,"verdict":{"state":"Pass"},"prompt_ids_sha256":"7d9efdebce284cc4c5ce1d3a24fd62794d1ac2228510aa0c228e9ee283ab2cd9"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"f8f4e9f89c28ef4f440267322b74bbf68eed96ea240d6a883bec0d50f2dbe576","stratum":"16k","engine":"apr","engine_version":"apr 0.70.0 (33b5e5b24 srv-tim-001)","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":31805.397976999997,"prompt_ms":10775.383497,"prompt_tps":886.1865568551281,"decode_tps":12.630910852008697,"input_tokens":9549,"output_tokens":265,"peak_rss_mb":8388.87890625,"verdict":{"state":"Pass"},"prompt_ids_sha256":"5d5cfe1a4a40e808aca7241c0e6a96d796456776e3fc689c00288b819bcc28cd"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"ca612af4633a006928c0400dcf7a90a769c47290edc85241d3ba156583b2f6b7","stratum":"16k","engine":"apr","engine_version":"apr 0.70.0 (33b5e5b24 srv-tim-001)","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":29428.911182,"prompt_ms":10185.579266,"prompt_tps":908.7357486772516,"decode_tps":12.921522051062617,"input_tokens":9256,"output_tokens":248,"peak_rss_mb":8388.87890625,"verdict":{"state":"Pass"},"prompt_ids_sha256":"3a050649a54bd19a4c5b2fc11ca233aacbd3f9ea410506f99d0d65a893a023cd"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"f80dddb74b021666f2f30078ee85f6a231fb529e1ed67f051a9b662ffb17b4e6","stratum":"16k","engine":"apr","engine_version":"apr 0.70.0 (33b5e5b24 srv-tim-001)","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":25976.29334,"prompt_ms":10162.108384,"prompt_tps":910.6377978186304,"decode_tps":12.875518647035811,"input_tokens":9254,"output_tokens":203,"peak_rss_mb":8388.87890625,"verdict":{"state":"Pass"},"prompt_ids_sha256":"29ad40656fdbdd17ccccd5a79690a1c5856bd2c5e066028c0776ebd8fcc59873"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"fa49d36e7eb9eed429d0d549464654396d8fe8fc5a0be7b4e667bddb37922c36","stratum":"32k","engine":"apr","engine_version":"apr 0.70.0 (33b5e5b24 srv-tim-001)","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":74470.42901899999,"prompt_ms":32893.093764,"prompt_tps":750.4006821947052,"decode_tps":6.285982019733049,"input_tokens":24683,"output_tokens":261,"peak_rss_mb":8388.87890625,"verdict":{"state":"Pass"},"prompt_ids_sha256":"10163b836e6b21a8d1943f429085e148470b19b5c857f7822b5661509f4d11d9"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"3b51ad3f76c868adaa04840a77eed635ade302cec8c85bf0d3795526ab237fa7","stratum":"32k","engine":"apr","engine_version":"apr 0.70.0 (33b5e5b24 srv-tim-001)","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":49217.077835,"prompt_ms":26298.244564,"prompt_tps":768.4543335513868,"decode_tps":7.4750146167659794,"input_tokens":20209,"output_tokens":171,"peak_rss_mb":8388.87890625,"verdict":{"state":"Pass"},"prompt_ids_sha256":"2d0742ae98d285c41c4c84ef725a4fdd40c2b06112ad0f5f6709cd4fd9da985b"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"f8815530d0c7dc219ea3341c95432e757c3ab7fb0308bd2146f2d90b840f726b","stratum":"32k","engine":"apr","engine_version":"apr 0.70.0 (33b5e5b24 srv-tim-001)","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":37245.72366,"prompt_ms":20470.964226,"prompt_tps":826.2434447771478,"decode_tps":8.545940855027707,"input_tokens":16914,"output_tokens":143,"peak_rss_mb":8388.87890625,"verdict":{"state":"Fail"},"prompt_ids_sha256":"dd1693371bba23714bd583154ef11f1d6a58342d9feb0fbc8b3f89281a3c430a"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"717524cb0e8f36fac7a56af480fe9ecfe30ed9e797cefb9908a1c663d247b0c7","stratum":"32k","engine":"apr","engine_version":"apr 0.70.0 (33b5e5b24 srv-tim-001)","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":58139.73880399999,"prompt_ms":24664.906609,"prompt_tps":774.9471872326278,"decode_tps":7.779414361568559,"input_tokens":19114,"output_tokens":260,"peak_rss_mb":8388.87890625,"verdict":{"state":"Pass"},"prompt_ids_sha256":"fe2fbf0e6c29ac51fe639ae75e221ab52400551682caf9dcc5f560088f5991f0"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"03c15ba04bbacf6b92b3ab9bd18cdca26071fbb4a09dc4de9bef2ad078ba580f","stratum":"32k","engine":"apr","engine_version":"apr 0.70.0 (33b5e5b24 srv-tim-001)","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":60908.583259,"prompt_ms":29778.376351,"prompt_tps":742.0485166666232,"decode_tps":6.950599818906801,"input_tokens":22097,"output_tokens":216,"peak_rss_mb":8388.87890625,"verdict":{"state":"Pass"},"prompt_ids_sha256":"1d1e1149ebcc7a12c4464cd97630fc968780b0fa6703886e8550b6cd7ab8f27c"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"351ae6cc9897a62bc51dc629ead6161e8242c49a7cfbeabdd8f05e9c0c3cdaea","stratum":"32k","engine":"apr","engine_version":"apr 0.70.0 (33b5e5b24 srv-tim-001)","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":53402.573034,"prompt_ms":25399.693559,"prompt_tps":770.0880309663896,"decode_tps":7.657699209315521,"input_tokens":19560,"output_tokens":214,"peak_rss_mb":8388.87890625,"verdict":{"state":"Fail"},"prompt_ids_sha256":"d4c0dfa77150a8a80c4404832981c8f2c1f4eca29ffbec592e3d21b0ba30e535"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"7bb9e2edd88d122d0553e7776b5ac605cc7da369d4410dba027a27df9290f060","stratum":"32k","engine":"apr","engine_version":"apr 0.70.0 (33b5e5b24 srv-tim-001)","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":60677.87208,"prompt_ms":28649.986563,"prompt_tps":742.8973815815748,"decode_tps":7.162022807929635,"input_tokens":21284,"output_tokens":229,"peak_rss_mb":8388.87890625,"verdict":{"state":"Pass"},"prompt_ids_sha256":"162822cb75db0d188fb71a133b6480a9768b7b1582914953aaf3b0494125830a"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"d667fffd336a3dd90c6245ba9527ce0fb0448c554d576a089e347230a6ecb2c0","stratum":"32k","engine":"apr","engine_version":"apr 0.70.0 (33b5e5b24 srv-tim-001)","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":45514.633539999995,"prompt_ms":24866.474899,"prompt_tps":771.5609099370799,"decode_tps":7.769320540931917,"input_tokens":19186,"output_tokens":160,"peak_rss_mb":8388.87890625,"verdict":{"state":"Pass"},"prompt_ids_sha256":"544ecff0b9504efcc6965bfe96d614b95a0c9a33f99d1f021276789c0ac5f294"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"38a1dbfd07591ad9a6e4b27e7916e661614e6cd7231082e0e4e2601297e5fe7f","stratum":"32k","engine":"apr","engine_version":"apr 0.70.0 (33b5e5b24 srv-tim-001)","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":51348.320473,"prompt_ms":25759.965347,"prompt_tps":758.1143738717933,"decode_tps":7.597396288031484,"input_tokens":19529,"output_tokens":194,"peak_rss_mb":8388.87890625,"verdict":{"state":"Pass"},"prompt_ids_sha256":"3fa656a12db52afb3add603a41a8269708e92167a90663f8d3cc7548b477f08c"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"3d8a6474dcfe17a53ec35178358b968c69405032d23e581b7a700fe39982960a","stratum":"32k","engine":"apr","engine_version":"apr 0.70.0 (33b5e5b24 srv-tim-001)","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":47665.748745000004,"prompt_ms":23138.892468,"prompt_tps":822.7705810175944,"decode_tps":7.804272138286482,"input_tokens":19038,"output_tokens":191,"peak_rss_mb":8388.87890625,"verdict":{"state":"Pass"},"prompt_ids_sha256":"64b5124f7d02001c4cf5434c7f64ad4222b626650acc205dded43af97165d1eb"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"bf2811a3edb86d54d69166c93ec73ebcf6ff64b5556d9d7d9f090726e8fee9a2","stratum":"32k","engine":"apr","engine_version":"apr 0.70.0 (33b5e5b24 srv-tim-001)","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":60517.061508,"prompt_ms":29429.890591,"prompt_tps":738.8746462635465,"decode_tps":7.024987841826957,"input_tokens":21745,"output_tokens":218,"peak_rss_mb":8388.87890625,"verdict":{"state":"Pass"},"prompt_ids_sha256":"ddea598580c58fd52253867387479e86617b6fc8bef87e6b9a6bf70c78966051"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"b515045eff1517e48974319c1c62e87bc8ef658b739d6e365b934ee04f6667f0","stratum":"32k","engine":"apr","engine_version":"apr 0.70.0 (33b5e5b24 srv-tim-001)","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":85856.008109,"prompt_ms":44656.1028,"prompt_tps":669.1806522802971,"decode_tps":5.420792150019019,"input_tokens":29883,"output_tokens":223,"peak_rss_mb":8388.87890625,"verdict":{"state":"Pass"},"prompt_ids_sha256":"cce8cd881a53898e5c7a62a4f4ca518a141c8988a66f1f84e53c4982ca7f1560"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"8f82c371a972bb3e1d51e43968dccb02cfdaa9e94eaabb04b2b6202addef14fa","stratum":"32k","engine":"apr","engine_version":"apr 0.70.0 (33b5e5b24 srv-tim-001)","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":58871.011276,"prompt_ms":33771.399462,"prompt_tps":713.3550988050553,"decode_tps":6.473755400780245,"input_tokens":24091,"output_tokens":162,"peak_rss_mb":8388.87890625,"verdict":{"state":"Pass"},"prompt_ids_sha256":"4db639b147d20927495dfc8f015d002c34a1a8dbb361f9a395d0f540f44c1f25"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"304c3c29ecc274958e1042ea4f3e6fb4108bdc048767ad229c669ae8025e23f7","stratum":"32k","engine":"apr","engine_version":"apr 0.70.0 (33b5e5b24 srv-tim-001)","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":52219.545570999995,"prompt_ms":25110.417631,"prompt_tps":764.941478961577,"decode_tps":7.764580232470494,"input_tokens":19208,"output_tokens":210,"peak_rss_mb":8388.87890625,"verdict":{"state":"Pass"},"prompt_ids_sha256":"7d297d57bb08bd16d5a0bfd093357da4f17a4f2954bc5aebe4e292401877ef3a"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"c9d4c24c8671432ccf62fc9a693dd62fc3430da536f4b20f53abb64ba65e010b","stratum":"32k","engine":"apr","engine_version":"apr 0.70.0 (33b5e5b24 srv-tim-001)","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":47833.085172,"prompt_ms":24132.49135,"prompt_tps":766.6427693550173,"decode_tps":7.992951205625143,"input_tokens":18501,"output_tokens":189,"peak_rss_mb":8388.87890625,"verdict":{"state":"Pass"},"prompt_ids_sha256":"2b348c36ba02c98acdd3b0264d0bf7c6abc97c4aa1e0b5d5a57e9a76895b8ff7"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"7173ebdf8174e7e3ac987fc47c136b866b83cdf52d03a1c1fce2fb4ed8192ded","stratum":"32k","engine":"apr","engine_version":"apr 0.70.0 (33b5e5b24 srv-tim-001)","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":97284.089121,"prompt_ms":47187.614328,"prompt_tps":666.3612994171202,"decode_tps":5.196791738066429,"input_tokens":31444,"output_tokens":260,"peak_rss_mb":8388.87890625,"verdict":{"state":"Pass"},"prompt_ids_sha256":"0f80c00a890f530a76c0f115c409d1e0b9ef5735899f43cac2e7a695fba889c4"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"0f963ad6b87e0d3342e5334a4115c16eaf148f32c1e0adc968706c77b5191039","stratum":"32k","engine":"apr","engine_version":"apr 0.70.0 (33b5e5b24 srv-tim-001)","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":85635.564659,"prompt_ms":40951.446084,"prompt_tps":689.1087543554596,"decode_tps":5.69200041602607,"input_tokens":28220,"output_tokens":254,"peak_rss_mb":8388.87890625,"verdict":{"state":"Pass"},"prompt_ids_sha256":"4c4b6ae7e6b58fe247abcafa33c758ba870176b098d102db06af0ac0686fd4b0"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"74f801a07d6f9f7366f719b4b2302c99b8c74f8eb9f099cb0088470f7ed180e1","stratum":"32k","engine":"apr","engine_version":"apr 0.70.0 (33b5e5b24 srv-tim-001)","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":48396.977448,"prompt_ms":30984.672467,"prompt_tps":734.6858361741481,"decode_tps":6.7997639676765065,"input_tokens":22764,"output_tokens":118,"peak_rss_mb":8388.87890625,"verdict":{"state":"Pass"},"prompt_ids_sha256":"70c486262966bf7809098d46926cd98aaf3f6670ceb98cb3f34cb817ffbf760e"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"430c7c0f69397873ade236dd3f5db7663f2f1fb4f40eb879109629b59e0b3ce3","stratum":"32k","engine":"apr","engine_version":"apr 0.70.0 (33b5e5b24 srv-tim-001)","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":68136.22171900001,"prompt_ms":28209.261338,"prompt_tps":774.6746622734981,"decode_tps":7.022396351544032,"input_tokens":21853,"output_tokens":280,"peak_rss_mb":8388.87890625,"verdict":{"state":"Pass"},"prompt_ids_sha256":"d344c43778457ceeaf9fcafd3565a371a4836ae8c8fd3ac08de51d30e2f26f57"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"8003dd98e1763cde69c838d89b0bab41aaa2622c580b9e3de39b8b3c1e182f85","stratum":"32k","engine":"apr","engine_version":"apr 0.70.0 (33b5e5b24 srv-tim-001)","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":54534.067579,"prompt_ms":32159.803281999997,"prompt_tps":776.0930557050291,"decode_tps":6.3180298232873415,"input_tokens":24959,"output_tokens":141,"peak_rss_mb":8388.87890625,"verdict":{"state":"Pass"},"prompt_ids_sha256":"83323d291c7ce06a7d2db0bdee88fcedd25ebe222a3c6653fc9c2257e2379150"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"b931d437202ef5adb659042b0e7255fd1754602c9d52039ef36d831ad232674b","stratum":"32k","engine":"apr","engine_version":"apr 0.70.0 (33b5e5b24 srv-tim-001)","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":50999.203403,"prompt_ms":23902.287528999997,"prompt_tps":824.5235932372087,"decode_tps":7.617559019119001,"input_tokens":19708,"output_tokens":206,"peak_rss_mb":8388.87890625,"verdict":{"state":"Pass"},"prompt_ids_sha256":"0adbf65326e98c30c97a86ac1cd456158dddac02a101d1cc351a815782a5cdba"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"084e31008f5a675e5153246d96a5142e0adc8215e66f598266bbb76a10be6db2","stratum":"32k","engine":"apr","engine_version":"apr 0.70.0 (33b5e5b24 srv-tim-001)","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":92749.529857,"prompt_ms":41447.325003000005,"prompt_tps":728.2255247549829,"decode_tps":5.365215275177939,"input_tokens":30183,"output_tokens":275,"peak_rss_mb":8388.87890625,"verdict":{"state":"Pass"},"prompt_ids_sha256":"008c43328b72985f0930804bf4dba749371420359c750afa424dd5e6d3047e5a"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"14b3251ddf800d7bdfdb7a775c2281eba042c909e8a46700689825014ded27de","stratum":"32k","engine":"apr","engine_version":"apr 0.70.0 (33b5e5b24 srv-tim-001)","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":68833.68444699999,"prompt_ms":22007.879698,"prompt_tps":816.3894135441307,"decode_tps":8.122079627974516,"input_tokens":17967,"output_tokens":380,"peak_rss_mb":8388.87890625,"verdict":{"state":"Pass"},"prompt_ids_sha256":"cc964f6596ba0bf32e87225ab3a6e88f89eac8e4ba6bc237cab69ecee8f5ee60"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"ca55757f48d1d3f049d77fd86f25cc77c581af66809d307a89d2a5b7bb14d278","stratum":"32k","engine":"apr","engine_version":"apr 0.70.0 (33b5e5b24 srv-tim-001)","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":86033.36284700001,"prompt_ms":26862.292726,"prompt_tps":778.1167532199872,"decode_tps":7.189083530765419,"input_tokens":20902,"output_tokens":425,"peak_rss_mb":8388.87890625,"verdict":{"state":"Fail"},"prompt_ids_sha256":"36511a899244f7383295611d61080169d7b333ce858c223384ee57588db0cc22"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"5bc59b96abf0b69ee35dd697a3d38ea685e54eb0cd1544a880e08b0f64c3905f","stratum":"32k","engine":"apr","engine_version":"apr 0.70.0 (33b5e5b24 srv-tim-001)","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":53719.943499,"prompt_ms":24505.006704,"prompt_tps":762.0075450520881,"decode_tps":7.784680693633494,"input_tokens":18673,"output_tokens":227,"peak_rss_mb":8388.87890625,"verdict":{"state":"Pass"},"prompt_ids_sha256":"01878ef6e12e6090d8bf4d912ac05afd77064731e6d2c22d029d8e95916d2ef8"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"4608b8ddaa9de754634659da66dee0e3d59c7097dfbebe7ef8ff664f7a8e2a3c","stratum":"32k","engine":"apr","engine_version":"apr 0.70.0 (33b5e5b24 srv-tim-001)","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":75819.470417,"prompt_ms":27288.328696,"prompt_tps":737.7146553849177,"decode_tps":7.243278292420036,"input_tokens":20131,"output_tokens":351,"peak_rss_mb":8388.87890625,"verdict":{"state":"Pass"},"prompt_ids_sha256":"ea75c65f1593e8994329bf6520bd50cfd827175845144b2decd4a39d6a816003"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"dc10ee3c8ec1879df8935451288b060dcba94ff3db9fb2d0e714e938b65c15ca","stratum":"32k","engine":"apr","engine_version":"apr 0.70.0 (33b5e5b24 srv-tim-001)","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":63513.925825,"prompt_ms":38120.1242,"prompt_tps":691.4458059399502,"decode_tps":5.961428510771628,"input_tokens":26358,"output_tokens":151,"peak_rss_mb":8388.87890625,"verdict":{"state":"Pass"},"prompt_ids_sha256":"60fa312565da30886df98a128049dec4b80d7fbab7adb34688ce8d3b13131f87"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"841dd09f216b308d2acb091c53e79c3c22c6dc9c70d99ddd5aaa007e65fb8f0f","stratum":"32k","engine":"apr","engine_version":"apr 0.70.0 (33b5e5b24 srv-tim-001)","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":42919.447862,"prompt_ms":23019.445875,"prompt_tps":795.8054290044895,"decode_tps":8.061622173674264,"input_tokens":18319,"output_tokens":160,"peak_rss_mb":8388.87890625,"verdict":{"state":"Pass"},"prompt_ids_sha256":"693837d52c6a4861282f0d455277e48fac6d7e207bdaa4e7a172c2acf8726055"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"24bc258f12dfbb5600afabdff2efa2edeaf9efb4c7e13dae0bb627e6c0add649","stratum":"32k","engine":"apr","engine_version":"apr 0.70.0 (33b5e5b24 srv-tim-001)","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":68259.759293,"prompt_ms":28649.20257,"prompt_tps":796.8459137464921,"decode_tps":6.77546351651715,"input_tokens":22829,"output_tokens":268,"peak_rss_mb":8388.87890625,"verdict":{"state":"Pass"},"prompt_ids_sha256":"0662c740da8b7967cc8500d02856b1bb61b045eb9df620e43bddc27d9f806dfc"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"78a121fb8ef6ffe62c572f12c3113443d88b19491cd0db0981294e8ed29d38bb","stratum":"32k","engine":"apr","engine_version":"apr 0.70.0 (33b5e5b24 srv-tim-001)","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":45479.559186000006,"prompt_ms":22542.700639,"prompt_tps":826.8751955900026,"decode_tps":7.953324662661029,"input_tokens":18640,"output_tokens":182,"peak_rss_mb":8388.87890625,"verdict":{"state":"Pass"},"prompt_ids_sha256":"ba43eb4396f1eb5beb8d13c853dad39cd3e1cf8d77a0732070175d726fd6b01c"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"c0b75e2f0a6bcd1a3341d0cbb4e96aa16449934b09e961f790c377d972340d08","stratum":"32k","engine":"apr","engine_version":"apr 0.70.0 (33b5e5b24 srv-tim-001)","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":54883.01437,"prompt_ms":26556.61794,"prompt_tps":803.8297665851045,"decode_tps":7.141790367720514,"input_tokens":21347,"output_tokens":202,"peak_rss_mb":8388.87890625,"verdict":{"state":"Pass"},"prompt_ids_sha256":"957b6f511e3f28dc0e7f71f4f420374158278789df3325c0cb389818c1e26247"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"21a87ab49fd3587febc4c89bb82872bb6967bf0a9492abc590112d5f097f1bd5","stratum":"32k","engine":"apr","engine_version":"apr 0.70.0 (33b5e5b24 srv-tim-001)","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":57641.814448,"prompt_ms":30396.642397,"prompt_tps":772.4537366112963,"decode_tps":6.580061973070466,"input_tokens":23480,"output_tokens":179,"peak_rss_mb":8388.87890625,"verdict":{"state":"Pass"},"prompt_ids_sha256":"7b8e789020dabaf591806d9d9bf0c6232547b067051bbd4cb26f58de384c2ce8"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"0534621eb0e2215a3c13b22ebf197a2a472f4306bf24f0e380154a527dbc1463","stratum":"32k","engine":"apr","engine_version":"apr 0.70.0 (33b5e5b24 srv-tim-001)","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":66722.265646,"prompt_ms":28955.298575,"prompt_tps":789.5618807308396,"decode_tps":6.761969552455363,"input_tokens":22862,"output_tokens":255,"peak_rss_mb":8388.87890625,"verdict":{"state":"Pass"},"prompt_ids_sha256":"e49800ebb35f74199fe8a55797fc6e015fb00fd0fff65e78dd6f91889e9f5ff3"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"add564d1303cf5966124647b302705301dcd4528a1caad315f4d86aa3485e61a","stratum":"32k","engine":"apr","engine_version":"apr 0.70.0 (33b5e5b24 srv-tim-001)","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":59853.609220000006,"prompt_ms":29806.649592,"prompt_tps":786.2675047614255,"decode_tps":6.632140478536505,"input_tokens":23436,"output_tokens":199,"peak_rss_mb":8388.87890625,"verdict":{"state":"Pass"},"prompt_ids_sha256":"d501682a3b448ff13e405df0b3c77e50f0da19852d59188afedbf391ff330a0c"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"756c7e34ba781b2a93fb5ba02710dfea9f422786e59c39eb6e2b38e11ad4bf4a","stratum":"32k","engine":"apr","engine_version":"apr 0.70.0 (33b5e5b24 srv-tim-001)","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":54230.911933999996,"prompt_ms":22697.028035,"prompt_tps":829.579977209558,"decode_tps":7.879099760416964,"input_tokens":18829,"output_tokens":248,"peak_rss_mb":8388.87890625,"verdict":{"state":"Pass"},"prompt_ids_sha256":"d6675b3bdd9d86978cd61242f476ccbc7623c2446ae06132244a6f42efbc7255"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"40d8b0a4203d03441d9dcc41d1b76704f65ec7ea5ebd165cac2457a566b70162","stratum":"32k","engine":"apr","engine_version":"apr 0.70.0 (33b5e5b24 srv-tim-001)","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":55558.053515,"prompt_ms":21147.773042,"prompt_tps":843.2566381615322,"decode_tps":8.208177038824408,"input_tokens":17833,"output_tokens":282,"peak_rss_mb":8388.87890625,"verdict":{"state":"Pass"},"prompt_ids_sha256":"0200f2f9a1370d6f38cae62051efcd98e40658a6fc1ec422e8b37f6701ce5412"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"3db9e737af60c9d2537ac8d8b93aeaffc7b6cc68294a49978e5c870ffb36eb4f","stratum":"32k","engine":"apr","engine_version":"apr 0.70.0 (33b5e5b24 srv-tim-001)","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":48950.960273,"prompt_ms":24041.668077,"prompt_tps":820.4505584564808,"decode_tps":7.604017591062681,"input_tokens":19725,"output_tokens":189,"peak_rss_mb":8388.87890625,"verdict":{"state":"Pass"},"prompt_ids_sha256":"fffd8b1255cac61466278bf8094821380a784a6aefa5c63410027ecce55429ed"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"c354c595f4a8ea1b09659b55a5ce312a495e4eb6400c77183cd34cca3cb8327b","stratum":"32k","engine":"apr","engine_version":"apr 0.70.0 (33b5e5b24 srv-tim-001)","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":50226.680118,"prompt_ms":23481.726403,"prompt_tps":797.0453142495036,"decode_tps":7.904969599183868,"input_tokens":18716,"output_tokens":211,"peak_rss_mb":8388.87890625,"verdict":{"state":"Pass"},"prompt_ids_sha256":"05dadf955da5b93f28985b714df19e2ef0206facca769a7ffa72479563a4975a"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"c42dfb333a42fbec1745318326627fa05d41440da37c5d2d57304daf1e4bb3b6","stratum":"32k","engine":"apr","engine_version":"apr 0.70.0 (33b5e5b24 srv-tim-001)","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":48296.049144000004,"prompt_ms":24658.310267,"prompt_tps":814.0460470587686,"decode_tps":7.505580251986077,"input_tokens":20073,"output_tokens":177,"peak_rss_mb":8388.87890625,"verdict":{"state":"Pass"},"prompt_ids_sha256":"80097ce1ced3c4a30308e27f670ccbf58abb86ff4f9385521228aac8db4ed868"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"8136fdc9902571f25cbf93bf14de71435aea774d50ab0fb50ba2196f422389af","stratum":"32k","engine":"apr","engine_version":"apr 0.70.0 (33b5e5b24 srv-tim-001)","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":59092.18808,"prompt_ms":25606.558969,"prompt_tps":805.8091688531866,"decode_tps":7.327958418464374,"input_tokens":20634,"output_tokens":245,"peak_rss_mb":8388.87890625,"verdict":{"state":"Pass"},"prompt_ids_sha256":"a3c875c6ba4ab5c35526ae9bc83a120f426b85c20fe3a644c37dd3cb99b85be3"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"a267f9989c8533bce8be2d300c85a8509c45a80b1fe16836591ae91341138b2a","stratum":"32k","engine":"apr","engine_version":"apr 0.70.0 (33b5e5b24 srv-tim-001)","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":87118.88124399999,"prompt_ms":40307.993749,"prompt_tps":726.1586915554724,"decode_tps":5.518665023647035,"input_tokens":29270,"output_tokens":258,"peak_rss_mb":8388.87890625,"verdict":{"state":"Pass"},"prompt_ids_sha256":"52b26517747f4815b31f168051fd64d5054550a410ec132c26ad9f221325ae4b"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"60d2f019ddec1435b5b1230d70553a3833405b30dae55c8dbc8f6bcc1f0f6487","stratum":"32k","engine":"apr","engine_version":"apr 0.70.0 (33b5e5b24 srv-tim-001)","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":50291.485515,"prompt_ms":23220.426473,"prompt_tps":820.6136964003039,"decode_tps":7.805492860105431,"input_tokens":19055,"output_tokens":211,"peak_rss_mb":8388.87890625,"verdict":{"state":"Pass"},"prompt_ids_sha256":"926956c06005c8e58a90a81567de45b1b9f4cf512f4714410a8c2bdff97eaf09"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"4be2763e70a83f3b538045b385bf920c45f0ab971c6641e4573f745b32673998","stratum":"32k","engine":"apr","engine_version":"apr 0.70.0 (33b5e5b24 srv-tim-001)","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":64664.90169399999,"prompt_ms":20333.969754,"prompt_tps":840.7605699638142,"decode_tps":8.446564313781408,"input_tokens":17096,"output_tokens":374,"peak_rss_mb":8388.87890625,"verdict":{"state":"Fail"},"prompt_ids_sha256":"2209238bca60b627f187342a3fa75e2cc850f42325091d4f4918c6c267a9a260"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"4a66cbc27f5a915eaa9fed54adca1621f3d27e77b95dee234328d0114d722209","stratum":"32k","engine":"apr","engine_version":"apr 0.70.0 (33b5e5b24 srv-tim-001)","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":83331.04063599999,"prompt_ms":36448.006159,"prompt_tps":744.4851683141968,"decode_tps":5.870888405548316,"input_tokens":27135,"output_tokens":275,"peak_rss_mb":8388.87890625,"verdict":{"state":"Pass"},"prompt_ids_sha256":"7d7c836939bcc09cc2d8280e28b6a392d04a2145d05417cf9d72dd860d4e04fe"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"f552e516d8a086ee988e3c5488a31467a2bb6b35b277aecfb98ad15c96d6d3da","stratum":"32k","engine":"apr","engine_version":"apr 0.70.0 (33b5e5b24 srv-tim-001)","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":49309.554906000005,"prompt_ms":23310.347044,"prompt_tps":819.5073185286207,"decode_tps":7.785385436525612,"input_tokens":19103,"output_tokens":202,"peak_rss_mb":8388.87890625,"verdict":{"state":"Pass"},"prompt_ids_sha256":"c2c806c56c4d144111f554ef8f5f933837eea605a07ee70ebb09c16490c92ca1"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"5c2a849b4a443f183000bbd6996f84b95d3df274e267f4a0b9be650a53080e75","stratum":"32k","engine":"apr","engine_version":"apr 0.70.0 (33b5e5b24 srv-tim-001)","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":78098.04002,"prompt_ms":30329.922127,"prompt_tps":781.142790304402,"decode_tps":6.56062604534166,"input_tokens":23692,"output_tokens":313,"peak_rss_mb":8388.87890625,"verdict":{"state":"Pass"},"prompt_ids_sha256":"000dbf6068631c787dc66205d5a9ad463a2c755eb612acc7ece318bff11f5bfc"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"a2b4707bfaf04f8599408ccfb45a44703a000f71a094a68ba1759061b2a43672","stratum":"32k","engine":"apr","engine_version":"apr 0.70.0 (33b5e5b24 srv-tim-001)","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":41744.872708999996,"prompt_ms":23818.70437,"prompt_tps":820.4476488911558,"decode_tps":7.659392814979827,"input_tokens":19542,"output_tokens":137,"peak_rss_mb":8388.87890625,"verdict":{"state":"Pass"},"prompt_ids_sha256":"fa651c50ada370ebabdf777b8b3eac38d443ada3b2e4762702cf325ab1421721"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"3ab4ac140f83056d2501ee334f17e3ee310e217094d568b4fec0c75a1a30dce9","stratum":"32k","engine":"apr","engine_version":"apr 0.70.0 (33b5e5b24 srv-tim-001)","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":68193.148408,"prompt_ms":28293.099911,"prompt_tps":794.3986367948893,"decode_tps":6.852093188286345,"input_tokens":22476,"output_tokens":273,"peak_rss_mb":8388.87890625,"verdict":{"state":"Pass"},"prompt_ids_sha256":"a4b4fb3bd5fe96a03ed1ee01a571c200b6a1ab2e777a7638743f0a24ae89a56b"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"bf6a52843bdf826313ab90adf25c9d7a253496a9a3c5a46c08ef62ef5249470d","stratum":"32k","engine":"apr","engine_version":"apr 0.70.0 (33b5e5b24 srv-tim-001)","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":57343.66835,"prompt_ms":26445.460882,"prompt_tps":803.6917978036244,"decode_tps":7.161691637966168,"input_tokens":21254,"output_tokens":221,"peak_rss_mb":8388.87890625,"verdict":{"state":"Pass"},"prompt_ids_sha256":"29750ae08524616eb9b008167fc5e1577e4b6f505976a9986f03dac0a1eb099b"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"4c7ad86de1bf01701febc9297c4271016e389826ac06ac0fdc766c598c4ba789","stratum":"32k","engine":"apr","engine_version":"apr 0.70.0 (33b5e5b24 srv-tim-001)","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":50283.915443,"prompt_ms":24034.195854,"prompt_tps":815.4214985619464,"decode_tps":7.63443234592059,"input_tokens":19598,"output_tokens":200,"peak_rss_mb":8388.87890625,"verdict":{"state":"Pass"},"prompt_ids_sha256":"36e696fa27b6f800c61cc29b6ce76dc253720bc7ae8a736483bb1a628c4bf41c"} diff --git a/evidence/prm-001/s1-v2/33b5e5b24/rows-llama_cpp.jsonl b/evidence/prm-001/s1-v2/33b5e5b24/rows-llama_cpp.jsonl new file mode 100644 index 0000000000..39aeff7a9c --- /dev/null +++ b/evidence/prm-001/s1-v2/33b5e5b24/rows-llama_cpp.jsonl @@ -0,0 +1,200 @@ +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"f1ca076d355345060d79de6ab291b725d78bfd3596f2cfc01444d40d11e2ea79","stratum":"2k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":3788.665915,"prompt_ms":280.942,"prompt_tps":3495.3833887421606,"decode_tps":68.51205797947094,"input_tokens":982,"output_tokens":241,"peak_rss_mb":1187.6328125,"verdict":{"state":"Fail"},"prompt_ids_sha256":"27e16faa8fbdff4c4d6e0ff52b176400b21425330e38ee63f75e165299d93d1b"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"7788e95d4d3ecd6288d7d51fa0ed8459158b9b7ff7b94bbbffd244bbb7e1dad1","stratum":"2k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":2187.835902,"prompt_ms":196.643,"prompt_tps":3513.982191077231,"decode_tps":68.41066872212004,"input_tokens":691,"output_tokens":132,"peak_rss_mb":1376.8046875,"verdict":{"state":"Fail"},"prompt_ids_sha256":"10e34de14fde098ef400a5795d6d775f2af564296e76433ce573ff85c38f81a0"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"366eed3dd5e0055ebfdd837af58bd800debdd62e4c0a7f7da2dffa84d06ada9f","stratum":"2k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":4879.728306,"prompt_ms":360.808,"prompt_tps":3874.63692601051,"decode_tps":67.73430330655916,"input_tokens":1398,"output_tokens":302,"peak_rss_mb":1553.625,"verdict":{"state":"Pass"},"prompt_ids_sha256":"513376357643c1fb6950a0630d0a17b9cd1678c7d4961730e6cdd82a5e6755c4"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"f9d7c31e37b0ed40fbfd87af8984c8860ec6c047dc68a8b55bcb88ee67ee6452","stratum":"2k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":2329.289924,"prompt_ms":204.534,"prompt_tps":3485.9729922653446,"decode_tps":68.15332438677943,"input_tokens":713,"output_tokens":140,"peak_rss_mb":1757.7265625,"verdict":{"state":"Pass"},"prompt_ids_sha256":"13634b8349a0b7acd08e20bc05d86f438f3a09e1be22555abc534924c2845212"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"f8879c54b81ffd2aaeaf6398391a818b5f56d1c37ffd4a71100bf19864c1b902","stratum":"2k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":1797.8179910000001,"prompt_ms":254.863,"prompt_tps":3727.4928098625537,"decode_tps":67.7881998369016,"input_tokens":950,"output_tokens":101,"peak_rss_mb":1935.3671875,"verdict":{"state":"Pass"},"prompt_ids_sha256":"6526b0ed8df35ece7bfb8f728415890c02540d9548fa313c47f20bb0514294f2"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"296c5544001c3a5e623d3bc13a3b59728bad8d086d6f5baaa8ec44409d3e5cea","stratum":"2k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":2114.363107,"prompt_ms":105.912,"prompt_tps":3040.259838356371,"decode_tps":68.64955486183374,"input_tokens":322,"output_tokens":134,"peak_rss_mb":2117.01171875,"verdict":{"state":"Fail"},"prompt_ids_sha256":"1f540bb4461f766244ab8dbc1d7a1c48162b6cb270d8ae0eb239e075d14fcb7f"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"275cbe4c206c2eb59aca3ca92f0bbd7f171bc949a205afe8e44882e4e56fdc5c","stratum":"2k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":4556.927252,"prompt_ms":475.384,"prompt_tps":4005.1831782306513,"decode_tps":67.63901676159253,"input_tokens":1904,"output_tokens":274,"peak_rss_mb":2234.234375,"verdict":{"state":"Pass"},"prompt_ids_sha256":"e482de36d4b6c8e06ae0882d309b33305eabf0479dedcbcc690ff717f0dc969c"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"2d012d3f8e232bfdbed32db1d29426b65b9c7725d2a001308ac4fb59f8cdafa9","stratum":"2k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":3045.7075950000003,"prompt_ms":458.586,"prompt_tps":3983.985555599168,"decode_tps":67.77352797890569,"input_tokens":1827,"output_tokens":171,"peak_rss_mb":2453.45703125,"verdict":{"state":"Pass"},"prompt_ids_sha256":"aee8112d87fb3291454282615ad62f23da8c7bfa3b2e162d7e909ec867ad11b3"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"e777a873cb9229ece5b23335a8dd1bb421f1b5e298f76024a7af842e2cfcadfb","stratum":"2k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":2272.77378,"prompt_ms":476.728,"prompt_tps":3962.4272121629106,"decode_tps":67.79205371935844,"input_tokens":1889,"output_tokens":117,"peak_rss_mb":2667.0546875,"verdict":{"state":"Pass"},"prompt_ids_sha256":"dd8abf75e8df1fb4ba891ef3ac0728486da2bed010880eb742d6d79e669fe432"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"a7a62e12ad053746008e769e95d9c06f39e8b741a52ad4f650d05d1ca720c754","stratum":"2k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":3427.228138,"prompt_ms":459.2,"prompt_tps":3976.480836236934,"decode_tps":67.58559976154413,"input_tokens":1826,"output_tokens":196,"peak_rss_mb":2880.90234375,"verdict":{"state":"Pass"},"prompt_ids_sha256":"7081fe352adf097f8a152f710ba69d4a79e4044130c1c3982d8f715f175e3c32"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"66ec2c8042874218dcf97952a6a557fe1223cb59db4a28bf7615cd2a6a4f269b","stratum":"2k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":2059.5322610000003,"prompt_ms":115.73,"prompt_tps":3136.6110775079924,"decode_tps":68.29814992680804,"input_tokens":363,"output_tokens":128,"peak_rss_mb":3092.62890625,"verdict":{"state":"Pass"},"prompt_ids_sha256":"7f8167b3b624a3f6f0dd6b3c48781ebe5c7aa79d9a20f2016a592cbdb64df348"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"5f449a13b53a4663c67c7b0cea0588720e0a2e63e79561ba6adcd58d9bd87d77","stratum":"2k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":5740.652001,"prompt_ms":441.492,"prompt_tps":3950.241453978781,"decode_tps":67.4255790171598,"input_tokens":1744,"output_tokens":355,"peak_rss_mb":3211.21484375,"verdict":{"state":"Fail"},"prompt_ids_sha256":"4be9a7ed9292e82aa712abd073d04ef245be3607a90208a80f7195e73fbdfd62"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"3b86db746737d9338f2ac5c3c0aa7184acd7f05314b46fb8ae1a8bbdd83ac4ad","stratum":"2k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":2122.635174,"prompt_ms":191.144,"prompt_tps":3437.199179676056,"decode_tps":68.01428515906706,"input_tokens":657,"output_tokens":127,"peak_rss_mb":3429.79296875,"verdict":{"state":"Pass"},"prompt_ids_sha256":"8278ed707fc9d1928b2ec8f690a552e0a16913c5847057eb444ca2509a21b839"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"a5f04841ad4bdf65c809d00511d1b4669ed6f52b6ec0be2eac5ee0417b29297f","stratum":"2k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":3187.444375,"prompt_ms":423.351,"prompt_tps":3850.233021771532,"decode_tps":67.16175891080695,"input_tokens":1630,"output_tokens":182,"peak_rss_mb":3605.3984375,"verdict":{"state":"Pass"},"prompt_ids_sha256":"f02afc68f2366821fc07f38c65009e232df057bd81d998875a7600a85ff58c1c"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"f1206fea7a5282d2607d846f5d4f0a9fe7b6982f8d0ab75a42632d7a8ca4e487","stratum":"2k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":3046.04458,"prompt_ms":269.061,"prompt_tps":3735.212461114766,"decode_tps":67.50053314294722,"input_tokens":1005,"output_tokens":183,"peak_rss_mb":3813.04296875,"verdict":{"state":"Pass"},"prompt_ids_sha256":"f76934c956f6886a06b4e394e263a24c39df6734ad36fed9be55913886901237"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"993b00de597f00fefd6e145197fc5659223b89690cc0a671c636d955b4c961c5","stratum":"2k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":5235.017234,"prompt_ms":419.933,"prompt_tps":3776.792964591971,"decode_tps":67.28646145772427,"input_tokens":1586,"output_tokens":320,"peak_rss_mb":4001.26171875,"verdict":{"state":"Pass"},"prompt_ids_sha256":"79f32211f992adecbdc1e0848ba2e99956661eb798fbac366156478f0670d2d2"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"47015b6bbe7f1a4b0b27cd8396c3430142487ab219124adf8c7f331e365c7ec3","stratum":"2k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":2937.683173,"prompt_ms":428.672,"prompt_tps":3872.42460435951,"decode_tps":67.20574192665256,"input_tokens":1660,"output_tokens":164,"peak_rss_mb":4211.94921875,"verdict":{"state":"Pass"},"prompt_ids_sha256":"c8ceab8a8499f6119d42fc491d582972ddcbb9d0eeb513d16f0a8bc73f03a953"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"9af203f89adb6dc0348f46161c0da6bdf7ebea3752a638220571ad3a65854634","stratum":"2k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":3506.837502,"prompt_ms":437.869,"prompt_tps":3907.561393932888,"decode_tps":67.24083425468888,"input_tokens":1711,"output_tokens":202,"peak_rss_mb":4420.046875,"verdict":{"state":"Pass"},"prompt_ids_sha256":"40d56ab8f83a8a2a5627f8c21de53e8cc3086149ca9a2d3f7bf948f43294f577"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"34d4c5a7a0e46bbd1b00935ef3000cc8ddba8dd64cbdffa9e01f1207863f945c","stratum":"2k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":5105.831991,"prompt_ms":373.429,"prompt_tps":3890.9672253627864,"decode_tps":67.29859980210342,"input_tokens":1453,"output_tokens":314,"peak_rss_mb":4630.90625,"verdict":{"state":"Fail"},"prompt_ids_sha256":"6ea38a67c9dcd29895861eb72204a5cb6730ab44d894a6ab601f1753ff7c9df4"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"167d286e0f8a0bae32d26b9d8abc056df02e030188cf395ff255acc3cb1d640d","stratum":"2k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":3532.9113039999997,"prompt_ms":254.457,"prompt_tps":3709.860605131712,"decode_tps":67.54472727409181,"input_tokens":944,"output_tokens":217,"peak_rss_mb":4836.91796875,"verdict":{"state":"Fail"},"prompt_ids_sha256":"2fcc6f03b34f1893afd08fe8ec1dcf30568ee3397ef74662c46ef4085107c1d6"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"fab75467c60ebd40cc94c13296129957a26a7eb9865a58175dd09db9610f1631","stratum":"2k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":3510.800329,"prompt_ms":300.423,"prompt_tps":3505.057868405548,"decode_tps":67.58503680037134,"input_tokens":1053,"output_tokens":213,"peak_rss_mb":5024.0546875,"verdict":{"state":"Pass"},"prompt_ids_sha256":"e857ce0443d4fb4bf8d286c247650ecf93b401aff929197f55c9aa7777881371"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"c274c3708f67fd97e467895e6c780665af6a13a28c8a727d25ea1863e3328dc8","stratum":"2k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":3459.4494369999998,"prompt_ms":420.482,"prompt_tps":3671.976446078548,"decode_tps":67.15613024104664,"input_tokens":1544,"output_tokens":200,"peak_rss_mb":5214.3984375,"verdict":{"state":"Pass"},"prompt_ids_sha256":"6624743530f806ebfeef29164bc538770adccaf112ec5c254583759b9530f2af"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"11ca5b1e87d2bbf217c1fd81b2c2ea63a96390dea93e9d9258aa7dff541f8de8","stratum":"2k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":4565.2265290000005,"prompt_ms":392.788,"prompt_tps":3897.776917828447,"decode_tps":67.17037117370087,"input_tokens":1531,"output_tokens":276,"peak_rss_mb":5419.828125,"verdict":{"state":"Pass"},"prompt_ids_sha256":"7f03bbbdd56a316389c60433ff343e431f7bf8984965b1d7c775d0a2b663d42c"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"7ba9cfea210f85f585654545dc13dc487fe37b56924c0996577c87a463100bac","stratum":"2k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":1647.149459,"prompt_ms":97.526,"prompt_tps":2809.5072083341884,"decode_tps":68.06332611862076,"input_tokens":274,"output_tokens":101,"peak_rss_mb":5624.48046875,"verdict":{"state":"Pass"},"prompt_ids_sha256":"4a21bcf8590e6ea0af00d401671ef07b3715194ed9f4846feab7f8ce28239f10"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"b1c9fb8c5a26aa63dcdb57e6ec6ce35a2c5a73287516f57082f77944c007add1","stratum":"2k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":2209.5865959999996,"prompt_ms":216.671,"prompt_tps":3655.311509154432,"decode_tps":67.79396387198034,"input_tokens":792,"output_tokens":133,"peak_rss_mb":5739.375,"verdict":{"state":"Pass"},"prompt_ids_sha256":"2f02b9baa0667bc81604dd82b8f81abd390a46442f75a67fed1f9ca6ef54d929"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"9d82e15d5df69ff6b6a2f1d4dd81ba144dca2978ee4f352de1b134cf86a4bfdd","stratum":"2k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":4459.807895,"prompt_ms":427.506,"prompt_tps":3777.724757079433,"decode_tps":67.2020344849631,"input_tokens":1615,"output_tokens":268,"peak_rss_mb":5919.03125,"verdict":{"state":"Fail"},"prompt_ids_sha256":"7e2fd107b311a51bce7d7d6826cbde8412778be86ccc57cd60587b5bf24a54f4"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"4dca9a17db342792f25605430639922e342855e3341653071266173c29d9a859","stratum":"2k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":4132.531851000001,"prompt_ms":315.014,"prompt_tps":3752.214187305961,"decode_tps":66.74035802851662,"input_tokens":1182,"output_tokens":250,"peak_rss_mb":6128.6484375,"verdict":{"state":"Fail"},"prompt_ids_sha256":"cb47cabf619ed11eda1601028c79ced363020fe66316b9895d5602b2f527c0d8"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"e8e5551522f3e72418df3e0fa2227dded865de9949b65a4b85becc6cd3acbc35","stratum":"2k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":1557.7307190000001,"prompt_ms":116.043,"prompt_tps":3309.11817171221,"decode_tps":67.3836679567221,"input_tokens":384,"output_tokens":93,"peak_rss_mb":6321.73828125,"verdict":{"state":"Pass"},"prompt_ids_sha256":"0655ead49567576a1c526b6f710c1807619174567ee1f131159a9ae03c68b8ed"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"e0cf54bdaa7fbbbd164db0ebf846b77b0b78b65b8bfd4a392fcc3171c64a2504","stratum":"2k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":1987.489528,"prompt_ms":183.766,"prompt_tps":3292.230336406082,"decode_tps":67.17522486621883,"input_tokens":605,"output_tokens":119,"peak_rss_mb":6439.55078125,"verdict":{"state":"Pass"},"prompt_ids_sha256":"a7da9728eaf7dc43519a3329e56fbddffb918e60cd11e3b60c2fdd066013112c"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"7492d83b3314d90dae86dcb3530baaa502e8d425427daa63a8e9fc03de96b069","stratum":"2k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":2161.3729749999998,"prompt_ms":341.063,"prompt_tps":3796.9524691919087,"decode_tps":66.88638176116989,"input_tokens":1295,"output_tokens":118,"peak_rss_mb":6613.08984375,"verdict":{"state":"Pass"},"prompt_ids_sha256":"3710b42517c7bd8dcd003e4e02e46f2547485a103ea66db61273afabb7a3c2bf"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"0d9bcac1ca95dc9fb8293713b22add78150aaaacfa6a269e8065ea5605feee17","stratum":"2k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":5019.881084,"prompt_ms":259.748,"prompt_tps":3711.2894035757736,"decode_tps":66.9997164247671,"input_tokens":964,"output_tokens":315,"peak_rss_mb":6808.0078125,"verdict":{"state":"Fail"},"prompt_ids_sha256":"5ddc4f9e03e618df9e68b2df8ad867702dff23019b64b9d2ef32f96ad0949717"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"3f51712000a76356352961b38d5b137cdf5179679655ed3fbfef2acad4eeb0e8","stratum":"2k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":2244.879758,"prompt_ms":203.046,"prompt_tps":3388.394747988141,"decode_tps":67.0401275793943,"input_tokens":688,"output_tokens":133,"peak_rss_mb":6998.7578125,"verdict":{"state":"Pass"},"prompt_ids_sha256":"d7e97aaa3ac6609d18fe9c3ba27bfcd91550e16482bf475e2d4f19eb26acd77a"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"b7e5b398d98a7f3dc848eee0e437e07104d372e9365e6f3ce0c101e6667ed890","stratum":"2k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":1944.00348,"prompt_ms":103.417,"prompt_tps":2978.2337526712245,"decode_tps":67.01514542286556,"input_tokens":308,"output_tokens":120,"peak_rss_mb":7172.796875,"verdict":{"state":"Pass"},"prompt_ids_sha256":"b05019b1e8f3ab2be02e8b81733a49d390f98ae5a16b0e8f13d09593ea04b9e6"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"903ef35fb09bf1cb357547ce61b9b7828b90c24845aa5f9fecd8923e022be75f","stratum":"2k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":2778.8371899999997,"prompt_ms":258.413,"prompt_tps":3711.1136049656943,"decode_tps":67.03663350103261,"input_tokens":959,"output_tokens":167,"peak_rss_mb":7289.0234375,"verdict":{"state":"Fail"},"prompt_ids_sha256":"ec9c7f9e8c70ed80adfbcb818a9db16b1472d69cc4b7d5152bd8e139204963b5"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"34186bc542a9bc6191d88846aadf532b79f67209e4b17474d1d4b0a1cf87cc8c","stratum":"2k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":1904.058925,"prompt_ms":182.203,"prompt_tps":2886.8898975318734,"decode_tps":67.2974197448226,"input_tokens":526,"output_tokens":113,"peak_rss_mb":7474.96484375,"verdict":{"state":"Pass"},"prompt_ids_sha256":"717745ed7fb623d700dffd2b1f7f0d7ae241a0fc248f74ed4b935313a1a249fc"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"c9a02d51ce74d1c65af0fa596da511b8d2a571aa8f65f8d380ed9838b4c5243e","stratum":"2k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":4111.040422,"prompt_ms":272.093,"prompt_tps":3722.991771195878,"decode_tps":67.00790375440252,"input_tokens":1023,"output_tokens":254,"peak_rss_mb":7695.9296875,"verdict":{"state":"Fail"},"prompt_ids_sha256":"0c2f2e99acdaace2521543f52e497a9e01039e659d7c95ffb47efa75d6a0e742"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"d09cb9713ee02abec5dce5dd4cd0b58237e449a0032fdaf96dcc6ac95a0a6101","stratum":"2k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":2052.274206,"prompt_ms":159.574,"prompt_tps":2970.4087131988917,"decode_tps":67.15735573966947,"input_tokens":484,"output_tokens":123,"peak_rss_mb":7934.44140625,"verdict":{"state":"Pass"},"prompt_ids_sha256":"6ab504c554d5d36c516e432f36c59eac000d1d9af5d19121bcc80d31a4dd1ace"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"a889b6ddfa18809d32510029c8f832ddf1dc03f3209feee377d47043c757104e","stratum":"2k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":1918.143636,"prompt_ms":185.291,"prompt_tps":3205.768224036786,"decode_tps":67.14825555426988,"input_tokens":604,"output_tokens":113,"peak_rss_mb":8106.55859375,"verdict":{"state":"Pass"},"prompt_ids_sha256":"909814ad448169a742c52875b82d6ef865c02703afda46a7450ffbe8394e794a"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"c1d14d2cd0dbbdd90638158ee715c79fc76baa8ff2c6322160dac0bb5bd27771","stratum":"2k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":2712.1600209999997,"prompt_ms":128.835,"prompt_tps":2499.3208367291495,"decode_tps":66.7286270604961,"input_tokens":332,"output_tokens":168,"peak_rss_mb":8327.5703125,"verdict":{"state":"Fail"},"prompt_ids_sha256":"82eb2fb253fce96176b306689444000ece53e63e156d9fa4863a938a3e705b71"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"955705510e7439ac4b8e27505f87bcb7d3c71b677cb11bddc4a91d5c44e16a04","stratum":"2k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":3186.921764,"prompt_ms":459.495,"prompt_tps":3878.170600332974,"decode_tps":65.96503852957933,"input_tokens":1792,"output_tokens":177,"peak_rss_mb":8496.4296875,"verdict":{"state":"Fail"},"prompt_ids_sha256":"51eeb6fcedffb81fd02fc9c5f05805cb72211d244fb56aad4117e964b187a2d0"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"bf94290e1f98fc502cb4f962bce2883c3eaa9ca2fe6cb67f6b87c7083d3ec0e1","stratum":"2k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":4139.867432,"prompt_ms":337.6,"prompt_tps":3744.075829383886,"decode_tps":66.75082134588573,"input_tokens":1274,"output_tokens":248,"peak_rss_mb":8758.9921875,"verdict":{"state":"Fail"},"prompt_ids_sha256":"05a06916a4b3576cd25ee421c0588937e59d599607237eeab47bec5ef1db27eb"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"258be4b71904b9f67bdda4ea10d76b9be53c374b04a9d486facd87a97fab0756","stratum":"2k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":2463.686096,"prompt_ms":322.887,"prompt_tps":3725.761644166535,"decode_tps":64.73399196519745,"input_tokens":1213,"output_tokens":134,"peak_rss_mb":9007.57421875,"verdict":{"state":"Pass"},"prompt_ids_sha256":"d1fbddde370dc12c4aec16fbd2fc2fb32418489c008dd55dd3d29e40a5fbd855"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"1bf59f2a1774a0efe188bbfa1ac4a7af14b2059017de009fc98df2a6b8382017","stratum":"2k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":5658.395382,"prompt_ms":489.897,"prompt_tps":3635.4580656750295,"decode_tps":64.12039640215524,"input_tokens":1791,"output_tokens":326,"peak_rss_mb":9250.86328125,"verdict":{"state":"Fail"},"prompt_ids_sha256":"6c5a3de9322024e61926b8105522954447661224770b9881c91a10be3aa69a7c"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"305cdbe50000c485ff10ec6127fbe0744109a126be589ccbb6d243834c3c13ef","stratum":"2k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":2722.554379,"prompt_ms":282.878,"prompt_tps":3634.0754671625223,"decode_tps":65.80340730914615,"input_tokens":1038,"output_tokens":152,"peak_rss_mb":9329.12890625,"verdict":{"state":"Fail"},"prompt_ids_sha256":"0d2fc4877c6ceccc5bcecc9e02a8dc8468a8265069ac9e54235fb1f170a59c3e"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"7680438d94dac7e6c61561a71575435fcd22d1543b508ec9434d00f686511896","stratum":"2k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":4750.999116,"prompt_ms":252.904,"prompt_tps":3594.2492012779553,"decode_tps":66.64187497938765,"input_tokens":919,"output_tokens":294,"peak_rss_mb":9390.859375,"verdict":{"state":"Fail"},"prompt_ids_sha256":"0ab5ac2510b12bcacb54e31107686f33fa3f624fb80ca84c81fbced4c8e24f46"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"d928b82fd55ca6c509650e822de0f3946d5ccb3d5fbbabba30784e0f6483e2ff","stratum":"2k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":4885.350393999999,"prompt_ms":376.033,"prompt_tps":3770.945635090537,"decode_tps":63.91953690409248,"input_tokens":1428,"output_tokens":282,"peak_rss_mb":9425.8828125,"verdict":{"state":"Fail"},"prompt_ids_sha256":"ef93f1ea44b6a4051e70202045ed0b6322e8987557045343e51ca8cdc8074df0"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"64cce1452c38a191c5f249f069acb21eb4098bd2620cddec04b346c3075849dc","stratum":"2k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":4182.274738,"prompt_ms":202.762,"prompt_tps":2973.9300263362957,"decode_tps":61.963188090045115,"input_tokens":613,"output_tokens":237,"peak_rss_mb":9423.62109375,"verdict":{"state":"Fail"},"prompt_ids_sha256":"337e04b1920ed7b30ceedf4ff49f4bf841d2ce7657bafdb2a9165eea27d39ea6"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"69b831cc2059acf0f511c8d788337f36e8ef890f6b86b91f6ffa159b8369d1e5","stratum":"2k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":3021.455301,"prompt_ms":528.575,"prompt_tps":3649.434801116208,"decode_tps":63.71587613971246,"input_tokens":1939,"output_tokens":152,"peak_rss_mb":9432.21875,"verdict":{"state":"Pass"},"prompt_ids_sha256":"d93d068e263876cd1cb37558b0015f0fdcf11be3da9eefe2c4aff454b8a9e616"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"14d125f8a319774d2ddc89c330d806c3371695a96ec1cc8757cb63f20c9afe37","stratum":"2k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":3028.883056,"prompt_ms":186.66,"prompt_tps":3048.3231543983716,"decode_tps":65.94681556985687,"input_tokens":579,"output_tokens":178,"peak_rss_mb":9430.05078125,"verdict":{"state":"Pass"},"prompt_ids_sha256":"04222af6320ccf0f9b9bc386e01fc9ed98736d6df5ee61bd98894f8facc87347"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"0b54a849b9e7541ec803e57fab1147aee8d209935102b9230c0a024229ccdac7","stratum":"2k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":2255.502349,"prompt_ms":187.516,"prompt_tps":2959.7474348855567,"decode_tps":66.43557308091904,"input_tokens":565,"output_tokens":131,"peak_rss_mb":9430.05078125,"verdict":{"state":"Pass"},"prompt_ids_sha256":"6ea89d378295c73f8cfc8d17f5f2fcab76f6abb4e8c59463e8a4a4025885fed7"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"ae0f31c8b5b43715e0f5f31727de5648fe149c06faef682adb8898f4dddc0d59","stratum":"8k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":3959.179894,"prompt_ms":548.99,"prompt_tps":3843.421555948196,"decode_tps":64.8900609155447,"input_tokens":2159,"output_tokens":217,"peak_rss_mb":9430.05078125,"verdict":{"state":"Pass"},"prompt_ids_sha256":"a81f7ea1e1114b3b88a0717effd69745ef46e82f2efdcdbd465afae931ebf1f0"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"6324ab9b418859326df39301f2fc9ca30815280479d95ad375d91e6252a4b82b","stratum":"8k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":5831.42368,"prompt_ms":884.288,"prompt_tps":3985.1270174422816,"decode_tps":64.76683937823834,"input_tokens":3573,"output_tokens":310,"peak_rss_mb":9430.05078125,"verdict":{"state":"Pass"},"prompt_ids_sha256":"3ae7c4e045aeaa9df4ea1cf4961254d625990b616146ceda3ec66666473b7cff"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"187723585ce31ae0a9dfb2bd1c94ee372724ba456ade6c3ef28816fb1e369e4d","stratum":"8k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":3449.312003,"prompt_ms":575.858,"prompt_tps":3849.9074424597734,"decode_tps":65.1183617683513,"input_tokens":2266,"output_tokens":179,"peak_rss_mb":9430.05078125,"verdict":{"state":"Pass"},"prompt_ids_sha256":"eb266f1f82f6cd93fd1590494e67349a0bed128c01781f180d09e71b40998f5d"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"dd16771d607cda7d13b2528bec93be7261e9d739a6d40c855b0794500da92522","stratum":"8k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":5410.678807,"prompt_ms":1205.286,"prompt_tps":4072.8922430029056,"decode_tps":64.36992165541749,"input_tokens":4958,"output_tokens":263,"peak_rss_mb":9466.203125,"verdict":{"state":"Fail"},"prompt_ids_sha256":"51ee3f91d5eaa6d044852fa83079495964304a501fbc737ecc01becf473718b6"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"40ce4b133ac399054d67393d53e91be97d3cafaa33622094f90a35fdea53c703","stratum":"8k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":9893.540419,"prompt_ms":1690.963,"prompt_tps":4075.78403548747,"decode_tps":63.57139098308864,"input_tokens":6941,"output_tokens":512,"peak_rss_mb":9464.78515625,"verdict":{"state":"Pass"},"prompt_ids_sha256":"212c625213d37b2c9f816b691908099bd9f9a58e680ca2d15863dcdabb735b51"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"bc9a159757fe4720bfdc9073b5c2d8879c8adec829ab9308b1b1031768b5c3f0","stratum":"8k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":7098.766436,"prompt_ms":1591.524,"prompt_tps":4065.914180370514,"decode_tps":64.04312136378697,"input_tokens":6520,"output_tokens":341,"peak_rss_mb":9464.78515625,"verdict":{"state":"Pass"},"prompt_ids_sha256":"966d327b207b9dda0ee074cf93126d6091ceef81459965ec9b5b1a03824d5f5f"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"33b63a5283685481a92f5226d573249f59330892462fc75a3c5ff14e9c61ba82","stratum":"8k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":4584.333576999999,"prompt_ms":1666.321,"prompt_tps":4046.0391485194023,"decode_tps":64.3330863617471,"input_tokens":6791,"output_tokens":179,"peak_rss_mb":9464.78515625,"verdict":{"state":"Pass"},"prompt_ids_sha256":"8d3561cb81010d0287b8d36013ce4022187e868e05fb5a2acba13d93ddd0ba75"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"df48ffe5f40ed18ea677fa829b51ee4eaaee5c8caa5a7462bbe212d83396c540","stratum":"8k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":6169.2974779999995,"prompt_ms":834.303,"prompt_tps":4068.0663979393576,"decode_tps":66.31086299904257,"input_tokens":3443,"output_tokens":343,"peak_rss_mb":9464.78515625,"verdict":{"state":"Pass"},"prompt_ids_sha256":"513d5ac32c370db50606bba64bf9369a630e11ff0935f649326b6b1082750baf"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"cbc1450d830a7bda671630905c4b86039bf483e305a0d8fecf6971de02ad0406","stratum":"8k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":6131.0274549999995,"prompt_ms":1650.493,"prompt_tps":4077.569550431296,"decode_tps":64.38384544933831,"input_tokens":6779,"output_tokens":281,"peak_rss_mb":9464.78515625,"verdict":{"state":"Fail"},"prompt_ids_sha256":"c9b299004f9b7cc8d7af8cf6262f62e552c1752be74d07b00d51c812552f258f"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"62b4bd91655d82632f2ad6754eac98e25881dcdfeb3eff15172402b9da179800","stratum":"8k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":2576.0899679999998,"prompt_ms":950.395,"prompt_tps":4066.730149043293,"decode_tps":66.27417626721748,"input_tokens":3914,"output_tokens":97,"peak_rss_mb":9464.78515625,"verdict":{"state":"Pass"},"prompt_ids_sha256":"1d6f64f5e542ef9606b341f0aa9e816aad85f379d21ff095d0138038b8b4ba4a"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"a6c9bb4b1ea341e343c764aed2b694ca7f4c6ab69461291cf6e20a2d1fe53a47","stratum":"8k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":4984.457474,"prompt_ms":738.111,"prompt_tps":4053.5908555759233,"decode_tps":67.38693627999518,"input_tokens":3041,"output_tokens":275,"peak_rss_mb":9464.78515625,"verdict":{"state":"Fail"},"prompt_ids_sha256":"a5e089a434b2f460a61b44fe52bddb2e49e8dc5668c0e6de4a0f0bad9f8fe848"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"1239ceb996f617d8cda28ed536075072f460a875c5a4a25a2ef422a9d923043e","stratum":"8k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":3636.991185,"prompt_ms":1040.219,"prompt_tps":4086.639448039307,"decode_tps":66.23586429725363,"input_tokens":4300,"output_tokens":165,"peak_rss_mb":9464.78515625,"verdict":{"state":"Pass"},"prompt_ids_sha256":"f276cbcdd3b7b2349836fbdb1d397d0d986373840f6ddaddc46c581e5f7288df"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"90134b8a8a941ee22156f752e986d85a39c92f11186fff94174ce25e7fc2c65d","stratum":"8k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":4092.066455,"prompt_ms":782.398,"prompt_tps":3953.230964291831,"decode_tps":66.49918956082486,"input_tokens":3142,"output_tokens":212,"peak_rss_mb":9464.78515625,"verdict":{"state":"Fail"},"prompt_ids_sha256":"1fb374c347b9be341a38517cceab0992299ad2aaea74befbaf3906626c37d34b"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"b3c917322e600c6a75b1fdabd254bbb864628227545b9f51c05fedcb33835e55","stratum":"8k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":4527.778848999999,"prompt_ms":672.99,"prompt_tps":4006.00306096673,"decode_tps":67.2438745116782,"input_tokens":2745,"output_tokens":252,"peak_rss_mb":9464.78515625,"verdict":{"state":"Fail"},"prompt_ids_sha256":"a24ccc6f60289e1cfdec88d1bedae95fd3e07227a8da2a84b31e646d817ee93c"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"38f0a202313d9e2120e55b92747ced425cac793e09a4db44c212449b1cd8566e","stratum":"8k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":8999.001568000001,"prompt_ms":1113.796,"prompt_tps":4075.2525597147055,"decode_tps":65.95778185657603,"input_tokens":4588,"output_tokens":512,"peak_rss_mb":9464.78515625,"verdict":{"state":"Fail"},"prompt_ids_sha256":"95b34da10969f03fa2337931389f1835b5efc1eb5f79585e13cb239ef9503d00"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"e12a747f914c805b9dcbc4349d5ed7fe706e6d0334848a677837cef3d8b95400","stratum":"8k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":4152.250489,"prompt_ms":598.325,"prompt_tps":4041.281912004345,"decode_tps":66.72945479108705,"input_tokens":2467,"output_tokens":229,"peak_rss_mb":9464.78515625,"verdict":{"state":"Pass"},"prompt_ids_sha256":"0b4a3d5a626c23d2a0a7f227b0d4edfddd53b4da5a6bb9963171c2d1ab882b65"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"a74db9b8ff37392fa217b05b5e70a2564d3c8a6960f02eebe3db8016184e114a","stratum":"8k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":6416.5149679999995,"prompt_ms":1432.422,"prompt_tps":4127.275342043057,"decode_tps":64.89795623108219,"input_tokens":5961,"output_tokens":315,"peak_rss_mb":9464.78515625,"verdict":{"state":"Pass"},"prompt_ids_sha256":"ad6c508530582a081bce003291bb7ee3c17effe3ed0527e73172c39694fbc89b"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"27565a444276a49af4557cade6d92398cbccd183c6366e78a5453683923bfe6b","stratum":"8k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":5423.524783999999,"prompt_ms":1922.675,"prompt_tps":4070.370707477862,"decode_tps":63.50196334356619,"input_tokens":7875,"output_tokens":214,"peak_rss_mb":9464.78515625,"verdict":{"state":"Pass"},"prompt_ids_sha256":"72815a6948e2f27d022cec776bb31fd4a99af1123e5c14a25c6fcb175a28e00a"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"e0d0c14ea617d00ecec166c8e598898b3d1b757d639df1729ddf1e3af37872cd","stratum":"8k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":2844.116835,"prompt_ms":689.352,"prompt_tps":4029.8715315252584,"decode_tps":66.55182211209886,"input_tokens":2827,"output_tokens":131,"peak_rss_mb":9474.72265625,"verdict":{"state":"Pass"},"prompt_ids_sha256":"5019b3cebd26be737bb7fe45f6ddb10faa0e2c3bb6fd157fa1eee1deea81d19d"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"7269a7b0e6abf53a633d344677435b768b4342ad507d8833da2aa1b5b90747ee","stratum":"8k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":5445.273189,"prompt_ms":1121.67,"prompt_tps":4092.112653454224,"decode_tps":65.53674594932498,"input_tokens":4639,"output_tokens":274,"peak_rss_mb":9472.4296875,"verdict":{"state":"Pass"},"prompt_ids_sha256":"b76967bf52a3d1d50f5eeb855e43df852e8ff8be2ed8bb4909ea519ca09fa086"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"f87c2ffe3aafeec5f8308352f5190b3c8ec03ed0eafd6d94b000ef39bb3d922f","stratum":"8k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":6586.9527259999995,"prompt_ms":1196.139,"prompt_tps":4118.25047089009,"decode_tps":65.08914253930496,"input_tokens":4975,"output_tokens":342,"peak_rss_mb":9472.4296875,"verdict":{"state":"Fail"},"prompt_ids_sha256":"805c7c9887566084d332a0ff4e3decc7ee4a100a69194e545f3460c67c1fb92a"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"c7323e9dc8cc06122998a5fb83ccb487d2538b7444f9852de80f63983e93dc44","stratum":"8k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":6487.793186,"prompt_ms":924.614,"prompt_tps":4065.480297724239,"decode_tps":65.57916405461523,"input_tokens":3808,"output_tokens":355,"peak_rss_mb":9472.4296875,"verdict":{"state":"Fail"},"prompt_ids_sha256":"b5cb29cd4d1d2b02437000ebf0115b5483e5b7bf00187bc0d72bb47ba8be7019"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"3bed1317707a01f00daf828a5434d3681278c66100d991520153baa4dc7ebd61","stratum":"8k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":5361.985294,"prompt_ms":513.177,"prompt_tps":3928.469124687973,"decode_tps":66.94887536550036,"input_tokens":2065,"output_tokens":315,"peak_rss_mb":9472.4296875,"verdict":{"state":"Fail"},"prompt_ids_sha256":"92f0703d894d6fb99af6cf992d9c3d76abd4fad16e00adf056499d5bf79df42a"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"a974a820da6ee897219e5353ad22358cd5cec02a3534baa60388a3f554850d96","stratum":"8k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":5209.314326000001,"prompt_ms":1053.926,"prompt_tps":4072.3921793370696,"decode_tps":65.52443999562178,"input_tokens":4341,"output_tokens":265,"peak_rss_mb":9472.4296875,"verdict":{"state":"Fail"},"prompt_ids_sha256":"4017ef5b163960754cdb6c87147d51d9110da0dcbb8d7f37fc5b1b1af7c2f8d4"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"1c50664db77045b861287a6f03ded90d514b8d0a32f04336caa5c9c90777b18c","stratum":"8k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":3109.560815,"prompt_ms":551.544,"prompt_tps":3930.783400780355,"decode_tps":66.7340201587425,"input_tokens":2217,"output_tokens":159,"peak_rss_mb":9472.4296875,"verdict":{"state":"Pass"},"prompt_ids_sha256":"ad1b4023aa37427996cc325caba661c60a9cf8343640671dcb1983cfbc5c89b8"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"7e09255545a53208daf6239daa699089bbd30c3ec111d44a59cbe007a8004a57","stratum":"8k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":3475.383935,"prompt_ms":952.264,"prompt_tps":4063.999059084456,"decode_tps":65.52221202987813,"input_tokens":3919,"output_tokens":158,"peak_rss_mb":9472.4296875,"verdict":{"state":"Pass"},"prompt_ids_sha256":"7979c0667fd9abec64e255baa8d0377e7856b52e4e008e1f1b7c35c4471ccdf2"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"1aee819c753ccc3dd6976511c879d33fd99478a25bfe8c5639d006aebe6d00ad","stratum":"8k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":3570.958146,"prompt_ms":1166.951,"prompt_tps":4076.435085963335,"decode_tps":65.36651908860138,"input_tokens":4806,"output_tokens":146,"peak_rss_mb":9472.4296875,"verdict":{"state":"Pass"},"prompt_ids_sha256":"93d0c8edbd6a7cca000f9f23b65b7bd49a90c3e64f4d169cf4705ff14a6bbaf9"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"e92967237c1b1abeb4a548c343928a0afda2c3df6c49e94cec87b7146276b045","stratum":"8k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":5898.443946,"prompt_ms":985.726,"prompt_tps":4034.589733861134,"decode_tps":65.31090527255573,"input_tokens":4026,"output_tokens":310,"peak_rss_mb":9472.4296875,"verdict":{"state":"Fail"},"prompt_ids_sha256":"96c30189e094a2ed3603508cd4222ab22c468239354545620eebf61674389fb1"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"5ef3cfe455ce3ddedb5fd853a62ccbda685b16f06e7ec8818c46f639474fd302","stratum":"8k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":5736.403941,"prompt_ms":688.713,"prompt_tps":4016.1867134786194,"decode_tps":65.63938983090925,"input_tokens":2815,"output_tokens":323,"peak_rss_mb":9472.4296875,"verdict":{"state":"Pass"},"prompt_ids_sha256":"bc74840b53b9b199026a3b937a88491510f44071bb5e597f5a05fc460493e8e5"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"08db5d324448f2faa678eca074285b6275f42db2c667f594ede9fa4adfc9c566","stratum":"8k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":5459.679552,"prompt_ms":1402.761,"prompt_tps":4027.058066199445,"decode_tps":63.89617944296333,"input_tokens":5698,"output_tokens":251,"peak_rss_mb":9472.4296875,"verdict":{"state":"Pass"},"prompt_ids_sha256":"759d47a6fcd52490cc43e3f9f91db6587e8b59bf2ed44a06dae3647f1cab6df2"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"bc995711883be2b01bb079a4433bb34b0eb50a9cda6440856b8c0665d7a4afb5","stratum":"8k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":3031.208526,"prompt_ms":685.038,"prompt_tps":4008.536752705689,"decode_tps":66.21956199438605,"input_tokens":2795,"output_tokens":145,"peak_rss_mb":9472.4296875,"verdict":{"state":"Pass"},"prompt_ids_sha256":"6a64782f6b63fcb21b6814ed2634f945270c9fb500f9ec47abffff02026b3b33"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"9f29c99424409b984aeea5524ec870e016e2e16c617351a938ed700a057ce9ba","stratum":"8k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":6008.92166,"prompt_ms":1830.49,"prompt_tps":4061.2076547809606,"decode_tps":63.353305297845914,"input_tokens":7483,"output_tokens":257,"peak_rss_mb":9472.4296875,"verdict":{"state":"Pass"},"prompt_ids_sha256":"f79d101859efe06916d34257165ec054aa373990f5d969fc6c2513c9ee1dd354"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"3d765c54312ef5c279e895fb042a9c08ef9bcd6b44e1098b9db17f68d379f845","stratum":"8k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":5045.6199799999995,"prompt_ms":860.411,"prompt_tps":4027.145166670347,"decode_tps":63.746923203098966,"input_tokens":3514,"output_tokens":254,"peak_rss_mb":9491.68359375,"verdict":{"state":"Pass"},"prompt_ids_sha256":"afa4aed5cdc97dd82a2f3c6f1c3cf83de33f5a873bda726cf8cbedecc68e184a"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"b66cef24141dfcf0cd4776c23e13ac2a297f86fb966ea2f474d3ba2b22ac4689","stratum":"8k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":5769.966579,"prompt_ms":550.856,"prompt_tps":3928.431386787109,"decode_tps":64.1641468461452,"input_tokens":2213,"output_tokens":327,"peak_rss_mb":9489.34375,"verdict":{"state":"Pass"},"prompt_ids_sha256":"a7ed93b0333e5ba437c439c1d28332f7c2c21e02f641095bc38eadd1a02cd167"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"bf69f8045076d4b114aef30013614158379bdcec1dddb96497b724fd4bd16df2","stratum":"8k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":4335.970449,"prompt_ms":1171.523,"prompt_tps":4018.7004437813007,"decode_tps":64.94059426607686,"input_tokens":4757,"output_tokens":197,"peak_rss_mb":9489.34375,"verdict":{"state":"Pass"},"prompt_ids_sha256":"ef2a1c7ad19234288174165afe4ff5dc308ed4a5d5448495732bf50390f28fde"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"0ba74445847c37901a423c32a811b0f15026d75d3dd2225fd67ab1a474f97ae2","stratum":"8k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":3825.556261,"prompt_ms":1345.37,"prompt_tps":4076.2020856716,"decode_tps":64.50629488655066,"input_tokens":5533,"output_tokens":147,"peak_rss_mb":9489.34375,"verdict":{"state":"Pass"},"prompt_ids_sha256":"5fc6cf167f33d8897f20f6457b0677a778f646264bdb552be4e00fb52b0e6b9f"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"a08ed5ccc3331e0a7179c7d49737607bccb9a9cfbc16c6533afacfa6072b133f","stratum":"8k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":2396.711656,"prompt_ms":573.513,"prompt_tps":3977.2420154381857,"decode_tps":67.13096885855383,"input_tokens":2330,"output_tokens":111,"peak_rss_mb":9489.34375,"verdict":{"state":"Pass"},"prompt_ids_sha256":"baf2d30022d1dbc62e43697b26bd1999ee610fcd6b4275293cd352e091390a04"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"b0bdb087e1d95680866dad2b47061fa38e925929fe937f39b834992f44838399","stratum":"8k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":5008.138475,"prompt_ms":1496.316,"prompt_tps":4074.674066173188,"decode_tps":64.06592771337083,"input_tokens":6146,"output_tokens":216,"peak_rss_mb":9489.34375,"verdict":{"state":"Pass"},"prompt_ids_sha256":"62861c84c3f9994a8f69de8848af2f7ea98e8e6de26bf95f24f760b6c5975b69"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"6e56cee326693c4915a7feef43eee6a597f75ff2e54f970e8f3197135869aed1","stratum":"8k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":6705.66471,"prompt_ms":1714.338,"prompt_tps":4079.1255866696065,"decode_tps":63.252515067727,"input_tokens":7042,"output_tokens":305,"peak_rss_mb":9489.34375,"verdict":{"state":"Pass"},"prompt_ids_sha256":"6383d5bff555ad33b2339b6120920cf9522aa807a1285f41b3efcface1525924"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"c81288783ae98e2b5a143cdfc4738ce6c817ae09e2b01a0979e18a0de5f90bb6","stratum":"8k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":5093.330561,"prompt_ms":1806.11,"prompt_tps":4073.948984281135,"decode_tps":63.40417671779221,"input_tokens":7407,"output_tokens":197,"peak_rss_mb":9496.90625,"verdict":{"state":"Pass"},"prompt_ids_sha256":"3c18df6c64b92c1d238f0b099ce92e3e85ec2f7c4a8a113995df1d4036e878bd"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"0354fce0d010d6d69a1539589a7d6046495172ee6865f418b0a39c852edae583","stratum":"8k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":6175.56995,"prompt_ms":981.294,"prompt_tps":4070.135963330052,"decode_tps":65.45480245499681,"input_tokens":4043,"output_tokens":327,"peak_rss_mb":9494.6796875,"verdict":{"state":"Pass"},"prompt_ids_sha256":"04371fb2a9cfff6184614b3943d1f0afebf2446e3ae21acce139cb914a34632b"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"b45b72bc1a202cb0504dfd27dd16f60af1ec09570fe31e7ecd927168837a02cf","stratum":"8k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":4843.953477999999,"prompt_ms":1458.262,"prompt_tps":4078.8280843908706,"decode_tps":64.38701549157784,"input_tokens":5997,"output_tokens":209,"peak_rss_mb":9494.6796875,"verdict":{"state":"Pass"},"prompt_ids_sha256":"b4ac6d8b05c4c10cf4f5c3388011ec14c3513f17aefa6dc582e2dff4d2968405"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"b6dabc525c8486ab7bab78b2968c241327371b5c234ca1b8a87adb93bb8768ab","stratum":"8k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":4873.575125,"prompt_ms":1175.951,"prompt_tps":4071.598221354461,"decode_tps":65.33641721222575,"input_tokens":4837,"output_tokens":231,"peak_rss_mb":9494.6796875,"verdict":{"state":"Pass"},"prompt_ids_sha256":"f6a335774a9b2c8389c1c66b5157283cfdff4ebf3dead387979335582189e2cd"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"fd684a94b71725b926c4fb5be284b7c83518a537c466c552d7dbc8c9c1f64bec","stratum":"8k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":3696.1149669999995,"prompt_ms":561.888,"prompt_tps":3981.220456745828,"decode_tps":67.02225887466565,"input_tokens":2286,"output_tokens":198,"peak_rss_mb":9494.6796875,"verdict":{"state":"Pass"},"prompt_ids_sha256":"15033c21d4c89645a7f23d359d8c331c1cdae370c57dc97d3aa3ba5c4b935347"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"c5a7b5a62a110bae58ca259b02ea467d3dcdbde7ad3308a657e8028e63322e39","stratum":"8k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":4500.712842999999,"prompt_ms":967.749,"prompt_tps":4085.7701738777305,"decode_tps":65.58408413755446,"input_tokens":4003,"output_tokens":222,"peak_rss_mb":9494.6796875,"verdict":{"state":"Pass"},"prompt_ids_sha256":"6906e62e5d443f9ed949a4eb821a1ea194d8a3d072af1e52d8d0df23e1fe9f62"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"b71b016e3bf4e4aa82e2c46d8b1c2f137099cf683949b4c65e5900bad8de95ce","stratum":"8k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":2316.1416670000003,"prompt_ms":506.73,"prompt_tps":3958.7156868549328,"decode_tps":67.02961048416903,"input_tokens":2055,"output_tokens":110,"peak_rss_mb":9494.6796875,"verdict":{"state":"Pass"},"prompt_ids_sha256":"24e56a79ebb675cd95f21cf12c693fbac4af6e2b009309f49d8f85d8b61164e5"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"611ff958e756f16b6143ffe425e3351e94a71034d0306f0e00ebd25696e66a31","stratum":"8k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":3030.960717,"prompt_ms":864.536,"prompt_tps":4066.921446880177,"decode_tps":65.8141736806128,"input_tokens":3565,"output_tokens":133,"peak_rss_mb":9494.6796875,"verdict":{"state":"Pass"},"prompt_ids_sha256":"ac1e1de733551d5421552bcc206f001ac1b3efa35aa1a05a6ab785fd99ff12d8"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"1f97a6ca5af77396cc4febaacda124d2f993c18d3e3b96e3e0b28ff92f20610d","stratum":"8k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":4469.837407,"prompt_ms":1923.354,"prompt_tps":4069.973598203971,"decode_tps":63.2444574190984,"input_tokens":7877,"output_tokens":152,"peak_rss_mb":9494.6796875,"verdict":{"state":"Pass"},"prompt_ids_sha256":"c42007b3ae5fcfb14a8d8a6f5d25753226fc0910d9bc95d65e8b75cc5e4b05f1"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"3811dbd8d38716269bb87eca6a6039f05923f1c129d3d2b09659f4cff15dfc2e","stratum":"8k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":4331.559435,"prompt_ms":1104.191,"prompt_tps":4094.4003347246985,"decode_tps":65.30231856046126,"input_tokens":4570,"output_tokens":200,"peak_rss_mb":9494.6796875,"verdict":{"state":"Pass"},"prompt_ids_sha256":"ea94bd384af8194fc5c38a4dbcdc751afcf318dc8834c3f4560913718900fb35"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"f65d88532cd6ca202557458343fb035dbe140810881382b7ce5d8c668afda27d","stratum":"8k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":3737.452685,"prompt_ms":960.514,"prompt_tps":4071.7782354031274,"decode_tps":65.52952619485662,"input_tokens":3960,"output_tokens":173,"peak_rss_mb":9494.6796875,"verdict":{"state":"Pass"},"prompt_ids_sha256":"d01aacddaa6f22bdfb64eca2e82ca55373391a16f47162da338cdbbad3508144"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"6bcf0a0fb95dfe5e90aa366595a006bc443f7a3d9de3c86ba176541bdbbc01cc","stratum":"16k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":7036.521818,"prompt_ms":3937.562,"prompt_tps":3989.016553898072,"decode_tps":58.95092180625624,"input_tokens":15756,"output_tokens":172,"peak_rss_mb":9494.6796875,"verdict":{"state":"Pass"},"prompt_ids_sha256":"e898c162c9392f2d3bd457479d776fac6602c99eba6b04a3d80e8cecc0c2cc92"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"c70e109e3c965254163267ebeee325ed9dfebe4479eb874c38a592a3488af34e","stratum":"16k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":6937.013604,"prompt_ms":2723.417,"prompt_tps":4062.5434885660184,"decode_tps":61.550920415168584,"input_tokens":11113,"output_tokens":243,"peak_rss_mb":9494.6796875,"verdict":{"state":"Pass"},"prompt_ids_sha256":"6c1223de3236d024d3bf9a1a600bf90f1709debccf3ee61c2bd92011cb7c1968"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"9ec99261cd227d8f5e528bab7391b88aaafa4483233ccb0731a46b4f64bee8dc","stratum":"16k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":6886.141527999999,"prompt_ms":3359.576,"prompt_tps":4025.80563737805,"decode_tps":60.32710085320192,"input_tokens":13574,"output_tokens":199,"peak_rss_mb":9494.6796875,"verdict":{"state":"Pass"},"prompt_ids_sha256":"404ea4c6921bceb338fc1491d0dbe4aa8a2711122ca4646724bc17b6bcaaa069"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"9444f47e49f1779a103b611560b50ba89a8f20df19b976c9d4e06c2b01e103a5","stratum":"16k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":10121.52761,"prompt_ms":2171.355,"prompt_tps":4067.9667765059144,"decode_tps":62.760098552765065,"input_tokens":8882,"output_tokens":486,"peak_rss_mb":9494.6796875,"verdict":{"state":"Fail"},"prompt_ids_sha256":"8f31bc51da4630c662e6ec7e0c67c440f57a03802d09a5f5f286399b72e3c666"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"5e5b0935bd95c58dc95d3caebb22a4f41d0573018dff4ce1633bb7668afdd1a2","stratum":"16k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":7854.825761,"prompt_ms":3605.501,"prompt_tps":4016.362774549224,"decode_tps":60.03319157294749,"input_tokens":14530,"output_tokens":240,"peak_rss_mb":9494.6796875,"verdict":{"state":"Pass"},"prompt_ids_sha256":"b191956bebc847f0567b83979f08d07982cecd6a68b6d78eba5b6a707c96236e"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"df963cc72999db8bd18665c41caca7b3cd058ec66923e7f5f3c161415d0cd4a2","stratum":"16k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":5784.054418000001,"prompt_ms":2852.742,"prompt_tps":4057.8503068276063,"decode_tps":61.84711694384734,"input_tokens":11625,"output_tokens":165,"peak_rss_mb":9511.45703125,"verdict":{"state":"Pass"},"prompt_ids_sha256":"9b005fd461f84a7451b7d950c5dac4626187c3e983e8ef186cb0ac942803b1d9"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"675cb632f8d8a2bb1d48f14a4d927d717d45d10dd3c0cacbe19f6e2cf94455a7","stratum":"16k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":6896.511108000001,"prompt_ms":2175.999,"prompt_tps":4079.045992208637,"decode_tps":63.482740801111085,"input_tokens":8925,"output_tokens":279,"peak_rss_mb":9509.52734375,"verdict":{"state":"Pass"},"prompt_ids_sha256":"d017282343b77dda0b7ab50f3cb163775efea0802dee5ceeed7c6495c76cd8bf"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"c4cf0db8bffc8083c5691e577702ca73a932cb4cde56dd1c9dac360353ee38b1","stratum":"16k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":11145.952758,"prompt_ms":2740.176,"prompt_tps":4064.702413275644,"decode_tps":62.31648800371021,"input_tokens":11187,"output_tokens":512,"peak_rss_mb":9509.52734375,"verdict":{"state":"Fail"},"prompt_ids_sha256":"dc38252b026854e36f129b97bd87960e52b6fcb44f4e545c4860459f962308d2"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"aa3e63ab8090ebf9be4f8d697c9f96d692a2763af91a2ad7bf96d1a26c384025","stratum":"16k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":9779.364403000001,"prompt_ms":2426.779,"prompt_tps":4012.314265122617,"decode_tps":62.87894918047536,"input_tokens":9786,"output_tokens":449,"peak_rss_mb":9509.52734375,"verdict":{"state":"Pass"},"prompt_ids_sha256":"a2e6135129fe41dfbb6cefed737ccaecbb1acd94d176936ba19a8d4d00969e76"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"ea0520ed2859350cd1372043b3a06ff52b3e2f75fb0195b3d0521d1f23b3e7d6","stratum":"16k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":4658.441879999999,"prompt_ms":2127.496,"prompt_tps":4088.844350353655,"decode_tps":63.441473687759704,"input_tokens":8748,"output_tokens":144,"peak_rss_mb":9509.52734375,"verdict":{"state":"Pass"},"prompt_ids_sha256":"c132e4a7a5ce9599d799aadfe7b680427fb4469c2031225692c05d86ceefb87c"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"54246f96c1844cc9afe81e68e68bd564d60308d58851d0af96c22aeb7059b962","stratum":"16k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":8708.209079999999,"prompt_ms":2715.824,"prompt_tps":4032.293697971591,"decode_tps":62.38475669080836,"input_tokens":11000,"output_tokens":362,"peak_rss_mb":9509.52734375,"verdict":{"state":"Pass"},"prompt_ids_sha256":"51ccafe7aeaa0580788c3daf75e8b77c84449cb6f48369075c3a4e317d97ec7a"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"014e7869f90798986ecf651ca3a959bdf86f6369b6ce3d5ff71ecf0fd7cf7417","stratum":"16k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":7689.847451000001,"prompt_ms":2952.573,"prompt_tps":4051.0429378037393,"decode_tps":61.83761754170243,"input_tokens":12010,"output_tokens":278,"peak_rss_mb":9509.52734375,"verdict":{"state":"Pass"},"prompt_ids_sha256":"9bf4cad8197210b509c2cc01811950c93a723c31ef9acd1dd2d4eb9ced1a48e1"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"2cdf5aadfe826822cbeef1cfad68e8d2d21d6bb99bc84ba02b50ad8224102206","stratum":"16k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":6950.0323929999995,"prompt_ms":2329.304,"prompt_tps":4094.78539512232,"decode_tps":63.24357192334971,"input_tokens":9587,"output_tokens":276,"peak_rss_mb":9509.52734375,"verdict":{"state":"Pass"},"prompt_ids_sha256":"3163837c3c6ce35fc9144f2561444c6a631999d70734909762d7308610035870"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"6aa76be5cbd8ad7332c2acf93046bc20e866b9847dd276b88d4b66abe11949c8","stratum":"16k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":5929.2524379999995,"prompt_ms":2236.82,"prompt_tps":4075.428510117041,"decode_tps":63.240052339766955,"input_tokens":9165,"output_tokens":221,"peak_rss_mb":9509.52734375,"verdict":{"state":"Pass"},"prompt_ids_sha256":"5ef3f3955e3349beecc7b3ead0a588b47ee06838c7a297707417f2fd7c975fc4"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"720dc2dbf89b8f4f37b09af101e95d2b2c11697ff7a5d59d50f131286980e9ab","stratum":"16k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":5352.358222000001,"prompt_ms":3194.312,"prompt_tps":4043.750266097989,"decode_tps":60.94720216186957,"input_tokens":12966,"output_tokens":119,"peak_rss_mb":9509.52734375,"verdict":{"state":"Pass"},"prompt_ids_sha256":"34405e806691b42d660adc2a2cfea20841a3de09c5c209d1aa96f85455b602c6"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"ebb26bc757178bfd19484adb04721a053df335e8059aed67cf8e11eb4d53cca1","stratum":"16k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":7423.162137,"prompt_ms":3388.982,"prompt_tps":4023.922227972884,"decode_tps":60.46058235208637,"input_tokens":13686,"output_tokens":229,"peak_rss_mb":9509.52734375,"verdict":{"state":"Fail"},"prompt_ids_sha256":"2a4712db6c149f45728ae2b1ed0ed2762692034e6140466b328757593cef7e3d"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"1ee0546fe5e55ded3962edb4c5d8eb0632d49a0d3fcbcb6de9ec2d9936a567f2","stratum":"16k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":5711.171754000001,"prompt_ms":2249.167,"prompt_tps":4047.7207784037378,"decode_tps":62.8912531224401,"input_tokens":9153,"output_tokens":200,"peak_rss_mb":9509.52734375,"verdict":{"state":"Pass"},"prompt_ids_sha256":"4125241c2b414624ad34f56859fa91c135d8d1ffcc58e335cf7051aadd0439c5"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"1c2d4d2a2013dc86953f03eddb0ce4b67ac53e05931b7b18ff6d527d2c880640","stratum":"16k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":7822.995158999999,"prompt_ms":2722.193,"prompt_tps":4083.105055372635,"decode_tps":61.64633064227382,"input_tokens":11164,"output_tokens":298,"peak_rss_mb":9509.52734375,"verdict":{"state":"Pass"},"prompt_ids_sha256":"e4d7241584af87981056a070fd2ec4e3eee56ef2016c62d5f7f71ab382b92dc9"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"f51557a52249a78f34a126d0646c5108cec49aa8bf9829fbfa487f75f82c7984","stratum":"16k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":5858.828914,"prompt_ms":2052.699,"prompt_tps":4087.788808782974,"decode_tps":63.14959887374796,"input_tokens":8440,"output_tokens":226,"peak_rss_mb":9509.52734375,"verdict":{"state":"Pass"},"prompt_ids_sha256":"44aebbb6cf4173813f2621076bab1a2a912ba2f0a714ee96b98f86cfe61ab844"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"69a9d1096c603ec6d3706769831302fbf6a198a02c189f771a730704506c2941","stratum":"16k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":6686.414640999999,"prompt_ms":3544.723,"prompt_tps":4044.039548365274,"decode_tps":59.14130266971404,"input_tokens":14384,"output_tokens":173,"peak_rss_mb":9509.52734375,"verdict":{"state":"Pass"},"prompt_ids_sha256":"dbd2749149ab437d0166e1de2f12680147b380a10ddb9b038ba1e6759f59d1a0"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"36528c9baa7523120f3683b655e9bd996e7ec070c288fa94b92aa17f033e18c4","stratum":"16k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":5977.997469,"prompt_ms":2094.982,"prompt_tps":4065.9060555174224,"decode_tps":62.37669763217497,"input_tokens":8567,"output_tokens":224,"peak_rss_mb":9509.52734375,"verdict":{"state":"Pass"},"prompt_ids_sha256":"036ef65a29008fd891473edcdf5366c0e021f70e465cd8fd70aee4f55c392c39"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"7d6b1ea94587f39adc61be730808d638df69dc46fb23c0292d839974de14c4aa","stratum":"16k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":5352.765576999999,"prompt_ms":2189.838,"prompt_tps":4065.597546485173,"decode_tps":62.74064789266439,"input_tokens":8952,"output_tokens":185,"peak_rss_mb":9509.52734375,"verdict":{"state":"Pass"},"prompt_ids_sha256":"5640f5c62747622fad6026aa1b84af1b301adde703da627336351517c166ff00"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"25c07e0a68726a522357ec60ea143c91cea4227fe4139c3db6f0c52ac274bb8d","stratum":"16k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":7309.777977,"prompt_ms":3608.965,"prompt_tps":4002.5325820560743,"decode_tps":59.69336157660021,"input_tokens":14494,"output_tokens":207,"peak_rss_mb":9509.52734375,"verdict":{"state":"Pass"},"prompt_ids_sha256":"3e4d4daca07a12047b37ab6fa0e74f224e28c54cd87dad01f94457face129e0c"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"cc6f12cb2a5dc9e2cd0cff0eda9338245b06a7b29be77caf7c7ad415dd948bb4","stratum":"16k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":6681.761633,"prompt_ms":2798.852,"prompt_tps":4029.151952300443,"decode_tps":61.29945234447807,"input_tokens":11326,"output_tokens":218,"peak_rss_mb":9509.52734375,"verdict":{"state":"Pass"},"prompt_ids_sha256":"7ef7e31a465dc80e0aa2ce7fa43085eba91d4ec00561981cadf511afc94587fc"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"88ef36ab5267cf425c39c87c20a1e697a4b612b9d36b17c39ca3b51cd24f9a9e","stratum":"16k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":7231.294757,"prompt_ms":2923.85,"prompt_tps":4030.986541717257,"decode_tps":61.08560244500688,"input_tokens":11835,"output_tokens":248,"peak_rss_mb":9509.52734375,"verdict":{"state":"Pass"},"prompt_ids_sha256":"537725b34a695cc189b0d1338ef38a956b5d0250fa0c9c4bd1b5740c766399ce"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"43d42945260ed2fee02c5b9fda7974cf8f66fd65dff63de4e17ea8faf547e878","stratum":"16k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":9020.418436,"prompt_ms":2392.825,"prompt_tps":4045.009559829908,"decode_tps":62.134957126879584,"input_tokens":9728,"output_tokens":397,"peak_rss_mb":9509.52734375,"verdict":{"state":"Pass"},"prompt_ids_sha256":"d9b5939d45f4b6cadbb25f9ce529b2543a1f3da2d3ab3decf3c7abe8c21fed68"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"059c0fd951a748e2ae74924c143d7c8e8d3c0c277645ba032d7f3fe28458b002","stratum":"16k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":9630.187094,"prompt_ms":3838.573,"prompt_tps":3974.654122769061,"decode_tps":59.32715724280859,"input_tokens":15306,"output_tokens":327,"peak_rss_mb":9509.52734375,"verdict":{"state":"Fail"},"prompt_ids_sha256":"698b52cf76abfa36ca28d8f350ef8107c942ea080b91406c1ac66d4b38c385b0"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"0882349afbe288f0c5a3e2112311c022c01875743aabcc1d12afe9ce23093012","stratum":"16k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":7263.123822,"prompt_ms":3418.903,"prompt_tps":4012.3981288735017,"decode_tps":60.38556181217071,"input_tokens":13767,"output_tokens":213,"peak_rss_mb":9509.52734375,"verdict":{"state":"Pass"},"prompt_ids_sha256":"ff993db7ae7f149d4e3a1d6d7840460f0cd8a26c0eea77c35f6196552a40d4e5"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"bcef4d3206e948203a70d22840d2a5bbbd067648e038d4939ac7913c2d0d977d","stratum":"16k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":7590.342986,"prompt_ms":2977.904,"prompt_tps":4038.746715810853,"decode_tps":61.03158697906176,"input_tokens":12076,"output_tokens":264,"peak_rss_mb":9509.52734375,"verdict":{"state":"Pass"},"prompt_ids_sha256":"39836f7486ad00c5fda9e448cf60e59f2860a0307cbf1871f34c46333a1c992c"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"f44f0554c5d06d6d9bd8a87e7c9adef7b5acc16f8dd84ee8990ad95cdb0d7a86","stratum":"16k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":7472.484743,"prompt_ms":2768.599,"prompt_tps":4021.5285781725697,"decode_tps":61.49259251639671,"input_tokens":11183,"output_tokens":273,"peak_rss_mb":9509.52734375,"verdict":{"state":"Pass"},"prompt_ids_sha256":"1f020914b5d4e264ef245a02a23947d4b1fd4c90b001bd77a260391e94464e4f"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"d49cd0c3fad3e8f5c978cbcb184e89f2cd360cdf37c1a97d192e6ca1e88aa974","stratum":"16k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":7172.242216,"prompt_ms":3458.126,"prompt_tps":3966.0209026507414,"decode_tps":59.95209096808589,"input_tokens":13764,"output_tokens":206,"peak_rss_mb":9536.6328125,"verdict":{"state":"Pass"},"prompt_ids_sha256":"2ad00e24d8988d50c0203eb5a7be9e633ca84a757788f60888e7586e09d1d1c8"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"6472e6c3bf01349b337af3cf2ddd2fc277b3af86eeb21bf5ca250923f3760dbb","stratum":"16k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":5645.401839,"prompt_ms":2157.266,"prompt_tps":4062.085992177135,"decode_tps":62.86145335680161,"input_tokens":8812,"output_tokens":196,"peak_rss_mb":9536.6328125,"verdict":{"state":"Pass"},"prompt_ids_sha256":"27a05690c12bac021035e63d41f559f99635ca5e4f2488ee991d136a21e34989"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"5a466ceed2939ee8936626418aadce90a670d794ce8d1289af467fe01430718d","stratum":"16k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":8181.762454999999,"prompt_ms":3649.782,"prompt_tps":3970.100132007884,"decode_tps":59.17983300994947,"input_tokens":14539,"output_tokens":254,"peak_rss_mb":9536.6328125,"verdict":{"state":"Pass"},"prompt_ids_sha256":"3f65be738f0bac2a6d1a61af3c99be128d0245394c8f562f5629e4378a22d17b"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"c8cd063621db62fd4dde1e4e2fe07d67562b8d073e89e6bd35c00b88b2c6d546","stratum":"16k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":6779.957711,"prompt_ms":3305.807,"prompt_tps":3975.428692600627,"decode_tps":59.86777684301853,"input_tokens":13191,"output_tokens":192,"peak_rss_mb":9536.6328125,"verdict":{"state":"Pass"},"prompt_ids_sha256":"7d874c65f58ba21b178b10b378237380b9c35c43a22721cead3d56854f952312"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"1c7345312237ba7892931e444d0b663807feae59083cbc849d13f6e4748d2bd2","stratum":"16k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":6318.367765999999,"prompt_ms":3697.899,"prompt_tps":3940.886433079974,"decode_tps":58.61742215562265,"input_tokens":14622,"output_tokens":134,"peak_rss_mb":9536.6328125,"verdict":{"state":"Pass"},"prompt_ids_sha256":"6dc5aeb4272351770a57d1673d2f6fceb604770bb1e63363de5dde21a07ecbae"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"0dc43a5ae0a19646ed73c5f0aa0495eda3a3b460efd42e35ab5a0d1fe2002192","stratum":"16k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":7361.108053999999,"prompt_ms":2191.189,"prompt_tps":3967.2524825562746,"decode_tps":59.340753278991585,"input_tokens":8742,"output_tokens":287,"peak_rss_mb":9536.6328125,"verdict":{"state":"Fail"},"prompt_ids_sha256":"5a5c9a9d4cc57194ef9db7c284d1261f917f9c29a19a53489aa80dac05004407"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"5189d6142784fae84f7d1634633cf93f49b3d2528950f68906f1c5ca9e8dfc5a","stratum":"16k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":10110.536955000001,"prompt_ms":4127.346,"prompt_tps":3912.683840899213,"decode_tps":57.68929498388933,"input_tokens":16198,"output_tokens":328,"peak_rss_mb":9536.6328125,"verdict":{"state":"Pass"},"prompt_ids_sha256":"490004b669067449f5c7e62e6e309d0ee30ed55781d9c2b9d140e4ac5dbb38c8"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"28c93295ce9295f699d1ab17eecef2d2ef643f6ec8434694ee70fa20ad70c0b9","stratum":"16k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":6210.290884,"prompt_ms":3663.698,"prompt_tps":3693.808823762221,"decode_tps":57.123027291954365,"input_tokens":13582,"output_tokens":121,"peak_rss_mb":9536.6328125,"verdict":{"state":"Pass"},"prompt_ids_sha256":"8c0ae16658161f3d1698b96183289aed7fd75bcda9877263efda1062664d7497"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"0f1bc13a198826247cef2fe67851805b3a7046ac9d291b02b47581866c6d896c","stratum":"16k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":6299.7847839999995,"prompt_ms":2528.471,"prompt_tps":3936.371032137604,"decode_tps":60.762921457551315,"input_tokens":10002,"output_tokens":206,"peak_rss_mb":9536.6328125,"verdict":{"state":"Pass"},"prompt_ids_sha256":"6a1e197e1c7efd9196871bec225a755aca2bd8ddfe47e0d46af2f3e0b14f1419"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"4e7c51716440d0055d37c6d118eca15cfb84381cb2984b437ddbf7920cee4651","stratum":"16k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":4670.388212,"prompt_ms":2585.053,"prompt_tps":3940.3447434153186,"decode_tps":60.46986182086848,"input_tokens":10235,"output_tokens":111,"peak_rss_mb":9536.6328125,"verdict":{"state":"Pass"},"prompt_ids_sha256":"f45b53c01e8955c24c49d0f24260bd34e4258513c7701ab5e462911006f37e99"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"d2107848512e9694b1e838662ca844b67f962cdc8d3826265b75f6b002d2982d","stratum":"16k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":6557.230818,"prompt_ms":2133.048,"prompt_tps":4009.755054738572,"decode_tps":61.78356002710692,"input_tokens":8602,"output_tokens":255,"peak_rss_mb":9536.6328125,"verdict":{"state":"Pass"},"prompt_ids_sha256":"5b324a8a6e5df50a926268dd114f6374ee8d731ed147592868a109e4fa8cbc33"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"f0466d3c53c99fb39ebfe065673d3b9bdd70fdf30ee02898eb6e886201e8d4a3","stratum":"16k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":8087.8195430000005,"prompt_ms":3542.262,"prompt_tps":3986.153480459661,"decode_tps":59.44449469382122,"input_tokens":14169,"output_tokens":256,"peak_rss_mb":9536.6328125,"verdict":{"state":"Pass"},"prompt_ids_sha256":"8600d8498d868371fc947edc7db322d590a85502079ad419abb0df4a4727df72"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"f570aee8b67fbb453024af27fe758441e49aa16bc6b09147772a0d43a92115ec","stratum":"16k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":5548.793465,"prompt_ms":3015.397,"prompt_tps":4063.478208673684,"decode_tps":60.76514939960904,"input_tokens":12302,"output_tokens":137,"peak_rss_mb":9536.6328125,"verdict":{"state":"Pass"},"prompt_ids_sha256":"8b2497a464c07d512e5a9659dcb4f07a47d97fc06c111ae926ed313355ce34d3"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"409826d3610e79d93d92ba45ad55d943307796f6b4218ad327e872390681ef26","stratum":"16k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":9429.172763,"prompt_ms":4085.906,"prompt_tps":3969.254309815253,"decode_tps":58.83798372800656,"input_tokens":16267,"output_tokens":296,"peak_rss_mb":9536.6328125,"verdict":{"state":"Pass"},"prompt_ids_sha256":"cc546fdfce8b6b26c4668049fff094491f2f8bee3cf437f37a5eb735bdb27740"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"b53f7c2d4a456e341e0f1e013eede2c53dbe5230bb2ef59c8437a9dbd9b1126f","stratum":"16k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":7657.22757,"prompt_ms":3294.516,"prompt_tps":4038.833018264292,"decode_tps":60.543712189525046,"input_tokens":13355,"output_tokens":246,"peak_rss_mb":9536.6328125,"verdict":{"state":"Pass"},"prompt_ids_sha256":"58c397b59aee26d82ba43edafd279603a863fc48170cd0fb176fdf91ffc3a98f"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"fd326fe7d4bacc97fcf64369c18bd23e1a718b755ec9d720b6010b7bea96f970","stratum":"16k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":4954.95298,"prompt_ms":2047.462,"prompt_tps":4080.661814480562,"decode_tps":63.36162205752468,"input_tokens":8404,"output_tokens":166,"peak_rss_mb":9536.6328125,"verdict":{"state":"Pass"},"prompt_ids_sha256":"450e403e1bc170e1470b9b1d2a97c64aa904b0167319be75d1fa671f5a99f4dd"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"fa7720559e61059c92e06f6156fc0da7ae0a6367cd814f36465e85f7d04800e7","stratum":"16k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":9833.428068,"prompt_ms":3945.542,"prompt_tps":4000.464321505132,"decode_tps":59.503258248772795,"input_tokens":15833,"output_tokens":335,"peak_rss_mb":9536.6328125,"verdict":{"state":"Pass"},"prompt_ids_sha256":"7d9efdebce284cc4c5ce1d3a24fd62794d1ac2228510aa0c228e9ee283ab2cd9"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"f8f4e9f89c28ef4f440267322b74bbf68eed96ea240d6a883bec0d50f2dbe576","stratum":"16k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":6118.4025,"prompt_ms":2330.032,"prompt_tps":4077.19722304243,"decode_tps":62.87361854690992,"input_tokens":9549,"output_tokens":219,"peak_rss_mb":9536.6328125,"verdict":{"state":"Pass"},"prompt_ids_sha256":"5d5cfe1a4a40e808aca7241c0e6a96d796456776e3fc689c00288b819bcc28cd"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"ca612af4633a006928c0400dcf7a90a769c47290edc85241d3ba156583b2f6b7","stratum":"16k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":4995.310407,"prompt_ms":2259.347,"prompt_tps":4075.071248462498,"decode_tps":63.18225842183514,"input_tokens":9256,"output_tokens":159,"peak_rss_mb":9536.6328125,"verdict":{"state":"Pass"},"prompt_ids_sha256":"3a050649a54bd19a4c5b2fc11ca233aacbd3f9ea410506f99d0d65a893a023cd"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"f80dddb74b021666f2f30078ee85f6a231fb529e1ed67f051a9b662ffb17b4e6","stratum":"16k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":6586.503757,"prompt_ms":2245.905,"prompt_tps":4098.570509438288,"decode_tps":63.32071870964065,"input_tokens":9254,"output_tokens":261,"peak_rss_mb":9536.6328125,"verdict":{"state":"Pass"},"prompt_ids_sha256":"29ad40656fdbdd17ccccd5a79690a1c5856bd2c5e066028c0776ebd8fcc59873"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"fa49d36e7eb9eed429d0d549464654396d8fe8fc5a0be7b4e667bddb37922c36","stratum":"32k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":8462.797102,"prompt_ms":6337.814,"prompt_tps":3886.829118052376,"decode_tps":54.64380383925264,"input_tokens":24683,"output_tokens":105,"peak_rss_mb":9536.6328125,"verdict":{"state":"Pass"},"prompt_ids_sha256":"10163b836e6b21a8d1943f429085e148470b19b5c857f7822b5661509f4d11d9"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"3b51ad3f76c868adaa04840a77eed635ade302cec8c85bf0d3795526ab237fa7","stratum":"32k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":8632.702551,"prompt_ms":5081.421,"prompt_tps":3967.394160019411,"decode_tps":57.879543461946504,"input_tokens":20209,"output_tokens":177,"peak_rss_mb":9536.6328125,"verdict":{"state":"Pass"},"prompt_ids_sha256":"2d0742ae98d285c41c4c84ef725a4fdd40c2b06112ad0f5f6709cd4fd9da985b"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"f8815530d0c7dc219ea3341c95432e757c3ab7fb0308bd2146f2d90b840f726b","stratum":"32k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":7057.774797,"prompt_ms":4190.464,"prompt_tps":4024.613980695217,"decode_tps":59.45628417249932,"input_tokens":16914,"output_tokens":151,"peak_rss_mb":9536.6328125,"verdict":{"state":"Fail"},"prompt_ids_sha256":"dd1693371bba23714bd583154ef11f1d6a58342d9feb0fbc8b3f89281a3c430a"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"717524cb0e8f36fac7a56af480fe9ecfe30ed9e797cefb9908a1c663d247b0c7","stratum":"32k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":8103.925399,"prompt_ms":4787.486,"prompt_tps":3982.257076052024,"decode_tps":58.31153796681387,"input_tokens":19114,"output_tokens":171,"peak_rss_mb":9536.6328125,"verdict":{"state":"Pass"},"prompt_ids_sha256":"fe2fbf0e6c29ac51fe639ae75e221ab52400551682caf9dcc5f560088f5991f0"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"03c15ba04bbacf6b92b3ab9bd18cdca26071fbb4a09dc4de9bef2ad078ba580f","stratum":"32k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":9382.717514,"prompt_ms":5603.929,"prompt_tps":3934.382466301768,"decode_tps":56.93209937409906,"input_tokens":22097,"output_tokens":195,"peak_rss_mb":9536.6328125,"verdict":{"state":"Pass"},"prompt_ids_sha256":"1d1e1149ebcc7a12c4464cd97630fc968780b0fa6703886e8550b6cd7ab8f27c"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"351ae6cc9897a62bc51dc629ead6161e8242c49a7cfbeabdd8f05e9c0c3cdaea","stratum":"32k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":10102.434475,"prompt_ms":4908.523,"prompt_tps":3974.922802643483,"decode_tps":58.198705618460714,"input_tokens":19560,"output_tokens":276,"peak_rss_mb":9536.6328125,"verdict":{"state":"Fail"},"prompt_ids_sha256":"d4c0dfa77150a8a80c4404832981c8f2c1f4eca29ffbec592e3d21b0ba30e535"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"7bb9e2edd88d122d0553e7776b5ac605cc7da369d4410dba027a27df9290f060","stratum":"32k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":8819.206372,"prompt_ms":5396.785,"prompt_tps":3934.75004099663,"decode_tps":56.55646534659687,"input_tokens":21284,"output_tokens":172,"peak_rss_mb":9536.6328125,"verdict":{"state":"Pass"},"prompt_ids_sha256":"162822cb75db0d188fb71a133b6480a9768b7b1582914953aaf3b0494125830a"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"d667fffd336a3dd90c6245ba9527ce0fb0448c554d576a089e347230a6ecb2c0","stratum":"32k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":8623.56455,"prompt_ms":5093.905,"prompt_tps":3756.8427365645807,"decode_tps":53.54220383581509,"input_tokens":19186,"output_tokens":167,"peak_rss_mb":9536.6328125,"verdict":{"state":"Pass"},"prompt_ids_sha256":"544ecff0b9504efcc6965bfe96d614b95a0c9a33f99d1f021276789c0ac5f294"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"38a1dbfd07591ad9a6e4b27e7916e661614e6cd7231082e0e4e2601297e5fe7f","stratum":"32k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":9536.813107,"prompt_ms":5412.267,"prompt_tps":3599.2311539693073,"decode_tps":53.78604750708673,"input_tokens":19529,"output_tokens":193,"peak_rss_mb":9536.6328125,"verdict":{"state":"Pass"},"prompt_ids_sha256":"3fa656a12db52afb3add603a41a8269708e92167a90663f8d3cc7548b477f08c"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"3d8a6474dcfe17a53ec35178358b968c69405032d23e581b7a700fe39982960a","stratum":"32k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":8864.361345000001,"prompt_ms":5233.148,"prompt_tps":3628.599840860606,"decode_tps":55.55021877885888,"input_tokens":19038,"output_tokens":182,"peak_rss_mb":9536.6328125,"verdict":{"state":"Pass"},"prompt_ids_sha256":"64b5124f7d02001c4cf5434c7f64ad4222b626650acc205dded43af97165d1eb"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"bf2811a3edb86d54d69166c93ec73ebcf6ff64b5556d9d7d9f090726e8fee9a2","stratum":"32k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":9859.782766,"prompt_ms":5748.962,"prompt_tps":3773.898662054123,"decode_tps":54.20432566944396,"input_tokens":21745,"output_tokens":199,"peak_rss_mb":9536.6328125,"verdict":{"state":"Pass"},"prompt_ids_sha256":"ddea598580c58fd52253867387479e86617b6fc8bef87e6b9a6bf70c78966051"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"b515045eff1517e48974319c1c62e87bc8ef658b739d6e365b934ee04f6667f0","stratum":"32k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":12741.144897,"prompt_ms":8018.469,"prompt_tps":3720.6603904061985,"decode_tps":52.26574363629097,"input_tokens":29883,"output_tokens":222,"peak_rss_mb":9536.6328125,"verdict":{"state":"Pass"},"prompt_ids_sha256":"cce8cd881a53898e5c7a62a4f4ca518a141c8988a66f1f84e53c4982ca7f1560"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"8f82c371a972bb3e1d51e43968dccb02cfdaa9e94eaabb04b2b6202addef14fa","stratum":"32k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":9923.062086,"prompt_ms":6722.965,"prompt_tps":3576.1007234159333,"decode_tps":53.3535143316621,"input_tokens":24091,"output_tokens":142,"peak_rss_mb":9536.6328125,"verdict":{"state":"Pass"},"prompt_ids_sha256":"4db639b147d20927495dfc8f015d002c34a1a8dbb361f9a395d0f540f44c1f25"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"304c3c29ecc274958e1042ea4f3e6fb4108bdc048767ad229c669ae8025e23f7","stratum":"32k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":8990.920398,"prompt_ms":4952.483,"prompt_tps":3868.564516021559,"decode_tps":56.87124010013888,"input_tokens":19208,"output_tokens":201,"peak_rss_mb":9536.6328125,"verdict":{"state":"Pass"},"prompt_ids_sha256":"7d297d57bb08bd16d5a0bfd093357da4f17a4f2954bc5aebe4e292401877ef3a"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"c9d4c24c8671432ccf62fc9a693dd62fc3430da536f4b20f53abb64ba65e010b","stratum":"32k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":8388.261367000001,"prompt_ms":4794.063,"prompt_tps":3848.92730863153,"decode_tps":57.54704313543964,"input_tokens":18501,"output_tokens":184,"peak_rss_mb":9536.6328125,"verdict":{"state":"Pass"},"prompt_ids_sha256":"2b348c36ba02c98acdd3b0264d0bf7c6abc97c4aa1e0b5d5a57e9a76895b8ff7"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"7173ebdf8174e7e3ac987fc47c136b866b83cdf52d03a1c1fce2fb4ed8192ded","stratum":"32k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":14014.017641,"prompt_ms":8372.519,"prompt_tps":3749.767543077537,"decode_tps":52.09796401919954,"input_tokens":31444,"output_tokens":274,"peak_rss_mb":9536.6328125,"verdict":{"state":"Pass"},"prompt_ids_sha256":"0f80c00a890f530a76c0f115c409d1e0b9ef5735899f43cac2e7a695fba889c4"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"0f963ad6b87e0d3342e5334a4115c16eaf148f32c1e0adc968706c77b5191039","stratum":"32k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":12329.236112,"prompt_ms":7340.546,"prompt_tps":3837.725422604803,"decode_tps":52.26231915947835,"input_tokens":28220,"output_tokens":234,"peak_rss_mb":9536.6328125,"verdict":{"state":"Pass"},"prompt_ids_sha256":"4c4b6ae7e6b58fe247abcafa33c758ba870176b098d102db06af0ac0686fd4b0"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"74f801a07d6f9f7366f719b4b2302c99b8c74f8eb9f099cb0088470f7ed180e1","stratum":"32k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":8631.203383999999,"prompt_ms":5876.544,"prompt_tps":3865.3671273455966,"decode_tps":55.013041196338435,"input_tokens":22764,"output_tokens":125,"peak_rss_mb":9536.6328125,"verdict":{"state":"Pass"},"prompt_ids_sha256":"70c486262966bf7809098d46926cd98aaf3f6670ceb98cb3f34cb817ffbf760e"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"430c7c0f69397873ade236dd3f5db7663f2f1fb4f40eb879109629b59e0b3ce3","stratum":"32k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":11350.331714,"prompt_ms":5611.141,"prompt_tps":3885.840687304062,"decode_tps":55.69430166659104,"input_tokens":21853,"output_tokens":298,"peak_rss_mb":9556.0390625,"verdict":{"state":"Pass"},"prompt_ids_sha256":"d344c43778457ceeaf9fcafd3565a371a4836ae8c8fd3ac08de51d30e2f26f57"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"8003dd98e1763cde69c838d89b0bab41aaa2622c580b9e3de39b8b3c1e182f85","stratum":"32k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":9035.236369,"prompt_ms":6394.847,"prompt_tps":3895.323844339044,"decode_tps":54.98608251424093,"input_tokens":24959,"output_tokens":120,"peak_rss_mb":9562.515625,"verdict":{"state":"Pass"},"prompt_ids_sha256":"83323d291c7ce06a7d2db0bdee88fcedd25ebe222a3c6653fc9c2257e2379150"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"b931d437202ef5adb659042b0e7255fd1754602c9d52039ef36d831ad232674b","stratum":"32k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":8722.074059,"prompt_ms":4989.596,"prompt_tps":3939.998348563692,"decode_tps":57.216005265109594,"input_tokens":19708,"output_tokens":186,"peak_rss_mb":9559.81640625,"verdict":{"state":"Pass"},"prompt_ids_sha256":"0adbf65326e98c30c97a86ac1cd456158dddac02a101d1cc351a815782a5cdba"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"084e31008f5a675e5153246d96a5142e0adc8215e66f598266bbb76a10be6db2","stratum":"32k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":12697.474459000001,"prompt_ms":7904.877,"prompt_tps":3812.077025360419,"decode_tps":52.76586021314195,"input_tokens":30183,"output_tokens":231,"peak_rss_mb":9559.81640625,"verdict":{"state":"Pass"},"prompt_ids_sha256":"008c43328b72985f0930804bf4dba749371420359c750afa424dd5e6d3047e5a"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"14b3251ddf800d7bdfdb7a775c2281eba042c909e8a46700689825014ded27de","stratum":"32k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":10978.966793,"prompt_ms":4888.826,"prompt_tps":3665.092600963912,"decode_tps":54.0298740489112,"input_tokens":17967,"output_tokens":291,"peak_rss_mb":9559.81640625,"verdict":{"state":"Pass"},"prompt_ids_sha256":"cc964f6596ba0bf32e87225ab3a6e88f89eac8e4ba6bc237cab69ecee8f5ee60"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"ca55757f48d1d3f049d77fd86f25cc77c581af66809d307a89d2a5b7bb14d278","stratum":"32k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":10549.464485,"prompt_ms":5744.997,"prompt_tps":3629.766908494469,"decode_tps":52.55528261897821,"input_tokens":20902,"output_tokens":224,"peak_rss_mb":9559.81640625,"verdict":{"state":"Pass"},"prompt_ids_sha256":"36511a899244f7383295611d61080169d7b333ce858c223384ee57588db0cc22"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"5bc59b96abf0b69ee35dd697a3d38ea685e54eb0cd1544a880e08b0f64c3905f","stratum":"32k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":9054.923213,"prompt_ms":5089.98,"prompt_tps":3658.953473294591,"decode_tps":54.09218660902844,"input_tokens":18673,"output_tokens":193,"peak_rss_mb":9559.81640625,"verdict":{"state":"Pass"},"prompt_ids_sha256":"01878ef6e12e6090d8bf4d912ac05afd77064731e6d2c22d029d8e95916d2ef8"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"4608b8ddaa9de754634659da66dee0e3d59c7097dfbebe7ef8ff664f7a8e2a3c","stratum":"32k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":12693.331304000001,"prompt_ms":5351.03,"prompt_tps":3752.9223345785767,"decode_tps":55.15597310885046,"input_tokens":20131,"output_tokens":381,"peak_rss_mb":9559.81640625,"verdict":{"state":"Pass"},"prompt_ids_sha256":"ea75c65f1593e8994329bf6520bd50cfd827175845144b2decd4a39d6a816003"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"dc10ee3c8ec1879df8935451288b060dcba94ff3db9fb2d0e714e938b65c15ca","stratum":"32k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":10292.041449999999,"prompt_ms":7054.647,"prompt_tps":3729.314875712421,"decode_tps":54.31694808277468,"input_tokens":26358,"output_tokens":151,"peak_rss_mb":9559.81640625,"verdict":{"state":"Pass"},"prompt_ids_sha256":"60fa312565da30886df98a128049dec4b80d7fbab7adb34688ce8d3b13131f87"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"841dd09f216b308d2acb091c53e79c3c22c6dc9c70d99ddd5aaa007e65fb8f0f","stratum":"32k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":7704.183252,"prompt_ms":4717.282,"prompt_tps":3872.992965016719,"decode_tps":57.34910911743305,"input_tokens":18319,"output_tokens":145,"peak_rss_mb":9559.81640625,"verdict":{"state":"Pass"},"prompt_ids_sha256":"693837d52c6a4861282f0d455277e48fac6d7e207bdaa4e7a172c2acf8726055"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"24bc258f12dfbb5600afabdff2efa2edeaf9efb4c7e13dae0bb627e6c0add649","stratum":"32k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":10720.146551,"prompt_ms":5791.915,"prompt_tps":3933.068769137669,"decode_tps":55.67642008981289,"input_tokens":22829,"output_tokens":254,"peak_rss_mb":9559.81640625,"verdict":{"state":"Pass"},"prompt_ids_sha256":"0662c740da8b7967cc8500d02856b1bb61b045eb9df620e43bddc27d9f806dfc"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"78a121fb8ef6ffe62c572f12c3113443d88b19491cd0db0981294e8ed29d38bb","stratum":"32k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":9243.380303,"prompt_ms":4699.49,"prompt_tps":3955.961178766207,"decode_tps":58.331684294157085,"input_tokens":18640,"output_tokens":238,"peak_rss_mb":9559.81640625,"verdict":{"state":"Pass"},"prompt_ids_sha256":"ba43eb4396f1eb5beb8d13c853dad39cd3e1cf8d77a0732070175d726fd6b01c"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"c0b75e2f0a6bcd1a3341d0cbb4e96aa16449934b09e961f790c377d972340d08","stratum":"32k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":9651.091392,"prompt_ms":5439.918,"prompt_tps":3915.132544277322,"decode_tps":56.41511352945664,"input_tokens":21347,"output_tokens":214,"peak_rss_mb":9559.81640625,"verdict":{"state":"Pass"},"prompt_ids_sha256":"957b6f511e3f28dc0e7f71f4f420374158278789df3325c0cb389818c1e26247"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"21a87ab49fd3587febc4c89bb82872bb6967bf0a9492abc590112d5f097f1bd5","stratum":"32k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":10077.798002,"prompt_ms":6193.017,"prompt_tps":3783.4548169333298,"decode_tps":55.54167788896967,"input_tokens":23480,"output_tokens":200,"peak_rss_mb":9559.81640625,"verdict":{"state":"Pass"},"prompt_ids_sha256":"7b8e789020dabaf591806d9d9bf0c6232547b067051bbd4cb26f58de384c2ce8"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"0534621eb0e2215a3c13b22ebf197a2a472f4306bf24f0e380154a527dbc1463","stratum":"32k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":9439.196917000001,"prompt_ms":5854.947,"prompt_tps":3896.3631950895538,"decode_tps":55.927704121139406,"input_tokens":22862,"output_tokens":169,"peak_rss_mb":9559.81640625,"verdict":{"state":"Pass"},"prompt_ids_sha256":"e49800ebb35f74199fe8a55797fc6e015fb00fd0fff65e78dd6f91889e9f5ff3"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"add564d1303cf5966124647b302705301dcd4528a1caad315f4d86aa3485e61a","stratum":"32k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":10862.522459999998,"prompt_ms":6025.394,"prompt_tps":3881.405929637132,"decode_tps":54.257544285459794,"input_tokens":23436,"output_tokens":241,"peak_rss_mb":9559.81640625,"verdict":{"state":"Pass"},"prompt_ids_sha256":"d501682a3b448ff13e405df0b3c77e50f0da19852d59188afedbf391ff330a0c"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"756c7e34ba781b2a93fb5ba02710dfea9f422786e59c39eb6e2b38e11ad4bf4a","stratum":"32k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":11095.443685999999,"prompt_ms":5136.01,"prompt_tps":3656.534936653161,"decode_tps":55.144755909510046,"input_tokens":18829,"output_tokens":299,"peak_rss_mb":9559.81640625,"verdict":{"state":"Pass"},"prompt_ids_sha256":"d6675b3bdd9d86978cd61242f476ccbc7623c2446ae06132244a6f42efbc7255"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"40d8b0a4203d03441d9dcc41d1b76704f65ec7ea5ebd165cac2457a566b70162","stratum":"32k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":8448.465336,"prompt_ms":4821.467,"prompt_tps":3688.503934590862,"decode_tps":55.36100117998019,"input_tokens":17833,"output_tokens":176,"peak_rss_mb":9559.81640625,"verdict":{"state":"Pass"},"prompt_ids_sha256":"0200f2f9a1370d6f38cae62051efcd98e40658a6fc1ec422e8b37f6701ce5412"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"3db9e737af60c9d2537ac8d8b93aeaffc7b6cc68294a49978e5c870ffb36eb4f","stratum":"32k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":9324.411767000001,"prompt_ms":5418.96,"prompt_tps":3630.955017198872,"decode_tps":55.24779874628727,"input_tokens":19725,"output_tokens":202,"peak_rss_mb":9565.828125,"verdict":{"state":"Pass"},"prompt_ids_sha256":"fffd8b1255cac61466278bf8094821380a784a6aefa5c63410027ecce55429ed"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"c354c595f4a8ea1b09659b55a5ce312a495e4eb6400c77183cd34cca3cb8327b","stratum":"32k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":8773.763124000001,"prompt_ms":5088.466,"prompt_tps":3668.492626264968,"decode_tps":57.27435693192477,"input_tokens":18716,"output_tokens":185,"peak_rss_mb":9563.29296875,"verdict":{"state":"Pass"},"prompt_ids_sha256":"05dadf955da5b93f28985b714df19e2ef0206facca769a7ffa72479563a4975a"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"c42dfb333a42fbec1745318326627fa05d41440da37c5d2d57304daf1e4bb3b6","stratum":"32k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":8503.733559,"prompt_ms":5473.799,"prompt_tps":3658.15405351932,"decode_tps":57.032245880742174,"input_tokens":20073,"output_tokens":152,"peak_rss_mb":9563.29296875,"verdict":{"state":"Pass"},"prompt_ids_sha256":"80097ce1ced3c4a30308e27f670ccbf58abb86ff4f9385521228aac8db4ed868"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"8136fdc9902571f25cbf93bf14de71435aea774d50ab0fb50ba2196f422389af","stratum":"32k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":9859.126221,"prompt_ms":5627.681,"prompt_tps":3657.8121609949103,"decode_tps":56.25925108107566,"input_tokens":20634,"output_tokens":214,"peak_rss_mb":9563.29296875,"verdict":{"state":"Pass"},"prompt_ids_sha256":"a3c875c6ba4ab5c35526ae9bc83a120f426b85c20fe3a644c37dd3cb99b85be3"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"a267f9989c8533bce8be2d300c85a8509c45a80b1fe16836591ae91341138b2a","stratum":"32k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":13083.558519,"prompt_ms":8354.022,"prompt_tps":3497.8361321049906,"decode_tps":52.83158772824172,"input_tokens":29270,"output_tokens":229,"peak_rss_mb":9563.29296875,"verdict":{"state":"Pass"},"prompt_ids_sha256":"52b26517747f4815b31f168051fd64d5054550a410ec132c26ad9f221325ae4b"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"60d2f019ddec1435b5b1230d70553a3833405b30dae55c8dbc8f6bcc1f0f6487","stratum":"32k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":8241.344348,"prompt_ms":5194.757,"prompt_tps":3658.6889434866735,"decode_tps":57.825171216717905,"input_tokens":19055,"output_tokens":145,"peak_rss_mb":9563.29296875,"verdict":{"state":"Pass"},"prompt_ids_sha256":"926956c06005c8e58a90a81567de45b1b9f4cf512f4714410a8c2bdff97eaf09"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"4be2763e70a83f3b538045b385bf920c45f0ab971c6641e4573f745b32673998","stratum":"32k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":10554.27379,"prompt_ms":4596.015,"prompt_tps":3709.0827597386,"decode_tps":58.25637607209386,"input_tokens":17096,"output_tokens":331,"peak_rss_mb":9563.29296875,"verdict":{"state":"Fail"},"prompt_ids_sha256":"2209238bca60b627f187342a3fa75e2cc850f42325091d4f4918c6c267a9a260"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"4a66cbc27f5a915eaa9fed54adca1621f3d27e77b95dee234328d0114d722209","stratum":"32k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":13797.546963,"prompt_ms":7767.401,"prompt_tps":3487.1381044959567,"decode_tps":53.49420801205073,"input_tokens":27135,"output_tokens":303,"peak_rss_mb":9563.29296875,"verdict":{"state":"Pass"},"prompt_ids_sha256":"7d7c836939bcc09cc2d8280e28b6a392d04a2145d05417cf9d72dd860d4e04fe"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"f552e516d8a086ee988e3c5488a31467a2bb6b35b277aecfb98ad15c96d6d3da","stratum":"32k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":9794.846112,"prompt_ms":5198.833,"prompt_tps":3665.053291767595,"decode_tps":57.298895631414155,"input_tokens":19103,"output_tokens":234,"peak_rss_mb":9563.29296875,"verdict":{"state":"Pass"},"prompt_ids_sha256":"c2c806c56c4d144111f554ef8f5f933837eea605a07ee70ebb09c16490c92ca1"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"5c2a849b4a443f183000bbd6996f84b95d3df274e267f4a0b9be650a53080e75","stratum":"32k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":9918.923654,"prompt_ms":6519.35,"prompt_tps":3626.588540268584,"decode_tps":55.15907169973016,"input_tokens":23692,"output_tokens":165,"peak_rss_mb":9563.29296875,"verdict":{"state":"Pass"},"prompt_ids_sha256":"000dbf6068631c787dc66205d5a9ad463a2c755eb612acc7ece318bff11f5bfc"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"a2b4707bfaf04f8599408ccfb45a44703a000f71a094a68ba1759061b2a43672","stratum":"32k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":9235.106328,"prompt_ms":5323.499,"prompt_tps":3661.6894264467783,"decode_tps":57.284216964412394,"input_tokens":19542,"output_tokens":196,"peak_rss_mb":9563.29296875,"verdict":{"state":"Pass"},"prompt_ids_sha256":"fa651c50ada370ebabdf777b8b3eac38d443ada3b2e4762702cf325ab1421721"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"3ab4ac140f83056d2501ee334f17e3ee310e217094d568b4fec0c75a1a30dce9","stratum":"32k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":11459.499041000001,"prompt_ms":6177.417,"prompt_tps":3630.481801698023,"decode_tps":55.79111805400581,"input_tokens":22476,"output_tokens":271,"peak_rss_mb":9563.29296875,"verdict":{"state":"Pass"},"prompt_ids_sha256":"a4b4fb3bd5fe96a03ed1ee01a571c200b6a1ab2e777a7638743f0a24ae89a56b"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"bf6a52843bdf826313ab90adf25c9d7a253496a9a3c5a46c08ef62ef5249470d","stratum":"32k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":10473.056042,"prompt_ms":5864.362,"prompt_tps":3615.909113386929,"decode_tps":56.124810162585966,"input_tokens":21254,"output_tokens":237,"peak_rss_mb":9563.29296875,"verdict":{"state":"Pass"},"prompt_ids_sha256":"29750ae08524616eb9b008167fc5e1577e4b6f505976a9986f03dac0a1eb099b"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"4c7ad86de1bf01701febc9297c4271016e389826ac06ac0fdc766c598c4ba789","stratum":"32k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-33b5e5b24","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":9585.223702000001,"prompt_ms":5318.479,"prompt_tps":3675.674943907835,"decode_tps":56.94404565789621,"input_tokens":19598,"output_tokens":214,"peak_rss_mb":9563.29296875,"verdict":{"state":"Pass"},"prompt_ids_sha256":"36e696fa27b6f800c61cc29b6ce76dc253720bc7ae8a736483bb1a628c4bf41c"} diff --git a/evidence/prm-001/s1-v2/33b5e5b24/speed-block.json b/evidence/prm-001/s1-v2/33b5e5b24/speed-block.json new file mode 100644 index 0000000000..d358d852f3 --- /dev/null +++ b/evidence/prm-001/s1-v2/33b5e5b24/speed-block.json @@ -0,0 +1,57 @@ +{ + "replay_version": "review-replay-v2", + "set_sha": "a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632", + "apr_tag": "untagged-33b5e5b24", + "cell": "gx10-cuda", + "gguf_sha256": "00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4", + "n_items": 200, + "p50_s": 26.978345700000002, + "p95_s": 68.833684447, + "p95_ci": [ + 60.908583259, + 85.635564659 + ], + "queue_budget_p95_s": 601.590268351, + "queue_budget_lane": "antigravity", + "within_budget": true, + "ttft_ms": { + "apr": 8470.244637, + "llama_cpp": 1923.354, + "ratio": 4.403892698380017 + }, + "prefill_tps": { + "2k": { + "apr": 942.2090502154527, + "llama_cpp": 3671.976446078548, + "ratio": 0.2565945245159935 + }, + "8k": { + "apr": 967.8218970121728, + "llama_cpp": 4065.914180370514, + "ratio": 0.23803303613358073 + }, + "16k": { + "apr": 900.2771390552361, + "llama_cpp": 4032.293697971591, + "ratio": 0.2232667574557162 + }, + "32k": { + "apr": 776.0930557050291, + "llama_cpp": 3749.767543077537, + "ratio": 0.2069709780110978 + } + }, + "decode_tps": { + "apr": 13.69316477765186, + "llama_cpp": 63.32071870964065, + "ratio": 0.21625093739763665 + }, + "peak_rss_mb": { + "apr": 8388.87890625, + "llama_cpp": 9565.828125, + "ratio": 0.8769631647808851 + }, + "parse_rate": 1.0, + "verdict_identity_vs_prev_tag": null, + "prompt_ids_sha": "f3c201362a6287c9c665509b8e8832e437131eddb206e0202010debe131dc0fa" +} diff --git a/evidence/prm-001/s1-v2/33b5e5b24/voters.txt b/evidence/prm-001/s1-v2/33b5e5b24/voters.txt new file mode 100644 index 0000000000..c51fe2d362 --- /dev/null +++ b/evidence/prm-001/s1-v2/33b5e5b24/voters.txt @@ -0,0 +1,23 @@ +antigravity 86.458702413 +antigravity 145.04 +antigravity 126.599 +antigravity 397.835016126 +antigravity 178.814814705 +antigravity 358.102140635 +antigravity 284.588141901 +antigravity 60.19135729 +antigravity 413.322599065 +antigravity 643.342708811 +antigravity 601.590268351 +antigravity 185.612191185 +antigravity 92.594668 +antigravity 119.534408545 +antigravity 136.293514917 +antigravity 204.236 +antigravity 46.304222969 +antigravity 196.058331158 +antigravity 50.69529767 +antigravity 510.870942614 +antigravity 190.496753786 +antigravity 215.11265965 +antigravity 390.196868974 diff --git a/evidence/prm-001/s1-v2/96eb79f51/rows-apr.jsonl b/evidence/prm-001/s1-v2/96eb79f51/rows-apr.jsonl new file mode 100644 index 0000000000..913d38765f --- /dev/null +++ b/evidence/prm-001/s1-v2/96eb79f51/rows-apr.jsonl @@ -0,0 +1,200 @@ +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"f1ca076d355345060d79de6ab291b725d78bfd3596f2cfc01444d40d11e2ea79","stratum":"2k","engine":"apr","engine_version":"apr 0.69.3 (96eb79f51)","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":7646.548533,"prompt_ms":null,"prompt_tps":null,"decode_tps":null,"input_tokens":982,"output_tokens":187,"peak_rss_mb":8261.4453125,"verdict":{"state":"Fail"},"prompt_ids_sha256":"27e16faa8fbdff4c4d6e0ff52b176400b21425330e38ee63f75e165299d93d1b"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"7788e95d4d3ecd6288d7d51fa0ed8459158b9b7ff7b94bbbffd244bbb7e1dad1","stratum":"2k","engine":"apr","engine_version":"apr 0.69.3 (96eb79f51)","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":4941.385534,"prompt_ms":null,"prompt_tps":null,"decode_tps":null,"input_tokens":691,"output_tokens":131,"peak_rss_mb":8284.93359375,"verdict":{"state":"Fail"},"prompt_ids_sha256":"10e34de14fde098ef400a5795d6d775f2af564296e76433ce573ff85c38f81a0"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"366eed3dd5e0055ebfdd837af58bd800debdd62e4c0a7f7da2dffa84d06ada9f","stratum":"2k","engine":"apr","engine_version":"apr 0.69.3 (96eb79f51)","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":6778.04837,"prompt_ms":null,"prompt_tps":null,"decode_tps":null,"input_tokens":1398,"output_tokens":149,"peak_rss_mb":8297.80078125,"verdict":{"state":"Pass"},"prompt_ids_sha256":"513376357643c1fb6950a0630d0a17b9cd1678c7d4961730e6cdd82a5e6755c4"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"f9d7c31e37b0ed40fbfd87af8984c8860ec6c047dc68a8b55bcb88ee67ee6452","stratum":"2k","engine":"apr","engine_version":"apr 0.69.3 (96eb79f51)","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":6065.204691,"prompt_ms":null,"prompt_tps":null,"decode_tps":null,"input_tokens":713,"output_tokens":164,"peak_rss_mb":8297.80078125,"verdict":{"state":"Fail"},"prompt_ids_sha256":"13634b8349a0b7acd08e20bc05d86f438f3a09e1be22555abc534924c2845212"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"f8879c54b81ffd2aaeaf6398391a818b5f56d1c37ffd4a71100bf19864c1b902","stratum":"2k","engine":"apr","engine_version":"apr 0.69.3 (96eb79f51)","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":5101.595497,"prompt_ms":null,"prompt_tps":null,"decode_tps":null,"input_tokens":950,"output_tokens":126,"peak_rss_mb":8297.80078125,"verdict":{"state":"Pass"},"prompt_ids_sha256":"6526b0ed8df35ece7bfb8f728415890c02540d9548fa313c47f20bb0514294f2"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"296c5544001c3a5e623d3bc13a3b59728bad8d086d6f5baaa8ec44409d3e5cea","stratum":"2k","engine":"apr","engine_version":"apr 0.69.3 (96eb79f51)","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":4192.052654,"prompt_ms":null,"prompt_tps":null,"decode_tps":null,"input_tokens":322,"output_tokens":123,"peak_rss_mb":8297.80078125,"verdict":{"state":"Fail"},"prompt_ids_sha256":"1f540bb4461f766244ab8dbc1d7a1c48162b6cb270d8ae0eb239e075d14fcb7f"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"275cbe4c206c2eb59aca3ca92f0bbd7f171bc949a205afe8e44882e4e56fdc5c","stratum":"2k","engine":"apr","engine_version":"apr 0.69.3 (96eb79f51)","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":12117.445706,"prompt_ms":null,"prompt_tps":null,"decode_tps":null,"input_tokens":1904,"output_tokens":263,"peak_rss_mb":8310.3671875,"verdict":{"state":"Fail"},"prompt_ids_sha256":"e482de36d4b6c8e06ae0882d309b33305eabf0479dedcbcc690ff717f0dc969c"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"2d012d3f8e232bfdbed32db1d29426b65b9c7725d2a001308ac4fb59f8cdafa9","stratum":"2k","engine":"apr","engine_version":"apr 0.69.3 (96eb79f51)","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":15447.954958,"prompt_ms":null,"prompt_tps":null,"decode_tps":null,"input_tokens":1827,"output_tokens":351,"peak_rss_mb":8310.3671875,"verdict":{"state":"Pass"},"prompt_ids_sha256":"aee8112d87fb3291454282615ad62f23da8c7bfa3b2e162d7e909ec867ad11b3"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"e777a873cb9229ece5b23335a8dd1bb421f1b5e298f76024a7af842e2cfcadfb","stratum":"2k","engine":"apr","engine_version":"apr 0.69.3 (96eb79f51)","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":7954.94046,"prompt_ms":null,"prompt_tps":null,"decode_tps":null,"input_tokens":1889,"output_tokens":157,"peak_rss_mb":8310.3671875,"verdict":{"state":"Pass"},"prompt_ids_sha256":"dd8abf75e8df1fb4ba891ef3ac0728486da2bed010880eb742d6d79e669fe432"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"a7a62e12ad053746008e769e95d9c06f39e8b741a52ad4f650d05d1ca720c754","stratum":"2k","engine":"apr","engine_version":"apr 0.69.3 (96eb79f51)","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":10240.693705,"prompt_ms":null,"prompt_tps":null,"decode_tps":null,"input_tokens":1826,"output_tokens":218,"peak_rss_mb":8310.3671875,"verdict":{"state":"Pass"},"prompt_ids_sha256":"7081fe352adf097f8a152f710ba69d4a79e4044130c1c3982d8f715f175e3c32"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"66ec2c8042874218dcf97952a6a557fe1223cb59db4a28bf7615cd2a6a4f269b","stratum":"2k","engine":"apr","engine_version":"apr 0.69.3 (96eb79f51)","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":3633.738136,"prompt_ms":null,"prompt_tps":null,"decode_tps":null,"input_tokens":363,"output_tokens":103,"peak_rss_mb":8310.3671875,"verdict":{"state":"Pass"},"prompt_ids_sha256":"7f8167b3b624a3f6f0dd6b3c48781ebe5c7aa79d9a20f2016a592cbdb64df348"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"5f449a13b53a4663c67c7b0cea0588720e0a2e63e79561ba6adcd58d9bd87d77","stratum":"2k","engine":"apr","engine_version":"apr 0.69.3 (96eb79f51)","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":14224.828501,"prompt_ms":null,"prompt_tps":null,"decode_tps":null,"input_tokens":1744,"output_tokens":325,"peak_rss_mb":8310.3671875,"verdict":{"state":"Fail"},"prompt_ids_sha256":"4be9a7ed9292e82aa712abd073d04ef245be3607a90208a80f7195e73fbdfd62"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"3b86db746737d9338f2ac5c3c0aa7184acd7f05314b46fb8ae1a8bbdd83ac4ad","stratum":"2k","engine":"apr","engine_version":"apr 0.69.3 (96eb79f51)","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":4888.024328,"prompt_ms":null,"prompt_tps":null,"decode_tps":null,"input_tokens":657,"output_tokens":130,"peak_rss_mb":8310.3671875,"verdict":{"state":"Pass"},"prompt_ids_sha256":"8278ed707fc9d1928b2ec8f690a552e0a16913c5847057eb444ca2509a21b839"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"a5f04841ad4bdf65c809d00511d1b4669ed6f52b6ec0be2eac5ee0417b29297f","stratum":"2k","engine":"apr","engine_version":"apr 0.69.3 (96eb79f51)","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":5758.130424,"prompt_ms":null,"prompt_tps":null,"decode_tps":null,"input_tokens":1630,"output_tokens":110,"peak_rss_mb":8310.3671875,"verdict":{"state":"Pass"},"prompt_ids_sha256":"f02afc68f2366821fc07f38c65009e232df057bd81d998875a7600a85ff58c1c"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"f1206fea7a5282d2607d846f5d4f0a9fe7b6982f8d0ab75a42632d7a8ca4e487","stratum":"2k","engine":"apr","engine_version":"apr 0.69.3 (96eb79f51)","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":7131.127067,"prompt_ms":null,"prompt_tps":null,"decode_tps":null,"input_tokens":1005,"output_tokens":182,"peak_rss_mb":8310.3671875,"verdict":{"state":"Pass"},"prompt_ids_sha256":"f76934c956f6886a06b4e394e263a24c39df6734ad36fed9be55913886901237"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"993b00de597f00fefd6e145197fc5659223b89690cc0a671c636d955b4c961c5","stratum":"2k","engine":"apr","engine_version":"apr 0.69.3 (96eb79f51)","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":10728.233937,"prompt_ms":null,"prompt_tps":null,"decode_tps":null,"input_tokens":1586,"output_tokens":244,"peak_rss_mb":8310.3671875,"verdict":{"state":"Pass"},"prompt_ids_sha256":"79f32211f992adecbdc1e0848ba2e99956661eb798fbac366156478f0670d2d2"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"47015b6bbe7f1a4b0b27cd8396c3430142487ab219124adf8c7f331e365c7ec3","stratum":"2k","engine":"apr","engine_version":"apr 0.69.3 (96eb79f51)","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":10310.61504,"prompt_ms":null,"prompt_tps":null,"decode_tps":null,"input_tokens":1660,"output_tokens":229,"peak_rss_mb":8310.3671875,"verdict":{"state":"Pass"},"prompt_ids_sha256":"c8ceab8a8499f6119d42fc491d582972ddcbb9d0eeb513d16f0a8bc73f03a953"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"9af203f89adb6dc0348f46161c0da6bdf7ebea3752a638220571ad3a65854634","stratum":"2k","engine":"apr","engine_version":"apr 0.69.3 (96eb79f51)","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":7979.066089,"prompt_ms":null,"prompt_tps":null,"decode_tps":null,"input_tokens":1711,"output_tokens":164,"peak_rss_mb":8310.3671875,"verdict":{"state":"Pass"},"prompt_ids_sha256":"40d56ab8f83a8a2a5627f8c21de53e8cc3086149ca9a2d3f7bf948f43294f577"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"34d4c5a7a0e46bbd1b00935ef3000cc8ddba8dd64cbdffa9e01f1207863f945c","stratum":"2k","engine":"apr","engine_version":"apr 0.69.3 (96eb79f51)","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":11331.380999,"prompt_ms":null,"prompt_tps":null,"decode_tps":null,"input_tokens":1453,"output_tokens":268,"peak_rss_mb":8310.3671875,"verdict":{"state":"Fail"},"prompt_ids_sha256":"6ea38a67c9dcd29895861eb72204a5cb6730ab44d894a6ab601f1753ff7c9df4"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"167d286e0f8a0bae32d26b9d8abc056df02e030188cf395ff255acc3cb1d640d","stratum":"2k","engine":"apr","engine_version":"apr 0.69.3 (96eb79f51)","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":8270.712935000001,"prompt_ms":null,"prompt_tps":null,"decode_tps":null,"input_tokens":944,"output_tokens":217,"peak_rss_mb":8310.3671875,"verdict":{"state":"Fail"},"prompt_ids_sha256":"2fcc6f03b34f1893afd08fe8ec1dcf30568ee3397ef74662c46ef4085107c1d6"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"fab75467c60ebd40cc94c13296129957a26a7eb9865a58175dd09db9610f1631","stratum":"2k","engine":"apr","engine_version":"apr 0.69.3 (96eb79f51)","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":8298.519769,"prompt_ms":null,"prompt_tps":null,"decode_tps":null,"input_tokens":1053,"output_tokens":211,"peak_rss_mb":8310.3671875,"verdict":{"state":"Pass"},"prompt_ids_sha256":"e857ce0443d4fb4bf8d286c247650ecf93b401aff929197f55c9aa7777881371"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"c274c3708f67fd97e467895e6c780665af6a13a28c8a727d25ea1863e3328dc8","stratum":"2k","engine":"apr","engine_version":"apr 0.69.3 (96eb79f51)","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":9211.32013,"prompt_ms":null,"prompt_tps":null,"decode_tps":null,"input_tokens":1544,"output_tokens":203,"peak_rss_mb":8310.3671875,"verdict":{"state":"Pass"},"prompt_ids_sha256":"6624743530f806ebfeef29164bc538770adccaf112ec5c254583759b9530f2af"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"11ca5b1e87d2bbf217c1fd81b2c2ea63a96390dea93e9d9258aa7dff541f8de8","stratum":"2k","engine":"apr","engine_version":"apr 0.69.3 (96eb79f51)","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":10910.974132,"prompt_ms":null,"prompt_tps":null,"decode_tps":null,"input_tokens":1531,"output_tokens":251,"peak_rss_mb":8310.3671875,"verdict":{"state":"Pass"},"prompt_ids_sha256":"7f03bbbdd56a316389c60433ff343e431f7bf8984965b1d7c775d0a2b663d42c"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"7ba9cfea210f85f585654545dc13dc487fe37b56924c0996577c87a463100bac","stratum":"2k","engine":"apr","engine_version":"apr 0.69.3 (96eb79f51)","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":3598.4329439999997,"prompt_ms":null,"prompt_tps":null,"decode_tps":null,"input_tokens":274,"output_tokens":104,"peak_rss_mb":8310.3671875,"verdict":{"state":"Pass"},"prompt_ids_sha256":"4a21bcf8590e6ea0af00d401671ef07b3715194ed9f4846feab7f8ce28239f10"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"b1c9fb8c5a26aa63dcdb57e6ec6ce35a2c5a73287516f57082f77944c007add1","stratum":"2k","engine":"apr","engine_version":"apr 0.69.3 (96eb79f51)","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":4199.657587,"prompt_ms":null,"prompt_tps":null,"decode_tps":null,"input_tokens":792,"output_tokens":102,"peak_rss_mb":8310.3671875,"verdict":{"state":"Pass"},"prompt_ids_sha256":"2f02b9baa0667bc81604dd82b8f81abd390a46442f75a67fed1f9ca6ef54d929"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"9d82e15d5df69ff6b6a2f1d4dd81ba144dca2978ee4f352de1b134cf86a4bfdd","stratum":"2k","engine":"apr","engine_version":"apr 0.69.3 (96eb79f51)","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":13089.143469999999,"prompt_ms":null,"prompt_tps":null,"decode_tps":null,"input_tokens":1615,"output_tokens":303,"peak_rss_mb":8310.3671875,"verdict":{"state":"Fail"},"prompt_ids_sha256":"7e2fd107b311a51bce7d7d6826cbde8412778be86ccc57cd60587b5bf24a54f4"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"4dca9a17db342792f25605430639922e342855e3341653071266173c29d9a859","stratum":"2k","engine":"apr","engine_version":"apr 0.69.3 (96eb79f51)","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":9370.266309,"prompt_ms":null,"prompt_tps":null,"decode_tps":null,"input_tokens":1182,"output_tokens":234,"peak_rss_mb":8310.3671875,"verdict":{"state":"Fail"},"prompt_ids_sha256":"cb47cabf619ed11eda1601028c79ced363020fe66316b9895d5602b2f527c0d8"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"e8e5551522f3e72418df3e0fa2227dded865de9949b65a4b85becc6cd3acbc35","stratum":"2k","engine":"apr","engine_version":"apr 0.69.3 (96eb79f51)","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":3381.066014,"prompt_ms":null,"prompt_tps":null,"decode_tps":null,"input_tokens":384,"output_tokens":94,"peak_rss_mb":8310.3671875,"verdict":{"state":"Pass"},"prompt_ids_sha256":"0655ead49567576a1c526b6f710c1807619174567ee1f131159a9ae03c68b8ed"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"e0cf54bdaa7fbbbd164db0ebf846b77b0b78b65b8bfd4a392fcc3171c64a2504","stratum":"2k","engine":"apr","engine_version":"apr 0.69.3 (96eb79f51)","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":3375.42733,"prompt_ms":null,"prompt_tps":null,"decode_tps":null,"input_tokens":605,"output_tokens":84,"peak_rss_mb":8310.3671875,"verdict":{"state":"Pass"},"prompt_ids_sha256":"a7da9728eaf7dc43519a3329e56fbddffb918e60cd11e3b60c2fdd066013112c"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"7492d83b3314d90dae86dcb3530baaa502e8d425427daa63a8e9fc03de96b069","stratum":"2k","engine":"apr","engine_version":"apr 0.69.3 (96eb79f51)","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":5965.539522,"prompt_ms":null,"prompt_tps":null,"decode_tps":null,"input_tokens":1295,"output_tokens":128,"peak_rss_mb":8310.3671875,"verdict":{"state":"Pass"},"prompt_ids_sha256":"3710b42517c7bd8dcd003e4e02e46f2547485a103ea66db61273afabb7a3c2bf"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"0d9bcac1ca95dc9fb8293713b22add78150aaaacfa6a269e8065ea5605feee17","stratum":"2k","engine":"apr","engine_version":"apr 0.69.3 (96eb79f51)","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":10202.497287,"prompt_ms":null,"prompt_tps":null,"decode_tps":null,"input_tokens":964,"output_tokens":272,"peak_rss_mb":8310.3671875,"verdict":{"state":"Fail"},"prompt_ids_sha256":"5ddc4f9e03e618df9e68b2df8ad867702dff23019b64b9d2ef32f96ad0949717"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"3f51712000a76356352961b38d5b137cdf5179679655ed3fbfef2acad4eeb0e8","stratum":"2k","engine":"apr","engine_version":"apr 0.69.3 (96eb79f51)","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":4995.059417,"prompt_ms":null,"prompt_tps":null,"decode_tps":null,"input_tokens":688,"output_tokens":131,"peak_rss_mb":8310.3671875,"verdict":{"state":"Pass"},"prompt_ids_sha256":"d7e97aaa3ac6609d18fe9c3ba27bfcd91550e16482bf475e2d4f19eb26acd77a"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"b7e5b398d98a7f3dc848eee0e437e07104d372e9365e6f3ce0c101e6667ed890","stratum":"2k","engine":"apr","engine_version":"apr 0.69.3 (96eb79f51)","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":2865.0890329999997,"prompt_ms":null,"prompt_tps":null,"decode_tps":null,"input_tokens":308,"output_tokens":79,"peak_rss_mb":8310.3671875,"verdict":{"state":"Pass"},"prompt_ids_sha256":"b05019b1e8f3ab2be02e8b81733a49d390f98ae5a16b0e8f13d09593ea04b9e6"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"903ef35fb09bf1cb357547ce61b9b7828b90c24845aa5f9fecd8923e022be75f","stratum":"2k","engine":"apr","engine_version":"apr 0.69.3 (96eb79f51)","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":7234.563354,"prompt_ms":null,"prompt_tps":null,"decode_tps":null,"input_tokens":959,"output_tokens":186,"peak_rss_mb":8310.3671875,"verdict":{"state":"Fail"},"prompt_ids_sha256":"ec9c7f9e8c70ed80adfbcb818a9db16b1472d69cc4b7d5152bd8e139204963b5"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"34186bc542a9bc6191d88846aadf532b79f67209e4b17474d1d4b0a1cf87cc8c","stratum":"2k","engine":"apr","engine_version":"apr 0.69.3 (96eb79f51)","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":4462.099597,"prompt_ms":null,"prompt_tps":null,"decode_tps":null,"input_tokens":526,"output_tokens":120,"peak_rss_mb":8310.3671875,"verdict":{"state":"Pass"},"prompt_ids_sha256":"717745ed7fb623d700dffd2b1f7f0d7ae241a0fc248f74ed4b935313a1a249fc"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"c9a02d51ce74d1c65af0fa596da511b8d2a571aa8f65f8d380ed9838b4c5243e","stratum":"2k","engine":"apr","engine_version":"apr 0.69.3 (96eb79f51)","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":11103.977467,"prompt_ms":null,"prompt_tps":null,"decode_tps":null,"input_tokens":1023,"output_tokens":294,"peak_rss_mb":8310.3671875,"verdict":{"state":"Fail"},"prompt_ids_sha256":"0c2f2e99acdaace2521543f52e497a9e01039e659d7c95ffb47efa75d6a0e742"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"d09cb9713ee02abec5dce5dd4cd0b58237e449a0032fdaf96dcc6ac95a0a6101","stratum":"2k","engine":"apr","engine_version":"apr 0.69.3 (96eb79f51)","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":4283.051697,"prompt_ms":null,"prompt_tps":null,"decode_tps":null,"input_tokens":484,"output_tokens":118,"peak_rss_mb":8310.3671875,"verdict":{"state":"Pass"},"prompt_ids_sha256":"6ab504c554d5d36c516e432f36c59eac000d1d9af5d19121bcc80d31a4dd1ace"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"a889b6ddfa18809d32510029c8f832ddf1dc03f3209feee377d47043c757104e","stratum":"2k","engine":"apr","engine_version":"apr 0.69.3 (96eb79f51)","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":3582.469446,"prompt_ms":null,"prompt_tps":null,"decode_tps":null,"input_tokens":604,"output_tokens":90,"peak_rss_mb":8310.3671875,"verdict":{"state":"Pass"},"prompt_ids_sha256":"909814ad448169a742c52875b82d6ef865c02703afda46a7450ffbe8394e794a"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"c1d14d2cd0dbbdd90638158ee715c79fc76baa8ff2c6322160dac0bb5bd27771","stratum":"2k","engine":"apr","engine_version":"apr 0.69.3 (96eb79f51)","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":6671.23275,"prompt_ms":null,"prompt_tps":null,"decode_tps":null,"input_tokens":332,"output_tokens":201,"peak_rss_mb":8310.3671875,"verdict":{"state":"Fail"},"prompt_ids_sha256":"82eb2fb253fce96176b306689444000ece53e63e156d9fa4863a938a3e705b71"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"955705510e7439ac4b8e27505f87bcb7d3c71b677cb11bddc4a91d5c44e16a04","stratum":"2k","engine":"apr","engine_version":"apr 0.69.3 (96eb79f51)","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":12006.172019,"prompt_ms":null,"prompt_tps":null,"decode_tps":null,"input_tokens":1792,"output_tokens":264,"peak_rss_mb":8310.3671875,"verdict":{"state":"Fail"},"prompt_ids_sha256":"51eeb6fcedffb81fd02fc9c5f05805cb72211d244fb56aad4117e964b187a2d0"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"bf94290e1f98fc502cb4f962bce2883c3eaa9ca2fe6cb67f6b87c7083d3ec0e1","stratum":"2k","engine":"apr","engine_version":"apr 0.69.3 (96eb79f51)","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":10243.137473,"prompt_ms":null,"prompt_tps":null,"decode_tps":null,"input_tokens":1274,"output_tokens":249,"peak_rss_mb":8310.3671875,"verdict":{"state":"Fail"},"prompt_ids_sha256":"05a06916a4b3576cd25ee421c0588937e59d599607237eeab47bec5ef1db27eb"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"258be4b71904b9f67bdda4ea10d76b9be53c374b04a9d486facd87a97fab0756","stratum":"2k","engine":"apr","engine_version":"apr 0.69.3 (96eb79f51)","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":5893.448309,"prompt_ms":null,"prompt_tps":null,"decode_tps":null,"input_tokens":1213,"output_tokens":131,"peak_rss_mb":8310.3671875,"verdict":{"state":"Pass"},"prompt_ids_sha256":"d1fbddde370dc12c4aec16fbd2fc2fb32418489c008dd55dd3d29e40a5fbd855"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"1bf59f2a1774a0efe188bbfa1ac4a7af14b2059017de009fc98df2a6b8382017","stratum":"2k","engine":"apr","engine_version":"apr 0.69.3 (96eb79f51)","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":14238.447363000001,"prompt_ms":null,"prompt_tps":null,"decode_tps":null,"input_tokens":1791,"output_tokens":320,"peak_rss_mb":8310.3671875,"verdict":{"state":"Fail"},"prompt_ids_sha256":"6c5a3de9322024e61926b8105522954447661224770b9881c91a10be3aa69a7c"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"305cdbe50000c485ff10ec6127fbe0744109a126be589ccbb6d243834c3c13ef","stratum":"2k","engine":"apr","engine_version":"apr 0.69.3 (96eb79f51)","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":7864.068896,"prompt_ms":null,"prompt_tps":null,"decode_tps":null,"input_tokens":1038,"output_tokens":199,"peak_rss_mb":8310.3671875,"verdict":{"state":"Fail"},"prompt_ids_sha256":"0d2fc4877c6ceccc5bcecc9e02a8dc8468a8265069ac9e54235fb1f170a59c3e"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"7680438d94dac7e6c61561a71575435fcd22d1543b508ec9434d00f686511896","stratum":"2k","engine":"apr","engine_version":"apr 0.69.3 (96eb79f51)","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":7384.378154,"prompt_ms":null,"prompt_tps":null,"decode_tps":null,"input_tokens":919,"output_tokens":192,"peak_rss_mb":8310.3671875,"verdict":{"state":"Fail"},"prompt_ids_sha256":"0ab5ac2510b12bcacb54e31107686f33fa3f624fb80ca84c81fbced4c8e24f46"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"d928b82fd55ca6c509650e822de0f3946d5ccb3d5fbbabba30784e0f6483e2ff","stratum":"2k","engine":"apr","engine_version":"apr 0.69.3 (96eb79f51)","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":7716.546454,"prompt_ms":null,"prompt_tps":null,"decode_tps":null,"input_tokens":1428,"output_tokens":171,"peak_rss_mb":8310.3671875,"verdict":{"state":"Fail"},"prompt_ids_sha256":"ef93f1ea44b6a4051e70202045ed0b6322e8987557045343e51ca8cdc8074df0"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"64cce1452c38a191c5f249f069acb21eb4098bd2620cddec04b346c3075849dc","stratum":"2k","engine":"apr","engine_version":"apr 0.69.3 (96eb79f51)","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":6833.0543529999995,"prompt_ms":null,"prompt_tps":null,"decode_tps":null,"input_tokens":613,"output_tokens":191,"peak_rss_mb":8310.3671875,"verdict":{"state":"Fail"},"prompt_ids_sha256":"337e04b1920ed7b30ceedf4ff49f4bf841d2ce7657bafdb2a9165eea27d39ea6"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"69b831cc2059acf0f511c8d788337f36e8ef890f6b86b91f6ffa159b8369d1e5","stratum":"2k","engine":"apr","engine_version":"apr 0.69.3 (96eb79f51)","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":7948.153223,"prompt_ms":null,"prompt_tps":null,"decode_tps":null,"input_tokens":1939,"output_tokens":152,"peak_rss_mb":8326.40234375,"verdict":{"state":"Pass"},"prompt_ids_sha256":"d93d068e263876cd1cb37558b0015f0fdcf11be3da9eefe2c4aff454b8a9e616"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"14d125f8a319774d2ddc89c330d806c3371695a96ec1cc8757cb63f20c9afe37","stratum":"2k","engine":"apr","engine_version":"apr 0.69.3 (96eb79f51)","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":6023.569746,"prompt_ms":null,"prompt_tps":null,"decode_tps":null,"input_tokens":579,"output_tokens":167,"peak_rss_mb":8326.40234375,"verdict":{"state":"Pass"},"prompt_ids_sha256":"04222af6320ccf0f9b9bc386e01fc9ed98736d6df5ee61bd98894f8facc87347"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"0b54a849b9e7541ec803e57fab1147aee8d209935102b9230c0a024229ccdac7","stratum":"2k","engine":"apr","engine_version":"apr 0.69.3 (96eb79f51)","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":4952.496112000001,"prompt_ms":null,"prompt_tps":null,"decode_tps":null,"input_tokens":565,"output_tokens":134,"peak_rss_mb":8326.40234375,"verdict":{"state":"Pass"},"prompt_ids_sha256":"6ea89d378295c73f8cfc8d17f5f2fcab76f6abb4e8c59463e8a4a4025885fed7"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"ae0f31c8b5b43715e0f5f31727de5648fe149c06faef682adb8898f4dddc0d59","stratum":"8k","engine":"apr","engine_version":"apr 0.69.3 (96eb79f51)","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":11454.704104999999,"prompt_ms":null,"prompt_tps":null,"decode_tps":null,"input_tokens":2159,"output_tokens":226,"peak_rss_mb":8329.515625,"verdict":{"state":"Pass"},"prompt_ids_sha256":"a81f7ea1e1114b3b88a0717effd69745ef46e82f2efdcdbd465afae931ebf1f0"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"6324ab9b418859326df39301f2fc9ca30815280479d95ad375d91e6252a4b82b","stratum":"8k","engine":"apr","engine_version":"apr 0.69.3 (96eb79f51)","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":16999.86347,"prompt_ms":null,"prompt_tps":null,"decode_tps":null,"input_tokens":3573,"output_tokens":280,"peak_rss_mb":8334.42578125,"verdict":{"state":"Pass"},"prompt_ids_sha256":"3ae7c4e045aeaa9df4ea1cf4961254d625990b616146ceda3ec66666473b7cff"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"187723585ce31ae0a9dfb2bd1c94ee372724ba456ade6c3ef28816fb1e369e4d","stratum":"8k","engine":"apr","engine_version":"apr 0.69.3 (96eb79f51)","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":7671.221261999999,"prompt_ms":null,"prompt_tps":null,"decode_tps":null,"input_tokens":2266,"output_tokens":129,"peak_rss_mb":8334.42578125,"verdict":{"state":"Pass"},"prompt_ids_sha256":"eb266f1f82f6cd93fd1590494e67349a0bed128c01781f180d09e71b40998f5d"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"dd16771d607cda7d13b2528bec93be7261e9d739a6d40c855b0794500da92522","stratum":"8k","engine":"apr","engine_version":"apr 0.69.3 (96eb79f51)","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":22358.72602,"prompt_ms":null,"prompt_tps":null,"decode_tps":null,"input_tokens":4958,"output_tokens":314,"peak_rss_mb":8334.46484375,"verdict":{"state":"Fail"},"prompt_ids_sha256":"51ee3f91d5eaa6d044852fa83079495964304a501fbc737ecc01becf473718b6"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"40ce4b133ac399054d67393d53e91be97d3cafaa33622094f90a35fdea53c703","stratum":"8k","engine":"apr","engine_version":"apr 0.69.3 (96eb79f51)","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":28753.003495,"prompt_ms":null,"prompt_tps":null,"decode_tps":null,"input_tokens":6941,"output_tokens":330,"peak_rss_mb":8334.46484375,"verdict":{"state":"Pass"},"prompt_ids_sha256":"212c625213d37b2c9f816b691908099bd9f9a58e680ca2d15863dcdabb735b51"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"bc9a159757fe4720bfdc9073b5c2d8879c8adec829ab9308b1b1031768b5c3f0","stratum":"8k","engine":"apr","engine_version":"apr 0.69.3 (96eb79f51)","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":29602.255118999998,"prompt_ms":null,"prompt_tps":null,"decode_tps":null,"input_tokens":6520,"output_tokens":361,"peak_rss_mb":8353.359375,"verdict":{"state":"Pass"},"prompt_ids_sha256":"966d327b207b9dda0ee074cf93126d6091ceef81459965ec9b5b1a03824d5f5f"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"33b63a5283685481a92f5226d573249f59330892462fc75a3c5ff14e9c61ba82","stratum":"8k","engine":"apr","engine_version":"apr 0.69.3 (96eb79f51)","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":29657.842814,"prompt_ms":null,"prompt_tps":null,"decode_tps":null,"input_tokens":6791,"output_tokens":349,"peak_rss_mb":8353.375,"verdict":{"state":"Pass"},"prompt_ids_sha256":"8d3561cb81010d0287b8d36013ce4022187e868e05fb5a2acba13d93ddd0ba75"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"df48ffe5f40ed18ea677fa829b51ee4eaaee5c8caa5a7462bbe212d83396c540","stratum":"8k","engine":"apr","engine_version":"apr 0.69.3 (96eb79f51)","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":17953.651576,"prompt_ms":null,"prompt_tps":null,"decode_tps":null,"input_tokens":3443,"output_tokens":306,"peak_rss_mb":8353.375,"verdict":{"state":"Pass"},"prompt_ids_sha256":"513d5ac32c370db50606bba64bf9369a630e11ff0935f649326b6b1082750baf"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"cbc1450d830a7bda671630905c4b86039bf483e305a0d8fecf6971de02ad0406","stratum":"8k","engine":"apr","engine_version":"apr 0.69.3 (96eb79f51)","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":21602.888137,"prompt_ms":null,"prompt_tps":null,"decode_tps":null,"input_tokens":6779,"output_tokens":227,"peak_rss_mb":8353.4296875,"verdict":{"state":"Pass"},"prompt_ids_sha256":"c9b299004f9b7cc8d7af8cf6262f62e552c1752be74d07b00d51c812552f258f"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"62b4bd91655d82632f2ad6754eac98e25881dcdfeb3eff15172402b9da179800","stratum":"8k","engine":"apr","engine_version":"apr 0.69.3 (96eb79f51)","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":19636.255628,"prompt_ms":null,"prompt_tps":null,"decode_tps":null,"input_tokens":3914,"output_tokens":315,"peak_rss_mb":8356.76171875,"verdict":{"state":"Pass"},"prompt_ids_sha256":"1d6f64f5e542ef9606b341f0aa9e816aad85f379d21ff095d0138038b8b4ba4a"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"a6c9bb4b1ea341e343c764aed2b694ca7f4c6ab69461291cf6e20a2d1fe53a47","stratum":"8k","engine":"apr","engine_version":"apr 0.69.3 (96eb79f51)","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":15418.528681,"prompt_ms":null,"prompt_tps":null,"decode_tps":null,"input_tokens":3041,"output_tokens":277,"peak_rss_mb":8356.76171875,"verdict":{"state":"Fail"},"prompt_ids_sha256":"a5e089a434b2f460a61b44fe52bddb2e49e8dc5668c0e6de4a0f0bad9f8fe848"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"1239ceb996f617d8cda28ed536075072f460a875c5a4a25a2ef422a9d923043e","stratum":"8k","engine":"apr","engine_version":"apr 0.69.3 (96eb79f51)","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":11743.05669,"prompt_ms":null,"prompt_tps":null,"decode_tps":null,"input_tokens":4300,"output_tokens":142,"peak_rss_mb":8356.76171875,"verdict":{"state":"Pass"},"prompt_ids_sha256":"f276cbcdd3b7b2349836fbdb1d397d0d986373840f6ddaddc46c581e5f7288df"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"90134b8a8a941ee22156f752e986d85a39c92f11186fff94174ce25e7fc2c65d","stratum":"8k","engine":"apr","engine_version":"apr 0.69.3 (96eb79f51)","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":16577.175181000002,"prompt_ms":null,"prompt_tps":null,"decode_tps":null,"input_tokens":3142,"output_tokens":297,"peak_rss_mb":8356.76171875,"verdict":{"state":"Fail"},"prompt_ids_sha256":"1fb374c347b9be341a38517cceab0992299ad2aaea74befbaf3906626c37d34b"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"b3c917322e600c6a75b1fdabd254bbb864628227545b9f51c05fedcb33835e55","stratum":"8k","engine":"apr","engine_version":"apr 0.69.3 (96eb79f51)","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":14980.787681,"prompt_ms":null,"prompt_tps":null,"decode_tps":null,"input_tokens":2745,"output_tokens":281,"peak_rss_mb":8356.76171875,"verdict":{"state":"Fail"},"prompt_ids_sha256":"a24ccc6f60289e1cfdec88d1bedae95fd3e07227a8da2a84b31e646d817ee93c"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"38f0a202313d9e2120e55b92747ced425cac793e09a4db44c212449b1cd8566e","stratum":"8k","engine":"apr","engine_version":"apr 0.69.3 (96eb79f51)","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":32071.765473,"prompt_ms":null,"prompt_tps":null,"decode_tps":null,"input_tokens":4588,"output_tokens":512,"peak_rss_mb":8356.76171875,"verdict":{"state":"Fail"},"prompt_ids_sha256":"95b34da10969f03fa2337931389f1835b5efc1eb5f79585e13cb239ef9503d00"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"e12a747f914c805b9dcbc4349d5ed7fe706e6d0334848a677837cef3d8b95400","stratum":"8k","engine":"apr","engine_version":"apr 0.69.3 (96eb79f51)","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":9324.647144,"prompt_ms":null,"prompt_tps":null,"decode_tps":null,"input_tokens":2467,"output_tokens":161,"peak_rss_mb":8356.76171875,"verdict":{"state":"Pass"},"prompt_ids_sha256":"0b4a3d5a626c23d2a0a7f227b0d4edfddd53b4da5a6bb9963171c2d1ab882b65"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"a74db9b8ff37392fa217b05b5e70a2564d3c8a6960f02eebe3db8016184e114a","stratum":"8k","engine":"apr","engine_version":"apr 0.69.3 (96eb79f51)","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":13023.945704000002,"prompt_ms":null,"prompt_tps":null,"decode_tps":null,"input_tokens":5961,"output_tokens":115,"peak_rss_mb":8356.7734375,"verdict":{"state":"Pass"},"prompt_ids_sha256":"ad6c508530582a081bce003291bb7ee3c17effe3ed0527e73172c39694fbc89b"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"27565a444276a49af4557cade6d92398cbccd183c6366e78a5453683923bfe6b","stratum":"8k","engine":"apr","engine_version":"apr 0.69.3 (96eb79f51)","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":22863.439116999998,"prompt_ms":null,"prompt_tps":null,"decode_tps":null,"input_tokens":7875,"output_tokens":205,"peak_rss_mb":8374.9296875,"verdict":{"state":"Pass"},"prompt_ids_sha256":"72815a6948e2f27d022cec776bb31fd4a99af1123e5c14a25c6fcb175a28e00a"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"e0d0c14ea617d00ecec166c8e598898b3d1b757d639df1729ddf1e3af37872cd","stratum":"8k","engine":"apr","engine_version":"apr 0.69.3 (96eb79f51)","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":9110.208341,"prompt_ms":null,"prompt_tps":null,"decode_tps":null,"input_tokens":2827,"output_tokens":144,"peak_rss_mb":8375.08203125,"verdict":{"state":"Pass"},"prompt_ids_sha256":"5019b3cebd26be737bb7fe45f6ddb10faa0e2c3bb6fd157fa1eee1deea81d19d"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"7269a7b0e6abf53a633d344677435b768b4342ad507d8833da2aa1b5b90747ee","stratum":"8k","engine":"apr","engine_version":"apr 0.69.3 (96eb79f51)","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":18501.580228000003,"prompt_ms":null,"prompt_tps":null,"decode_tps":null,"input_tokens":4639,"output_tokens":258,"peak_rss_mb":8375.0859375,"verdict":{"state":"Pass"},"prompt_ids_sha256":"b76967bf52a3d1d50f5eeb855e43df852e8ff8be2ed8bb4909ea519ca09fa086"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"f87c2ffe3aafeec5f8308352f5190b3c8ec03ed0eafd6d94b000ef39bb3d922f","stratum":"8k","engine":"apr","engine_version":"apr 0.69.3 (96eb79f51)","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":33407.207035,"prompt_ms":null,"prompt_tps":null,"decode_tps":null,"input_tokens":4975,"output_tokens":512,"peak_rss_mb":8375.0859375,"verdict":{"state":"Fail"},"prompt_ids_sha256":"805c7c9887566084d332a0ff4e3decc7ee4a100a69194e545f3460c67c1fb92a"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"c7323e9dc8cc06122998a5fb83ccb487d2538b7444f9852de80f63983e93dc44","stratum":"8k","engine":"apr","engine_version":"apr 0.69.3 (96eb79f51)","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":26607.406881,"prompt_ms":null,"prompt_tps":null,"decode_tps":null,"input_tokens":3808,"output_tokens":460,"peak_rss_mb":8375.0859375,"verdict":{"state":"Fail"},"prompt_ids_sha256":"b5cb29cd4d1d2b02437000ebf0115b5483e5b7bf00187bc0d72bb47ba8be7019"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"3bed1317707a01f00daf828a5434d3681278c66100d991520153baa4dc7ebd61","stratum":"8k","engine":"apr","engine_version":"apr 0.69.3 (96eb79f51)","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":14349.450412999999,"prompt_ms":null,"prompt_tps":null,"decode_tps":null,"input_tokens":2065,"output_tokens":301,"peak_rss_mb":8375.0859375,"verdict":{"state":"Fail"},"prompt_ids_sha256":"92f0703d894d6fb99af6cf992d9c3d76abd4fad16e00adf056499d5bf79df42a"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"a974a820da6ee897219e5353ad22358cd5cec02a3534baa60388a3f554850d96","stratum":"8k","engine":"apr","engine_version":"apr 0.69.3 (96eb79f51)","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":13080.217937000001,"prompt_ms":null,"prompt_tps":null,"decode_tps":null,"input_tokens":4341,"output_tokens":167,"peak_rss_mb":8375.0859375,"verdict":{"state":"Fail"},"prompt_ids_sha256":"4017ef5b163960754cdb6c87147d51d9110da0dcbb8d7f37fc5b1b1af7c2f8d4"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"1c50664db77045b861287a6f03ded90d514b8d0a32f04336caa5c9c90777b18c","stratum":"8k","engine":"apr","engine_version":"apr 0.69.3 (96eb79f51)","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":11734.694590000001,"prompt_ms":null,"prompt_tps":null,"decode_tps":null,"input_tokens":2217,"output_tokens":228,"peak_rss_mb":8375.0859375,"verdict":{"state":"Pass"},"prompt_ids_sha256":"ad1b4023aa37427996cc325caba661c60a9cf8343640671dcb1983cfbc5c89b8"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"7e09255545a53208daf6239daa699089bbd30c3ec111d44a59cbe007a8004a57","stratum":"8k","engine":"apr","engine_version":"apr 0.69.3 (96eb79f51)","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":11730.253426000001,"prompt_ms":null,"prompt_tps":null,"decode_tps":null,"input_tokens":3919,"output_tokens":157,"peak_rss_mb":8375.0859375,"verdict":{"state":"Pass"},"prompt_ids_sha256":"7979c0667fd9abec64e255baa8d0377e7856b52e4e008e1f1b7c35c4471ccdf2"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"1aee819c753ccc3dd6976511c879d33fd99478a25bfe8c5639d006aebe6d00ad","stratum":"8k","engine":"apr","engine_version":"apr 0.69.3 (96eb79f51)","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":13615.5462,"prompt_ms":null,"prompt_tps":null,"decode_tps":null,"input_tokens":4806,"output_tokens":162,"peak_rss_mb":8375.0859375,"verdict":{"state":"Pass"},"prompt_ids_sha256":"93d0c8edbd6a7cca000f9f23b65b7bd49a90c3e64f4d169cf4705ff14a6bbaf9"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"e92967237c1b1abeb4a548c343928a0afda2c3df6c49e94cec87b7146276b045","stratum":"8k","engine":"apr","engine_version":"apr 0.69.3 (96eb79f51)","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":19672.617267999998,"prompt_ms":null,"prompt_tps":null,"decode_tps":null,"input_tokens":4026,"output_tokens":310,"peak_rss_mb":8375.0859375,"verdict":{"state":"Fail"},"prompt_ids_sha256":"96c30189e094a2ed3603508cd4222ab22c468239354545620eebf61674389fb1"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"5ef3cfe455ce3ddedb5fd853a62ccbda685b16f06e7ec8818c46f639474fd302","stratum":"8k","engine":"apr","engine_version":"apr 0.69.3 (96eb79f51)","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":9652.820052,"prompt_ms":null,"prompt_tps":null,"decode_tps":null,"input_tokens":2815,"output_tokens":158,"peak_rss_mb":8375.0859375,"verdict":{"state":"Pass"},"prompt_ids_sha256":"bc74840b53b9b199026a3b937a88491510f44071bb5e597f5a05fc460493e8e5"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"08db5d324448f2faa678eca074285b6275f42db2c667f594ede9fa4adfc9c566","stratum":"8k","engine":"apr","engine_version":"apr 0.69.3 (96eb79f51)","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":20744.090526,"prompt_ms":null,"prompt_tps":null,"decode_tps":null,"input_tokens":5698,"output_tokens":253,"peak_rss_mb":8375.0859375,"verdict":{"state":"Pass"},"prompt_ids_sha256":"759d47a6fcd52490cc43e3f9f91db6587e8b59bf2ed44a06dae3647f1cab6df2"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"bc995711883be2b01bb079a4433bb34b0eb50a9cda6440856b8c0665d7a4afb5","stratum":"8k","engine":"apr","engine_version":"apr 0.69.3 (96eb79f51)","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":8675.179375999998,"prompt_ms":null,"prompt_tps":null,"decode_tps":null,"input_tokens":2795,"output_tokens":136,"peak_rss_mb":8375.0859375,"verdict":{"state":"Pass"},"prompt_ids_sha256":"6a64782f6b63fcb21b6814ed2634f945270c9fb500f9ec47abffff02026b3b33"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"9f29c99424409b984aeea5524ec870e016e2e16c617351a938ed700a057ce9ba","stratum":"8k","engine":"apr","engine_version":"apr 0.69.3 (96eb79f51)","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":25658.819978,"prompt_ms":null,"prompt_tps":null,"decode_tps":null,"input_tokens":7483,"output_tokens":262,"peak_rss_mb":8375.0859375,"verdict":{"state":"Pass"},"prompt_ids_sha256":"f79d101859efe06916d34257165ec054aa373990f5d969fc6c2513c9ee1dd354"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"3d765c54312ef5c279e895fb042a9c08ef9bcd6b44e1098b9db17f68d379f845","stratum":"8k","engine":"apr","engine_version":"apr 0.69.3 (96eb79f51)","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":20922.884415,"prompt_ms":null,"prompt_tps":null,"decode_tps":null,"input_tokens":3514,"output_tokens":364,"peak_rss_mb":8375.0859375,"verdict":{"state":"Pass"},"prompt_ids_sha256":"afa4aed5cdc97dd82a2f3c6f1c3cf83de33f5a873bda726cf8cbedecc68e184a"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"b66cef24141dfcf0cd4776c23e13ac2a297f86fb966ea2f474d3ba2b22ac4689","stratum":"8k","engine":"apr","engine_version":"apr 0.69.3 (96eb79f51)","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":16553.154469999998,"prompt_ms":null,"prompt_tps":null,"decode_tps":null,"input_tokens":2213,"output_tokens":345,"peak_rss_mb":8375.0859375,"verdict":{"state":"Pass"},"prompt_ids_sha256":"a7ed93b0333e5ba437c439c1d28332f7c2c21e02f641095bc38eadd1a02cd167"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"bf69f8045076d4b114aef30013614158379bdcec1dddb96497b724fd4bd16df2","stratum":"8k","engine":"apr","engine_version":"apr 0.69.3 (96eb79f51)","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":16174.927587999999,"prompt_ms":null,"prompt_tps":null,"decode_tps":null,"input_tokens":4757,"output_tokens":211,"peak_rss_mb":8375.0859375,"verdict":{"state":"Pass"},"prompt_ids_sha256":"ef2a1c7ad19234288174165afe4ff5dc308ed4a5d5448495732bf50390f28fde"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"0ba74445847c37901a423c32a811b0f15026d75d3dd2225fd67ab1a474f97ae2","stratum":"8k","engine":"apr","engine_version":"apr 0.69.3 (96eb79f51)","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":18611.230236,"prompt_ms":null,"prompt_tps":null,"decode_tps":null,"input_tokens":5533,"output_tokens":224,"peak_rss_mb":8375.0859375,"verdict":{"state":"Pass"},"prompt_ids_sha256":"5fc6cf167f33d8897f20f6457b0677a778f646264bdb552be4e00fb52b0e6b9f"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"a08ed5ccc3331e0a7179c7d49737607bccb9a9cfbc16c6533afacfa6072b133f","stratum":"8k","engine":"apr","engine_version":"apr 0.69.3 (96eb79f51)","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":6772.200596000001,"prompt_ms":null,"prompt_tps":null,"decode_tps":null,"input_tokens":2330,"output_tokens":104,"peak_rss_mb":8375.0859375,"verdict":{"state":"Pass"},"prompt_ids_sha256":"baf2d30022d1dbc62e43697b26bd1999ee610fcd6b4275293cd352e091390a04"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"b0bdb087e1d95680866dad2b47061fa38e925929fe937f39b834992f44838399","stratum":"8k","engine":"apr","engine_version":"apr 0.69.3 (96eb79f51)","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":21302.250685,"prompt_ms":null,"prompt_tps":null,"decode_tps":null,"input_tokens":6146,"output_tokens":241,"peak_rss_mb":8375.0859375,"verdict":{"state":"Pass"},"prompt_ids_sha256":"62861c84c3f9994a8f69de8848af2f7ea98e8e6de26bf95f24f760b6c5975b69"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"6e56cee326693c4915a7feef43eee6a597f75ff2e54f970e8f3197135869aed1","stratum":"8k","engine":"apr","engine_version":"apr 0.69.3 (96eb79f51)","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":26842.684061,"prompt_ms":null,"prompt_tps":null,"decode_tps":null,"input_tokens":7042,"output_tokens":298,"peak_rss_mb":8376.58203125,"verdict":{"state":"Pass"},"prompt_ids_sha256":"6383d5bff555ad33b2339b6120920cf9522aa807a1285f41b3efcface1525924"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"c81288783ae98e2b5a143cdfc4738ce6c817ae09e2b01a0979e18a0de5f90bb6","stratum":"8k","engine":"apr","engine_version":"apr 0.69.3 (96eb79f51)","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":19162.697772,"prompt_ms":null,"prompt_tps":null,"decode_tps":null,"input_tokens":7407,"output_tokens":169,"peak_rss_mb":8380.828125,"verdict":{"state":"Pass"},"prompt_ids_sha256":"3c18df6c64b92c1d238f0b099ce92e3e85ec2f7c4a8a113995df1d4036e878bd"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"0354fce0d010d6d69a1539589a7d6046495172ee6865f418b0a39c852edae583","stratum":"8k","engine":"apr","engine_version":"apr 0.69.3 (96eb79f51)","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":12186.932081,"prompt_ms":null,"prompt_tps":null,"decode_tps":null,"input_tokens":4043,"output_tokens":161,"peak_rss_mb":8380.828125,"verdict":{"state":"Pass"},"prompt_ids_sha256":"04371fb2a9cfff6184614b3943d1f0afebf2446e3ae21acce139cb914a34632b"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"b45b72bc1a202cb0504dfd27dd16f60af1ec09570fe31e7ecd927168837a02cf","stratum":"8k","engine":"apr","engine_version":"apr 0.69.3 (96eb79f51)","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":18674.904854,"prompt_ms":null,"prompt_tps":null,"decode_tps":null,"input_tokens":5997,"output_tokens":205,"peak_rss_mb":8380.828125,"verdict":{"state":"Pass"},"prompt_ids_sha256":"b4ac6d8b05c4c10cf4f5c3388011ec14c3513f17aefa6dc582e2dff4d2968405"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"b6dabc525c8486ab7bab78b2968c241327371b5c234ca1b8a87adb93bb8768ab","stratum":"8k","engine":"apr","engine_version":"apr 0.69.3 (96eb79f51)","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":18221.701447,"prompt_ms":null,"prompt_tps":null,"decode_tps":null,"input_tokens":4837,"output_tokens":243,"peak_rss_mb":8380.828125,"verdict":{"state":"Pass"},"prompt_ids_sha256":"f6a335774a9b2c8389c1c66b5157283cfdff4ebf3dead387979335582189e2cd"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"fd684a94b71725b926c4fb5be284b7c83518a537c466c552d7dbc8c9c1f64bec","stratum":"8k","engine":"apr","engine_version":"apr 0.69.3 (96eb79f51)","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":10812.074279,"prompt_ms":null,"prompt_tps":null,"decode_tps":null,"input_tokens":2286,"output_tokens":205,"peak_rss_mb":8380.859375,"verdict":{"state":"Pass"},"prompt_ids_sha256":"15033c21d4c89645a7f23d359d8c331c1cdae370c57dc97d3aa3ba5c4b935347"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"c5a7b5a62a110bae58ca259b02ea467d3dcdbde7ad3308a657e8028e63322e39","stratum":"8k","engine":"apr","engine_version":"apr 0.69.3 (96eb79f51)","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":15916.715553,"prompt_ms":null,"prompt_tps":null,"decode_tps":null,"input_tokens":4003,"output_tokens":234,"peak_rss_mb":8380.859375,"verdict":{"state":"Pass"},"prompt_ids_sha256":"6906e62e5d443f9ed949a4eb821a1ea194d8a3d072af1e52d8d0df23e1fe9f62"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"b71b016e3bf4e4aa82e2c46d8b1c2f137099cf683949b4c65e5900bad8de95ce","stratum":"8k","engine":"apr","engine_version":"apr 0.69.3 (96eb79f51)","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":8833.618397,"prompt_ms":null,"prompt_tps":null,"decode_tps":null,"input_tokens":2055,"output_tokens":164,"peak_rss_mb":8380.859375,"verdict":{"state":"Pass"},"prompt_ids_sha256":"24e56a79ebb675cd95f21cf12c693fbac4af6e2b009309f49d8f85d8b61164e5"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"611ff958e756f16b6143ffe425e3351e94a71034d0306f0e00ebd25696e66a31","stratum":"8k","engine":"apr","engine_version":"apr 0.69.3 (96eb79f51)","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":12874.658780000002,"prompt_ms":null,"prompt_tps":null,"decode_tps":null,"input_tokens":3565,"output_tokens":193,"peak_rss_mb":8380.859375,"verdict":{"state":"Pass"},"prompt_ids_sha256":"ac1e1de733551d5421552bcc206f001ac1b3efa35aa1a05a6ab785fd99ff12d8"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"1f97a6ca5af77396cc4febaacda124d2f993c18d3e3b96e3e0b28ff92f20610d","stratum":"8k","engine":"apr","engine_version":"apr 0.69.3 (96eb79f51)","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":30230.706902,"prompt_ms":null,"prompt_tps":null,"decode_tps":null,"input_tokens":7877,"output_tokens":309,"peak_rss_mb":8380.859375,"verdict":{"state":"Pass"},"prompt_ids_sha256":"c42007b3ae5fcfb14a8d8a6f5d25753226fc0910d9bc95d65e8b75cc5e4b05f1"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"3811dbd8d38716269bb87eca6a6039f05923f1c129d3d2b09659f4cff15dfc2e","stratum":"8k","engine":"apr","engine_version":"apr 0.69.3 (96eb79f51)","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":17913.444128,"prompt_ms":null,"prompt_tps":null,"decode_tps":null,"input_tokens":4570,"output_tokens":249,"peak_rss_mb":8380.859375,"verdict":{"state":"Pass"},"prompt_ids_sha256":"ea94bd384af8194fc5c38a4dbcdc751afcf318dc8834c3f4560913718900fb35"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"f65d88532cd6ca202557458343fb035dbe140810881382b7ce5d8c668afda27d","stratum":"8k","engine":"apr","engine_version":"apr 0.69.3 (96eb79f51)","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":11672.910702,"prompt_ms":null,"prompt_tps":null,"decode_tps":null,"input_tokens":3960,"output_tokens":155,"peak_rss_mb":8382.859375,"verdict":{"state":"Pass"},"prompt_ids_sha256":"d01aacddaa6f22bdfb64eca2e82ca55373391a16f47162da338cdbbad3508144"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"6bcf0a0fb95dfe5e90aa366595a006bc443f7a3d9de3c86ba176541bdbbc01cc","stratum":"16k","engine":"apr","engine_version":"apr 0.69.3 (96eb79f51)","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":44838.860304,"prompt_ms":null,"prompt_tps":null,"decode_tps":null,"input_tokens":15756,"output_tokens":239,"peak_rss_mb":8383.109375,"verdict":{"state":"Pass"},"prompt_ids_sha256":"e898c162c9392f2d3bd457479d776fac6602c99eba6b04a3d80e8cecc0c2cc92"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"c70e109e3c965254163267ebeee325ed9dfebe4479eb874c38a592a3488af34e","stratum":"16k","engine":"apr","engine_version":"apr 0.69.3 (96eb79f51)","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":33500.847411999996,"prompt_ms":null,"prompt_tps":null,"decode_tps":null,"input_tokens":11113,"output_tokens":247,"peak_rss_mb":8383.109375,"verdict":{"state":"Pass"},"prompt_ids_sha256":"6c1223de3236d024d3bf9a1a600bf90f1709debccf3ee61c2bd92011cb7c1968"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"9ec99261cd227d8f5e528bab7391b88aaafa4483233ccb0731a46b4f64bee8dc","stratum":"16k","engine":"apr","engine_version":"apr 0.69.3 (96eb79f51)","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":33981.768463,"prompt_ms":null,"prompt_tps":null,"decode_tps":null,"input_tokens":13574,"output_tokens":187,"peak_rss_mb":8383.109375,"verdict":{"state":"Pass"},"prompt_ids_sha256":"404ea4c6921bceb338fc1491d0dbe4aa8a2711122ca4646724bc17b6bcaaa069"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"9444f47e49f1779a103b611560b50ba89a8f20df19b976c9d4e06c2b01e103a5","stratum":"16k","engine":"apr","engine_version":"apr 0.69.3 (96eb79f51)","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":44944.307791,"prompt_ms":null,"prompt_tps":null,"decode_tps":null,"input_tokens":8882,"output_tokens":468,"peak_rss_mb":8383.1640625,"verdict":{"state":"Fail"},"prompt_ids_sha256":"8f31bc51da4630c662e6ec7e0c67c440f57a03802d09a5f5f286399b72e3c666"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"5e5b0935bd95c58dc95d3caebb22a4f41d0573018dff4ce1633bb7668afdd1a2","stratum":"16k","engine":"apr","engine_version":"apr 0.69.3 (96eb79f51)","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":30381.210511999998,"prompt_ms":null,"prompt_tps":null,"decode_tps":null,"input_tokens":14530,"output_tokens":132,"peak_rss_mb":8383.1640625,"verdict":{"state":"Pass"},"prompt_ids_sha256":"b191956bebc847f0567b83979f08d07982cecd6a68b6d78eba5b6a707c96236e"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"df963cc72999db8bd18665c41caca7b3cd058ec66923e7f5f3c161415d0cd4a2","stratum":"16k","engine":"apr","engine_version":"apr 0.69.3 (96eb79f51)","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":34979.078757,"prompt_ms":null,"prompt_tps":null,"decode_tps":null,"input_tokens":11625,"output_tokens":248,"peak_rss_mb":8383.1640625,"verdict":{"state":"Pass"},"prompt_ids_sha256":"9b005fd461f84a7451b7d950c5dac4626187c3e983e8ef186cb0ac942803b1d9"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"675cb632f8d8a2bb1d48f14a4d927d717d45d10dd3c0cacbe19f6e2cf94455a7","stratum":"16k","engine":"apr","engine_version":"apr 0.69.3 (96eb79f51)","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":28558.71424,"prompt_ms":null,"prompt_tps":null,"decode_tps":null,"input_tokens":8925,"output_tokens":254,"peak_rss_mb":8383.1640625,"verdict":{"state":"Pass"},"prompt_ids_sha256":"d017282343b77dda0b7ab50f3cb163775efea0802dee5ceeed7c6495c76cd8bf"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"c4cf0db8bffc8083c5691e577702ca73a932cb4cde56dd1c9dac360353ee38b1","stratum":"16k","engine":"apr","engine_version":"apr 0.69.3 (96eb79f51)","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":46667.984083999996,"prompt_ms":null,"prompt_tps":null,"decode_tps":null,"input_tokens":11187,"output_tokens":391,"peak_rss_mb":8383.1640625,"verdict":{"state":"Fail"},"prompt_ids_sha256":"dc38252b026854e36f129b97bd87960e52b6fcb44f4e545c4860459f962308d2"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"aa3e63ab8090ebf9be4f8d697c9f96d692a2763af91a2ad7bf96d1a26c384025","stratum":"16k","engine":"apr","engine_version":"apr 0.69.3 (96eb79f51)","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":39984.365044,"prompt_ms":null,"prompt_tps":null,"decode_tps":null,"input_tokens":9786,"output_tokens":367,"peak_rss_mb":8383.19140625,"verdict":{"state":"Pass"},"prompt_ids_sha256":"a2e6135129fe41dfbb6cefed737ccaecbb1acd94d176936ba19a8d4d00969e76"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"ea0520ed2859350cd1372043b3a06ff52b3e2f75fb0195b3d0521d1f23b3e7d6","stratum":"16k","engine":"apr","engine_version":"apr 0.69.3 (96eb79f51)","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":34473.237051,"prompt_ms":null,"prompt_tps":null,"decode_tps":null,"input_tokens":8748,"output_tokens":336,"peak_rss_mb":8383.19140625,"verdict":{"state":"Pass"},"prompt_ids_sha256":"c132e4a7a5ce9599d799aadfe7b680427fb4469c2031225692c05d86ceefb87c"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"54246f96c1844cc9afe81e68e68bd564d60308d58851d0af96c22aeb7059b962","stratum":"16k","engine":"apr","engine_version":"apr 0.69.3 (96eb79f51)","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":48024.868104,"prompt_ms":null,"prompt_tps":null,"decode_tps":null,"input_tokens":11000,"output_tokens":417,"peak_rss_mb":8383.19140625,"verdict":{"state":"Pass"},"prompt_ids_sha256":"51ccafe7aeaa0580788c3daf75e8b77c84449cb6f48369075c3a4e317d97ec7a"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"014e7869f90798986ecf651ca3a959bdf86f6369b6ce3d5ff71ecf0fd7cf7417","stratum":"16k","engine":"apr","engine_version":"apr 0.69.3 (96eb79f51)","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":34639.032099,"prompt_ms":null,"prompt_tps":null,"decode_tps":null,"input_tokens":12010,"output_tokens":233,"peak_rss_mb":8383.19140625,"verdict":{"state":"Pass"},"prompt_ids_sha256":"9bf4cad8197210b509c2cc01811950c93a723c31ef9acd1dd2d4eb9ced1a48e1"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"2cdf5aadfe826822cbeef1cfad68e8d2d21d6bb99bc84ba02b50ad8224102206","stratum":"16k","engine":"apr","engine_version":"apr 0.69.3 (96eb79f51)","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":41527.081095,"prompt_ms":null,"prompt_tps":null,"decode_tps":null,"input_tokens":9587,"output_tokens":394,"peak_rss_mb":8383.19140625,"verdict":{"state":"Pass"},"prompt_ids_sha256":"3163837c3c6ce35fc9144f2561444c6a631999d70734909762d7308610035870"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"6aa76be5cbd8ad7332c2acf93046bc20e866b9847dd276b88d4b66abe11949c8","stratum":"16k","engine":"apr","engine_version":"apr 0.69.3 (96eb79f51)","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":25890.394666,"prompt_ms":null,"prompt_tps":null,"decode_tps":null,"input_tokens":9165,"output_tokens":212,"peak_rss_mb":8383.19140625,"verdict":{"state":"Pass"},"prompt_ids_sha256":"5ef3f3955e3349beecc7b3ead0a588b47ee06838c7a297707417f2fd7c975fc4"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"720dc2dbf89b8f4f37b09af101e95d2b2c11697ff7a5d59d50f131286980e9ab","stratum":"16k","engine":"apr","engine_version":"apr 0.69.3 (96eb79f51)","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":22860.825531,"prompt_ms":null,"prompt_tps":null,"decode_tps":null,"input_tokens":12966,"output_tokens":87,"peak_rss_mb":8383.19140625,"verdict":{"state":"Pass"},"prompt_ids_sha256":"34405e806691b42d660adc2a2cfea20841a3de09c5c209d1aa96f85455b602c6"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"ebb26bc757178bfd19484adb04721a053df335e8059aed67cf8e11eb4d53cca1","stratum":"16k","engine":"apr","engine_version":"apr 0.69.3 (96eb79f51)","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":32379.675363,"prompt_ms":null,"prompt_tps":null,"decode_tps":null,"input_tokens":13686,"output_tokens":170,"peak_rss_mb":8383.30859375,"verdict":{"state":"Fail"},"prompt_ids_sha256":"2a4712db6c149f45728ae2b1ed0ed2762692034e6140466b328757593cef7e3d"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"1ee0546fe5e55ded3962edb4c5d8eb0632d49a0d3fcbcb6de9ec2d9936a567f2","stratum":"16k","engine":"apr","engine_version":"apr 0.69.3 (96eb79f51)","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":29743.221003,"prompt_ms":null,"prompt_tps":null,"decode_tps":null,"input_tokens":9153,"output_tokens":262,"peak_rss_mb":8383.30859375,"verdict":{"state":"Pass"},"prompt_ids_sha256":"4125241c2b414624ad34f56859fa91c135d8d1ffcc58e335cf7051aadd0439c5"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"1c2d4d2a2013dc86953f03eddb0ce4b67ac53e05931b7b18ff6d527d2c880640","stratum":"16k","engine":"apr","engine_version":"apr 0.69.3 (96eb79f51)","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":41558.543014999996,"prompt_ms":null,"prompt_tps":null,"decode_tps":null,"input_tokens":11164,"output_tokens":338,"peak_rss_mb":8383.30859375,"verdict":{"state":"Pass"},"prompt_ids_sha256":"e4d7241584af87981056a070fd2ec4e3eee56ef2016c62d5f7f71ab382b92dc9"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"f51557a52249a78f34a126d0646c5108cec49aa8bf9829fbfa487f75f82c7984","stratum":"16k","engine":"apr","engine_version":"apr 0.69.3 (96eb79f51)","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":28309.459481,"prompt_ms":null,"prompt_tps":null,"decode_tps":null,"input_tokens":8440,"output_tokens":265,"peak_rss_mb":8383.30859375,"verdict":{"state":"Pass"},"prompt_ids_sha256":"44aebbb6cf4173813f2621076bab1a2a912ba2f0a714ee96b98f86cfe61ab844"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"69a9d1096c603ec6d3706769831302fbf6a198a02c189f771a730704506c2941","stratum":"16k","engine":"apr","engine_version":"apr 0.69.3 (96eb79f51)","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":45229.14365,"prompt_ms":null,"prompt_tps":null,"decode_tps":null,"input_tokens":14384,"output_tokens":277,"peak_rss_mb":8383.30859375,"verdict":{"state":"Pass"},"prompt_ids_sha256":"dbd2749149ab437d0166e1de2f12680147b380a10ddb9b038ba1e6759f59d1a0"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"36528c9baa7523120f3683b655e9bd996e7ec070c288fa94b92aa17f033e18c4","stratum":"16k","engine":"apr","engine_version":"apr 0.69.3 (96eb79f51)","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":24674.173534,"prompt_ms":null,"prompt_tps":null,"decode_tps":null,"input_tokens":8567,"output_tokens":212,"peak_rss_mb":8383.30859375,"verdict":{"state":"Pass"},"prompt_ids_sha256":"036ef65a29008fd891473edcdf5366c0e021f70e465cd8fd70aee4f55c392c39"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"7d6b1ea94587f39adc61be730808d638df69dc46fb23c0292d839974de14c4aa","stratum":"16k","engine":"apr","engine_version":"apr 0.69.3 (96eb79f51)","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":30041.011543,"prompt_ms":null,"prompt_tps":null,"decode_tps":null,"input_tokens":8952,"output_tokens":273,"peak_rss_mb":8383.30859375,"verdict":{"state":"Pass"},"prompt_ids_sha256":"5640f5c62747622fad6026aa1b84af1b301adde703da627336351517c166ff00"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"25c07e0a68726a522357ec60ea143c91cea4227fe4139c3db6f0c52ac274bb8d","stratum":"16k","engine":"apr","engine_version":"apr 0.69.3 (96eb79f51)","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":36808.82939,"prompt_ms":null,"prompt_tps":null,"decode_tps":null,"input_tokens":14494,"output_tokens":193,"peak_rss_mb":8383.30859375,"verdict":{"state":"Pass"},"prompt_ids_sha256":"3e4d4daca07a12047b37ab6fa0e74f224e28c54cd87dad01f94457face129e0c"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"cc6f12cb2a5dc9e2cd0cff0eda9338245b06a7b29be77caf7c7ad415dd948bb4","stratum":"16k","engine":"apr","engine_version":"apr 0.69.3 (96eb79f51)","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":33386.224026,"prompt_ms":null,"prompt_tps":null,"decode_tps":null,"input_tokens":11326,"output_tokens":240,"peak_rss_mb":8383.30859375,"verdict":{"state":"Pass"},"prompt_ids_sha256":"7ef7e31a465dc80e0aa2ce7fa43085eba91d4ec00561981cadf511afc94587fc"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"88ef36ab5267cf425c39c87c20a1e697a4b612b9d36b17c39ca3b51cd24f9a9e","stratum":"16k","engine":"apr","engine_version":"apr 0.69.3 (96eb79f51)","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":33877.39127199999,"prompt_ms":null,"prompt_tps":null,"decode_tps":null,"input_tokens":11835,"output_tokens":230,"peak_rss_mb":8383.30859375,"verdict":{"state":"Pass"},"prompt_ids_sha256":"537725b34a695cc189b0d1338ef38a956b5d0250fa0c9c4bd1b5740c766399ce"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"43d42945260ed2fee02c5b9fda7974cf8f66fd65dff63de4e17ea8faf547e878","stratum":"16k","engine":"apr","engine_version":"apr 0.69.3 (96eb79f51)","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":31026.792871,"prompt_ms":null,"prompt_tps":null,"decode_tps":null,"input_tokens":9728,"output_tokens":259,"peak_rss_mb":8383.30859375,"verdict":{"state":"Pass"},"prompt_ids_sha256":"d9b5939d45f4b6cadbb25f9ce529b2543a1f3da2d3ab3decf3c7abe8c21fed68"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"059c0fd951a748e2ae74924c143d7c8e8d3c0c277645ba032d7f3fe28458b002","stratum":"16k","engine":"apr","engine_version":"apr 0.69.3 (96eb79f51)","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":50480.595897,"prompt_ms":null,"prompt_tps":null,"decode_tps":null,"input_tokens":15306,"output_tokens":304,"peak_rss_mb":8383.30859375,"verdict":{"state":"Fail"},"prompt_ids_sha256":"698b52cf76abfa36ca28d8f350ef8107c942ea080b91406c1ac66d4b38c385b0"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"0882349afbe288f0c5a3e2112311c022c01875743aabcc1d12afe9ce23093012","stratum":"16k","engine":"apr","engine_version":"apr 0.69.3 (96eb79f51)","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":36735.973987000005,"prompt_ms":null,"prompt_tps":null,"decode_tps":null,"input_tokens":13767,"output_tokens":211,"peak_rss_mb":8383.3359375,"verdict":{"state":"Pass"},"prompt_ids_sha256":"ff993db7ae7f149d4e3a1d6d7840460f0cd8a26c0eea77c35f6196552a40d4e5"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"bcef4d3206e948203a70d22840d2a5bbbd067648e038d4939ac7913c2d0d977d","stratum":"16k","engine":"apr","engine_version":"apr 0.69.3 (96eb79f51)","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":35401.557053,"prompt_ms":null,"prompt_tps":null,"decode_tps":null,"input_tokens":12076,"output_tokens":240,"peak_rss_mb":8398.3828125,"verdict":{"state":"Pass"},"prompt_ids_sha256":"39836f7486ad00c5fda9e448cf60e59f2860a0307cbf1871f34c46333a1c992c"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"f44f0554c5d06d6d9bd8a87e7c9adef7b5acc16f8dd84ee8990ad95cdb0d7a86","stratum":"16k","engine":"apr","engine_version":"apr 0.69.3 (96eb79f51)","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":39445.766116,"prompt_ms":null,"prompt_tps":null,"decode_tps":null,"input_tokens":11183,"output_tokens":313,"peak_rss_mb":8398.3828125,"verdict":{"state":"Pass"},"prompt_ids_sha256":"1f020914b5d4e264ef245a02a23947d4b1fd4c90b001bd77a260391e94464e4f"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"d49cd0c3fad3e8f5c978cbcb184e89f2cd360cdf37c1a97d192e6ca1e88aa974","stratum":"16k","engine":"apr","engine_version":"apr 0.69.3 (96eb79f51)","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":40759.71303,"prompt_ms":null,"prompt_tps":null,"decode_tps":null,"input_tokens":13764,"output_tokens":251,"peak_rss_mb":8398.3828125,"verdict":{"state":"Pass"},"prompt_ids_sha256":"2ad00e24d8988d50c0203eb5a7be9e633ca84a757788f60888e7586e09d1d1c8"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"6472e6c3bf01349b337af3cf2ddd2fc277b3af86eeb21bf5ca250923f3760dbb","stratum":"16k","engine":"apr","engine_version":"apr 0.69.3 (96eb79f51)","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":27357.703557,"prompt_ms":null,"prompt_tps":null,"decode_tps":null,"input_tokens":8812,"output_tokens":240,"peak_rss_mb":8399.0078125,"verdict":{"state":"Pass"},"prompt_ids_sha256":"27a05690c12bac021035e63d41f559f99635ca5e4f2488ee991d136a21e34989"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"5a466ceed2939ee8936626418aadce90a670d794ce8d1289af467fe01430718d","stratum":"16k","engine":"apr","engine_version":"apr 0.69.3 (96eb79f51)","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":37917.803554000006,"prompt_ms":null,"prompt_tps":null,"decode_tps":null,"input_tokens":14539,"output_tokens":204,"peak_rss_mb":8399.0078125,"verdict":{"state":"Pass"},"prompt_ids_sha256":"3f65be738f0bac2a6d1a61af3c99be128d0245394c8f562f5629e4378a22d17b"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"c8cd063621db62fd4dde1e4e2fe07d67562b8d073e89e6bd35c00b88b2c6d546","stratum":"16k","engine":"apr","engine_version":"apr 0.69.3 (96eb79f51)","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":31255.513393,"prompt_ms":null,"prompt_tps":null,"decode_tps":null,"input_tokens":13191,"output_tokens":170,"peak_rss_mb":8399.0078125,"verdict":{"state":"Pass"},"prompt_ids_sha256":"7d874c65f58ba21b178b10b378237380b9c35c43a22721cead3d56854f952312"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"1c7345312237ba7892931e444d0b663807feae59083cbc849d13f6e4748d2bd2","stratum":"16k","engine":"apr","engine_version":"apr 0.69.3 (96eb79f51)","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":31136.892627,"prompt_ms":null,"prompt_tps":null,"decode_tps":null,"input_tokens":14622,"output_tokens":137,"peak_rss_mb":8399.0078125,"verdict":{"state":"Pass"},"prompt_ids_sha256":"6dc5aeb4272351770a57d1673d2f6fceb604770bb1e63363de5dde21a07ecbae"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"0dc43a5ae0a19646ed73c5f0aa0495eda3a3b460efd42e35ab5a0d1fe2002192","stratum":"16k","engine":"apr","engine_version":"apr 0.69.3 (96eb79f51)","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":31569.848546,"prompt_ms":null,"prompt_tps":null,"decode_tps":null,"input_tokens":8742,"output_tokens":298,"peak_rss_mb":8399.0078125,"verdict":{"state":"Pass"},"prompt_ids_sha256":"5a5c9a9d4cc57194ef9db7c284d1261f917f9c29a19a53489aa80dac05004407"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"5189d6142784fae84f7d1634633cf93f49b3d2528950f68906f1c5ca9e8dfc5a","stratum":"16k","engine":"apr","engine_version":"apr 0.69.3 (96eb79f51)","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":51056.771236,"prompt_ms":null,"prompt_tps":null,"decode_tps":null,"input_tokens":16198,"output_tokens":280,"peak_rss_mb":8399.0078125,"verdict":{"state":"Pass"},"prompt_ids_sha256":"490004b669067449f5c7e62e6e309d0ee30ed55781d9c2b9d140e4ac5dbb38c8"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"28c93295ce9295f699d1ab17eecef2d2ef643f6ec8434694ee70fa20ad70c0b9","stratum":"16k","engine":"apr","engine_version":"apr 0.69.3 (96eb79f51)","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":34174.315403,"prompt_ms":null,"prompt_tps":null,"decode_tps":null,"input_tokens":13582,"output_tokens":189,"peak_rss_mb":8399.0078125,"verdict":{"state":"Pass"},"prompt_ids_sha256":"8c0ae16658161f3d1698b96183289aed7fd75bcda9877263efda1062664d7497"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"0f1bc13a198826247cef2fe67851805b3a7046ac9d291b02b47581866c6d896c","stratum":"16k","engine":"apr","engine_version":"apr 0.69.3 (96eb79f51)","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":26338.530811999997,"prompt_ms":null,"prompt_tps":null,"decode_tps":null,"input_tokens":10002,"output_tokens":192,"peak_rss_mb":8399.0078125,"verdict":{"state":"Pass"},"prompt_ids_sha256":"6a1e197e1c7efd9196871bec225a755aca2bd8ddfe47e0d46af2f3e0b14f1419"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"4e7c51716440d0055d37c6d118eca15cfb84381cb2984b437ddbf7920cee4651","stratum":"16k","engine":"apr","engine_version":"apr 0.69.3 (96eb79f51)","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":20258.183817999998,"prompt_ms":null,"prompt_tps":null,"decode_tps":null,"input_tokens":10235,"output_tokens":113,"peak_rss_mb":8399.0078125,"verdict":{"state":"Pass"},"prompt_ids_sha256":"f45b53c01e8955c24c49d0f24260bd34e4258513c7701ab5e462911006f37e99"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"d2107848512e9694b1e838662ca844b67f962cdc8d3826265b75f6b002d2982d","stratum":"16k","engine":"apr","engine_version":"apr 0.69.3 (96eb79f51)","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":34820.049242,"prompt_ms":null,"prompt_tps":null,"decode_tps":null,"input_tokens":8602,"output_tokens":347,"peak_rss_mb":8399.0078125,"verdict":{"state":"Pass"},"prompt_ids_sha256":"5b324a8a6e5df50a926268dd114f6374ee8d731ed147592868a109e4fa8cbc33"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"f0466d3c53c99fb39ebfe065673d3b9bdd70fdf30ee02898eb6e886201e8d4a3","stratum":"16k","engine":"apr","engine_version":"apr 0.69.3 (96eb79f51)","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":40645.628146,"prompt_ms":null,"prompt_tps":null,"decode_tps":null,"input_tokens":14169,"output_tokens":239,"peak_rss_mb":8399.0078125,"verdict":{"state":"Pass"},"prompt_ids_sha256":"8600d8498d868371fc947edc7db322d590a85502079ad419abb0df4a4727df72"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"f570aee8b67fbb453024af27fe758441e49aa16bc6b09147772a0d43a92115ec","stratum":"16k","engine":"apr","engine_version":"apr 0.69.3 (96eb79f51)","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":30070.964735,"prompt_ms":null,"prompt_tps":null,"decode_tps":null,"input_tokens":12302,"output_tokens":176,"peak_rss_mb":8399.0078125,"verdict":{"state":"Pass"},"prompt_ids_sha256":"8b2497a464c07d512e5a9659dcb4f07a47d97fc06c111ae926ed313355ce34d3"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"409826d3610e79d93d92ba45ad55d943307796f6b4218ad327e872390681ef26","stratum":"16k","engine":"apr","engine_version":"apr 0.69.3 (96eb79f51)","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":51801.515221,"prompt_ms":null,"prompt_tps":null,"decode_tps":null,"input_tokens":16267,"output_tokens":284,"peak_rss_mb":8399.05078125,"verdict":{"state":"Pass"},"prompt_ids_sha256":"cc546fdfce8b6b26c4668049fff094491f2f8bee3cf437f37a5eb735bdb27740"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"b53f7c2d4a456e341e0f1e013eede2c53dbe5230bb2ef59c8437a9dbd9b1126f","stratum":"16k","engine":"apr","engine_version":"apr 0.69.3 (96eb79f51)","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":37696.52263,"prompt_ms":null,"prompt_tps":null,"decode_tps":null,"input_tokens":13355,"output_tokens":232,"peak_rss_mb":8399.07421875,"verdict":{"state":"Pass"},"prompt_ids_sha256":"58c397b59aee26d82ba43edafd279603a863fc48170cd0fb176fdf91ffc3a98f"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"fd326fe7d4bacc97fcf64369c18bd23e1a718b755ec9d720b6010b7bea96f970","stratum":"16k","engine":"apr","engine_version":"apr 0.69.3 (96eb79f51)","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":25989.869208999997,"prompt_ms":null,"prompt_tps":null,"decode_tps":null,"input_tokens":8404,"output_tokens":234,"peak_rss_mb":8399.07421875,"verdict":{"state":"Pass"},"prompt_ids_sha256":"450e403e1bc170e1470b9b1d2a97c64aa904b0167319be75d1fa671f5a99f4dd"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"fa7720559e61059c92e06f6156fc0da7ae0a6367cd814f36465e85f7d04800e7","stratum":"16k","engine":"apr","engine_version":"apr 0.69.3 (96eb79f51)","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":49171.408309,"prompt_ms":null,"prompt_tps":null,"decode_tps":null,"input_tokens":15833,"output_tokens":277,"peak_rss_mb":8399.11328125,"verdict":{"state":"Pass"},"prompt_ids_sha256":"7d9efdebce284cc4c5ce1d3a24fd62794d1ac2228510aa0c228e9ee283ab2cd9"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"f8f4e9f89c28ef4f440267322b74bbf68eed96ea240d6a883bec0d50f2dbe576","stratum":"16k","engine":"apr","engine_version":"apr 0.69.3 (96eb79f51)","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":31093.919871000002,"prompt_ms":null,"prompt_tps":null,"decode_tps":null,"input_tokens":9549,"output_tokens":265,"peak_rss_mb":8399.11328125,"verdict":{"state":"Pass"},"prompt_ids_sha256":"5d5cfe1a4a40e808aca7241c0e6a96d796456776e3fc689c00288b819bcc28cd"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"ca612af4633a006928c0400dcf7a90a769c47290edc85241d3ba156583b2f6b7","stratum":"16k","engine":"apr","engine_version":"apr 0.69.3 (96eb79f51)","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":28871.333717999998,"prompt_ms":null,"prompt_tps":null,"decode_tps":null,"input_tokens":9256,"output_tokens":248,"peak_rss_mb":8399.11328125,"verdict":{"state":"Pass"},"prompt_ids_sha256":"3a050649a54bd19a4c5b2fc11ca233aacbd3f9ea410506f99d0d65a893a023cd"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"f80dddb74b021666f2f30078ee85f6a231fb529e1ed67f051a9b662ffb17b4e6","stratum":"16k","engine":"apr","engine_version":"apr 0.69.3 (96eb79f51)","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":25436.038104000003,"prompt_ms":null,"prompt_tps":null,"decode_tps":null,"input_tokens":9254,"output_tokens":203,"peak_rss_mb":8399.11328125,"verdict":{"state":"Pass"},"prompt_ids_sha256":"29ad40656fdbdd17ccccd5a79690a1c5856bd2c5e066028c0776ebd8fcc59873"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"fa49d36e7eb9eed429d0d549464654396d8fe8fc5a0be7b4e667bddb37922c36","stratum":"32k","engine":"apr","engine_version":"apr 0.69.3 (96eb79f51)","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":73001.855527,"prompt_ms":null,"prompt_tps":null,"decode_tps":null,"input_tokens":24683,"output_tokens":261,"peak_rss_mb":8399.2421875,"verdict":{"state":"Pass"},"prompt_ids_sha256":"10163b836e6b21a8d1943f429085e148470b19b5c857f7822b5661509f4d11d9"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"3b51ad3f76c868adaa04840a77eed635ade302cec8c85bf0d3795526ab237fa7","stratum":"32k","engine":"apr","engine_version":"apr 0.69.3 (96eb79f51)","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":47615.421244,"prompt_ms":null,"prompt_tps":null,"decode_tps":null,"input_tokens":20209,"output_tokens":171,"peak_rss_mb":8399.36328125,"verdict":{"state":"Pass"},"prompt_ids_sha256":"2d0742ae98d285c41c4c84ef725a4fdd40c2b06112ad0f5f6709cd4fd9da985b"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"f8815530d0c7dc219ea3341c95432e757c3ab7fb0308bd2146f2d90b840f726b","stratum":"32k","engine":"apr","engine_version":"apr 0.69.3 (96eb79f51)","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":36512.701483,"prompt_ms":null,"prompt_tps":null,"decode_tps":null,"input_tokens":16914,"output_tokens":143,"peak_rss_mb":8401.36328125,"verdict":{"state":"Fail"},"prompt_ids_sha256":"dd1693371bba23714bd583154ef11f1d6a58342d9feb0fbc8b3f89281a3c430a"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"717524cb0e8f36fac7a56af480fe9ecfe30ed9e797cefb9908a1c663d247b0c7","stratum":"32k","engine":"apr","engine_version":"apr 0.69.3 (96eb79f51)","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":56290.756118,"prompt_ms":null,"prompt_tps":null,"decode_tps":null,"input_tokens":19114,"output_tokens":260,"peak_rss_mb":8401.36328125,"verdict":{"state":"Pass"},"prompt_ids_sha256":"fe2fbf0e6c29ac51fe639ae75e221ab52400551682caf9dcc5f560088f5991f0"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"03c15ba04bbacf6b92b3ab9bd18cdca26071fbb4a09dc4de9bef2ad078ba580f","stratum":"32k","engine":"apr","engine_version":"apr 0.69.3 (96eb79f51)","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":58702.960523,"prompt_ms":null,"prompt_tps":null,"decode_tps":null,"input_tokens":22097,"output_tokens":216,"peak_rss_mb":8401.36328125,"verdict":{"state":"Pass"},"prompt_ids_sha256":"1d1e1149ebcc7a12c4464cd97630fc968780b0fa6703886e8550b6cd7ab8f27c"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"351ae6cc9897a62bc51dc629ead6161e8242c49a7cfbeabdd8f05e9c0c3cdaea","stratum":"32k","engine":"apr","engine_version":"apr 0.69.3 (96eb79f51)","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":51408.022142999995,"prompt_ms":null,"prompt_tps":null,"decode_tps":null,"input_tokens":19560,"output_tokens":214,"peak_rss_mb":8401.36328125,"verdict":{"state":"Fail"},"prompt_ids_sha256":"d4c0dfa77150a8a80c4404832981c8f2c1f4eca29ffbec592e3d21b0ba30e535"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"7bb9e2edd88d122d0553e7776b5ac605cc7da369d4410dba027a27df9290f060","stratum":"32k","engine":"apr","engine_version":"apr 0.69.3 (96eb79f51)","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":58096.046445,"prompt_ms":null,"prompt_tps":null,"decode_tps":null,"input_tokens":21284,"output_tokens":229,"peak_rss_mb":8401.36328125,"verdict":{"state":"Pass"},"prompt_ids_sha256":"162822cb75db0d188fb71a133b6480a9768b7b1582914953aaf3b0494125830a"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"d667fffd336a3dd90c6245ba9527ce0fb0448c554d576a089e347230a6ecb2c0","stratum":"32k","engine":"apr","engine_version":"apr 0.69.3 (96eb79f51)","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":43539.977094999995,"prompt_ms":null,"prompt_tps":null,"decode_tps":null,"input_tokens":19186,"output_tokens":160,"peak_rss_mb":8401.36328125,"verdict":{"state":"Pass"},"prompt_ids_sha256":"544ecff0b9504efcc6965bfe96d614b95a0c9a33f99d1f021276789c0ac5f294"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"38a1dbfd07591ad9a6e4b27e7916e661614e6cd7231082e0e4e2601297e5fe7f","stratum":"32k","engine":"apr","engine_version":"apr 0.69.3 (96eb79f51)","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":48661.36494,"prompt_ms":null,"prompt_tps":null,"decode_tps":null,"input_tokens":19529,"output_tokens":194,"peak_rss_mb":8401.36328125,"verdict":{"state":"Pass"},"prompt_ids_sha256":"3fa656a12db52afb3add603a41a8269708e92167a90663f8d3cc7548b477f08c"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"3d8a6474dcfe17a53ec35178358b968c69405032d23e581b7a700fe39982960a","stratum":"32k","engine":"apr","engine_version":"apr 0.69.3 (96eb79f51)","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":47176.451467,"prompt_ms":null,"prompt_tps":null,"decode_tps":null,"input_tokens":19038,"output_tokens":191,"peak_rss_mb":8401.36328125,"verdict":{"state":"Pass"},"prompt_ids_sha256":"64b5124f7d02001c4cf5434c7f64ad4222b626650acc205dded43af97165d1eb"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"bf2811a3edb86d54d69166c93ec73ebcf6ff64b5556d9d7d9f090726e8fee9a2","stratum":"32k","engine":"apr","engine_version":"apr 0.69.3 (96eb79f51)","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":58031.783367,"prompt_ms":null,"prompt_tps":null,"decode_tps":null,"input_tokens":21745,"output_tokens":218,"peak_rss_mb":8401.36328125,"verdict":{"state":"Pass"},"prompt_ids_sha256":"ddea598580c58fd52253867387479e86617b6fc8bef87e6b9a6bf70c78966051"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"b515045eff1517e48974319c1c62e87bc8ef658b739d6e365b934ee04f6667f0","stratum":"32k","engine":"apr","engine_version":"apr 0.69.3 (96eb79f51)","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":81490.744491,"prompt_ms":null,"prompt_tps":null,"decode_tps":null,"input_tokens":29883,"output_tokens":223,"peak_rss_mb":8401.53515625,"verdict":{"state":"Pass"},"prompt_ids_sha256":"cce8cd881a53898e5c7a62a4f4ca518a141c8988a66f1f84e53c4982ca7f1560"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"8f82c371a972bb3e1d51e43968dccb02cfdaa9e94eaabb04b2b6202addef14fa","stratum":"32k","engine":"apr","engine_version":"apr 0.69.3 (96eb79f51)","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":55679.067897,"prompt_ms":null,"prompt_tps":null,"decode_tps":null,"input_tokens":24091,"output_tokens":162,"peak_rss_mb":8401.6640625,"verdict":{"state":"Pass"},"prompt_ids_sha256":"4db639b147d20927495dfc8f015d002c34a1a8dbb361f9a395d0f540f44c1f25"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"304c3c29ecc274958e1042ea4f3e6fb4108bdc048767ad229c669ae8025e23f7","stratum":"32k","engine":"apr","engine_version":"apr 0.69.3 (96eb79f51)","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":50135.798577,"prompt_ms":null,"prompt_tps":null,"decode_tps":null,"input_tokens":19208,"output_tokens":210,"peak_rss_mb":8401.6640625,"verdict":{"state":"Pass"},"prompt_ids_sha256":"7d297d57bb08bd16d5a0bfd093357da4f17a4f2954bc5aebe4e292401877ef3a"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"c9d4c24c8671432ccf62fc9a693dd62fc3430da536f4b20f53abb64ba65e010b","stratum":"32k","engine":"apr","engine_version":"apr 0.69.3 (96eb79f51)","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":45857.954571999995,"prompt_ms":null,"prompt_tps":null,"decode_tps":null,"input_tokens":18501,"output_tokens":189,"peak_rss_mb":8401.6640625,"verdict":{"state":"Pass"},"prompt_ids_sha256":"2b348c36ba02c98acdd3b0264d0bf7c6abc97c4aa1e0b5d5a57e9a76895b8ff7"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"7173ebdf8174e7e3ac987fc47c136b866b83cdf52d03a1c1fce2fb4ed8192ded","stratum":"32k","engine":"apr","engine_version":"apr 0.69.3 (96eb79f51)","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":93789.858295,"prompt_ms":null,"prompt_tps":null,"decode_tps":null,"input_tokens":31444,"output_tokens":260,"peak_rss_mb":8404.5,"verdict":{"state":"Pass"},"prompt_ids_sha256":"0f80c00a890f530a76c0f115c409d1e0b9ef5735899f43cac2e7a695fba889c4"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"0f963ad6b87e0d3342e5334a4115c16eaf148f32c1e0adc968706c77b5191039","stratum":"32k","engine":"apr","engine_version":"apr 0.69.3 (96eb79f51)","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":82555.173866,"prompt_ms":null,"prompt_tps":null,"decode_tps":null,"input_tokens":28220,"output_tokens":254,"peak_rss_mb":8404.5,"verdict":{"state":"Pass"},"prompt_ids_sha256":"4c4b6ae7e6b58fe247abcafa33c758ba870176b098d102db06af0ac0686fd4b0"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"74f801a07d6f9f7366f719b4b2302c99b8c74f8eb9f099cb0088470f7ed180e1","stratum":"32k","engine":"apr","engine_version":"apr 0.69.3 (96eb79f51)","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":46000.144319,"prompt_ms":null,"prompt_tps":null,"decode_tps":null,"input_tokens":22764,"output_tokens":118,"peak_rss_mb":8404.5,"verdict":{"state":"Pass"},"prompt_ids_sha256":"70c486262966bf7809098d46926cd98aaf3f6670ceb98cb3f34cb817ffbf760e"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"430c7c0f69397873ade236dd3f5db7663f2f1fb4f40eb879109629b59e0b3ce3","stratum":"32k","engine":"apr","engine_version":"apr 0.69.3 (96eb79f51)","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":67221.564029,"prompt_ms":null,"prompt_tps":null,"decode_tps":null,"input_tokens":21853,"output_tokens":280,"peak_rss_mb":8404.5,"verdict":{"state":"Pass"},"prompt_ids_sha256":"d344c43778457ceeaf9fcafd3565a371a4836ae8c8fd3ac08de51d30e2f26f57"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"8003dd98e1763cde69c838d89b0bab41aaa2622c580b9e3de39b8b3c1e182f85","stratum":"32k","engine":"apr","engine_version":"apr 0.69.3 (96eb79f51)","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":54534.199015,"prompt_ms":null,"prompt_tps":null,"decode_tps":null,"input_tokens":24959,"output_tokens":141,"peak_rss_mb":8404.5,"verdict":{"state":"Pass"},"prompt_ids_sha256":"83323d291c7ce06a7d2db0bdee88fcedd25ebe222a3c6653fc9c2257e2379150"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"b931d437202ef5adb659042b0e7255fd1754602c9d52039ef36d831ad232674b","stratum":"32k","engine":"apr","engine_version":"apr 0.69.3 (96eb79f51)","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":50875.094098,"prompt_ms":null,"prompt_tps":null,"decode_tps":null,"input_tokens":19708,"output_tokens":206,"peak_rss_mb":8404.5,"verdict":{"state":"Pass"},"prompt_ids_sha256":"0adbf65326e98c30c97a86ac1cd456158dddac02a101d1cc351a815782a5cdba"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"084e31008f5a675e5153246d96a5142e0adc8215e66f598266bbb76a10be6db2","stratum":"32k","engine":"apr","engine_version":"apr 0.69.3 (96eb79f51)","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":92554.29305,"prompt_ms":null,"prompt_tps":null,"decode_tps":null,"input_tokens":30183,"output_tokens":275,"peak_rss_mb":8404.5,"verdict":{"state":"Pass"},"prompt_ids_sha256":"008c43328b72985f0930804bf4dba749371420359c750afa424dd5e6d3047e5a"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"14b3251ddf800d7bdfdb7a775c2281eba042c909e8a46700689825014ded27de","stratum":"32k","engine":"apr","engine_version":"apr 0.69.3 (96eb79f51)","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":67974.387392,"prompt_ms":null,"prompt_tps":null,"decode_tps":null,"input_tokens":17967,"output_tokens":380,"peak_rss_mb":8404.5,"verdict":{"state":"Pass"},"prompt_ids_sha256":"cc964f6596ba0bf32e87225ab3a6e88f89eac8e4ba6bc237cab69ecee8f5ee60"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"ca55757f48d1d3f049d77fd86f25cc77c581af66809d307a89d2a5b7bb14d278","stratum":"32k","engine":"apr","engine_version":"apr 0.69.3 (96eb79f51)","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":84448.50612199999,"prompt_ms":null,"prompt_tps":null,"decode_tps":null,"input_tokens":20902,"output_tokens":425,"peak_rss_mb":8404.5,"verdict":{"state":"Fail"},"prompt_ids_sha256":"36511a899244f7383295611d61080169d7b333ce858c223384ee57588db0cc22"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"5bc59b96abf0b69ee35dd697a3d38ea685e54eb0cd1544a880e08b0f64c3905f","stratum":"32k","engine":"apr","engine_version":"apr 0.69.3 (96eb79f51)","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":51151.417548,"prompt_ms":null,"prompt_tps":null,"decode_tps":null,"input_tokens":18673,"output_tokens":227,"peak_rss_mb":8404.5,"verdict":{"state":"Pass"},"prompt_ids_sha256":"01878ef6e12e6090d8bf4d912ac05afd77064731e6d2c22d029d8e95916d2ef8"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"4608b8ddaa9de754634659da66dee0e3d59c7097dfbebe7ef8ff664f7a8e2a3c","stratum":"32k","engine":"apr","engine_version":"apr 0.69.3 (96eb79f51)","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":71535.11113599999,"prompt_ms":null,"prompt_tps":null,"decode_tps":null,"input_tokens":20131,"output_tokens":351,"peak_rss_mb":8404.5,"verdict":{"state":"Pass"},"prompt_ids_sha256":"ea75c65f1593e8994329bf6520bd50cfd827175845144b2decd4a39d6a816003"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"dc10ee3c8ec1879df8935451288b060dcba94ff3db9fb2d0e714e938b65c15ca","stratum":"32k","engine":"apr","engine_version":"apr 0.69.3 (96eb79f51)","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":59489.092599999996,"prompt_ms":null,"prompt_tps":null,"decode_tps":null,"input_tokens":26358,"output_tokens":151,"peak_rss_mb":8404.5,"verdict":{"state":"Pass"},"prompt_ids_sha256":"60fa312565da30886df98a128049dec4b80d7fbab7adb34688ce8d3b13131f87"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"841dd09f216b308d2acb091c53e79c3c22c6dc9c70d99ddd5aaa007e65fb8f0f","stratum":"32k","engine":"apr","engine_version":"apr 0.69.3 (96eb79f51)","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":41560.087845,"prompt_ms":null,"prompt_tps":null,"decode_tps":null,"input_tokens":18319,"output_tokens":160,"peak_rss_mb":8404.5,"verdict":{"state":"Pass"},"prompt_ids_sha256":"693837d52c6a4861282f0d455277e48fac6d7e207bdaa4e7a172c2acf8726055"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"24bc258f12dfbb5600afabdff2efa2edeaf9efb4c7e13dae0bb627e6c0add649","stratum":"32k","engine":"apr","engine_version":"apr 0.69.3 (96eb79f51)","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":68300.386351,"prompt_ms":null,"prompt_tps":null,"decode_tps":null,"input_tokens":22829,"output_tokens":268,"peak_rss_mb":8404.5,"verdict":{"state":"Pass"},"prompt_ids_sha256":"0662c740da8b7967cc8500d02856b1bb61b045eb9df620e43bddc27d9f806dfc"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"78a121fb8ef6ffe62c572f12c3113443d88b19491cd0db0981294e8ed29d38bb","stratum":"32k","engine":"apr","engine_version":"apr 0.69.3 (96eb79f51)","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":45223.631675,"prompt_ms":null,"prompt_tps":null,"decode_tps":null,"input_tokens":18640,"output_tokens":182,"peak_rss_mb":8404.5,"verdict":{"state":"Pass"},"prompt_ids_sha256":"ba43eb4396f1eb5beb8d13c853dad39cd3e1cf8d77a0732070175d726fd6b01c"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"c0b75e2f0a6bcd1a3341d0cbb4e96aa16449934b09e961f790c377d972340d08","stratum":"32k","engine":"apr","engine_version":"apr 0.69.3 (96eb79f51)","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":54643.131143,"prompt_ms":null,"prompt_tps":null,"decode_tps":null,"input_tokens":21347,"output_tokens":202,"peak_rss_mb":8404.5,"verdict":{"state":"Pass"},"prompt_ids_sha256":"957b6f511e3f28dc0e7f71f4f420374158278789df3325c0cb389818c1e26247"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"21a87ab49fd3587febc4c89bb82872bb6967bf0a9492abc590112d5f097f1bd5","stratum":"32k","engine":"apr","engine_version":"apr 0.69.3 (96eb79f51)","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":56749.379315,"prompt_ms":null,"prompt_tps":null,"decode_tps":null,"input_tokens":23480,"output_tokens":179,"peak_rss_mb":8404.5,"verdict":{"state":"Pass"},"prompt_ids_sha256":"7b8e789020dabaf591806d9d9bf0c6232547b067051bbd4cb26f58de384c2ce8"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"0534621eb0e2215a3c13b22ebf197a2a472f4306bf24f0e380154a527dbc1463","stratum":"32k","engine":"apr","engine_version":"apr 0.69.3 (96eb79f51)","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":68353.485592,"prompt_ms":null,"prompt_tps":null,"decode_tps":null,"input_tokens":22862,"output_tokens":255,"peak_rss_mb":8404.5,"verdict":{"state":"Pass"},"prompt_ids_sha256":"e49800ebb35f74199fe8a55797fc6e015fb00fd0fff65e78dd6f91889e9f5ff3"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"add564d1303cf5966124647b302705301dcd4528a1caad315f4d86aa3485e61a","stratum":"32k","engine":"apr","engine_version":"apr 0.69.3 (96eb79f51)","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":63886.697718,"prompt_ms":null,"prompt_tps":null,"decode_tps":null,"input_tokens":23436,"output_tokens":199,"peak_rss_mb":8404.5,"verdict":{"state":"Pass"},"prompt_ids_sha256":"d501682a3b448ff13e405df0b3c77e50f0da19852d59188afedbf391ff330a0c"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"756c7e34ba781b2a93fb5ba02710dfea9f422786e59c39eb6e2b38e11ad4bf4a","stratum":"32k","engine":"apr","engine_version":"apr 0.69.3 (96eb79f51)","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":55976.888736,"prompt_ms":null,"prompt_tps":null,"decode_tps":null,"input_tokens":18829,"output_tokens":248,"peak_rss_mb":8404.5,"verdict":{"state":"Pass"},"prompt_ids_sha256":"d6675b3bdd9d86978cd61242f476ccbc7623c2446ae06132244a6f42efbc7255"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"40d8b0a4203d03441d9dcc41d1b76704f65ec7ea5ebd165cac2457a566b70162","stratum":"32k","engine":"apr","engine_version":"apr 0.69.3 (96eb79f51)","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":55498.147533,"prompt_ms":null,"prompt_tps":null,"decode_tps":null,"input_tokens":17833,"output_tokens":282,"peak_rss_mb":8404.5,"verdict":{"state":"Pass"},"prompt_ids_sha256":"0200f2f9a1370d6f38cae62051efcd98e40658a6fc1ec422e8b37f6701ce5412"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"3db9e737af60c9d2537ac8d8b93aeaffc7b6cc68294a49978e5c870ffb36eb4f","stratum":"32k","engine":"apr","engine_version":"apr 0.69.3 (96eb79f51)","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":48721.015158999995,"prompt_ms":null,"prompt_tps":null,"decode_tps":null,"input_tokens":19725,"output_tokens":189,"peak_rss_mb":8419.26953125,"verdict":{"state":"Pass"},"prompt_ids_sha256":"fffd8b1255cac61466278bf8094821380a784a6aefa5c63410027ecce55429ed"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"c354c595f4a8ea1b09659b55a5ce312a495e4eb6400c77183cd34cca3cb8327b","stratum":"32k","engine":"apr","engine_version":"apr 0.69.3 (96eb79f51)","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":49256.779541,"prompt_ms":null,"prompt_tps":null,"decode_tps":null,"input_tokens":18716,"output_tokens":211,"peak_rss_mb":8419.26953125,"verdict":{"state":"Pass"},"prompt_ids_sha256":"05dadf955da5b93f28985b714df19e2ef0206facca769a7ffa72479563a4975a"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"c42dfb333a42fbec1745318326627fa05d41440da37c5d2d57304daf1e4bb3b6","stratum":"32k","engine":"apr","engine_version":"apr 0.69.3 (96eb79f51)","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":47990.689746000004,"prompt_ms":null,"prompt_tps":null,"decode_tps":null,"input_tokens":20073,"output_tokens":177,"peak_rss_mb":8419.26953125,"verdict":{"state":"Pass"},"prompt_ids_sha256":"80097ce1ced3c4a30308e27f670ccbf58abb86ff4f9385521228aac8db4ed868"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"8136fdc9902571f25cbf93bf14de71435aea774d50ab0fb50ba2196f422389af","stratum":"32k","engine":"apr","engine_version":"apr 0.69.3 (96eb79f51)","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":58805.414674,"prompt_ms":null,"prompt_tps":null,"decode_tps":null,"input_tokens":20634,"output_tokens":245,"peak_rss_mb":8419.26953125,"verdict":{"state":"Pass"},"prompt_ids_sha256":"a3c875c6ba4ab5c35526ae9bc83a120f426b85c20fe3a644c37dd3cb99b85be3"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"a267f9989c8533bce8be2d300c85a8509c45a80b1fe16836591ae91341138b2a","stratum":"32k","engine":"apr","engine_version":"apr 0.69.3 (96eb79f51)","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":86297.605384,"prompt_ms":null,"prompt_tps":null,"decode_tps":null,"input_tokens":29270,"output_tokens":258,"peak_rss_mb":8419.2890625,"verdict":{"state":"Pass"},"prompt_ids_sha256":"52b26517747f4815b31f168051fd64d5054550a410ec132c26ad9f221325ae4b"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"60d2f019ddec1435b5b1230d70553a3833405b30dae55c8dbc8f6bcc1f0f6487","stratum":"32k","engine":"apr","engine_version":"apr 0.69.3 (96eb79f51)","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":49869.205497,"prompt_ms":null,"prompt_tps":null,"decode_tps":null,"input_tokens":19055,"output_tokens":211,"peak_rss_mb":8419.2890625,"verdict":{"state":"Pass"},"prompt_ids_sha256":"926956c06005c8e58a90a81567de45b1b9f4cf512f4714410a8c2bdff97eaf09"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"4be2763e70a83f3b538045b385bf920c45f0ab971c6641e4573f745b32673998","stratum":"32k","engine":"apr","engine_version":"apr 0.69.3 (96eb79f51)","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":64276.566084,"prompt_ms":null,"prompt_tps":null,"decode_tps":null,"input_tokens":17096,"output_tokens":374,"peak_rss_mb":8419.2890625,"verdict":{"state":"Fail"},"prompt_ids_sha256":"2209238bca60b627f187342a3fa75e2cc850f42325091d4f4918c6c267a9a260"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"4a66cbc27f5a915eaa9fed54adca1621f3d27e77b95dee234328d0114d722209","stratum":"32k","engine":"apr","engine_version":"apr 0.69.3 (96eb79f51)","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":84005.07043499999,"prompt_ms":null,"prompt_tps":null,"decode_tps":null,"input_tokens":27135,"output_tokens":275,"peak_rss_mb":8419.2890625,"verdict":{"state":"Pass"},"prompt_ids_sha256":"7d7c836939bcc09cc2d8280e28b6a392d04a2145d05417cf9d72dd860d4e04fe"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"f552e516d8a086ee988e3c5488a31467a2bb6b35b277aecfb98ad15c96d6d3da","stratum":"32k","engine":"apr","engine_version":"apr 0.69.3 (96eb79f51)","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":49046.977441999996,"prompt_ms":null,"prompt_tps":null,"decode_tps":null,"input_tokens":19103,"output_tokens":202,"peak_rss_mb":8419.2890625,"verdict":{"state":"Pass"},"prompt_ids_sha256":"c2c806c56c4d144111f554ef8f5f933837eea605a07ee70ebb09c16490c92ca1"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"5c2a849b4a443f183000bbd6996f84b95d3df274e267f4a0b9be650a53080e75","stratum":"32k","engine":"apr","engine_version":"apr 0.69.3 (96eb79f51)","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":78292.350555,"prompt_ms":null,"prompt_tps":null,"decode_tps":null,"input_tokens":23692,"output_tokens":313,"peak_rss_mb":8419.2890625,"verdict":{"state":"Pass"},"prompt_ids_sha256":"000dbf6068631c787dc66205d5a9ad463a2c755eb612acc7ece318bff11f5bfc"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"a2b4707bfaf04f8599408ccfb45a44703a000f71a094a68ba1759061b2a43672","stratum":"32k","engine":"apr","engine_version":"apr 0.69.3 (96eb79f51)","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":41655.888149,"prompt_ms":null,"prompt_tps":null,"decode_tps":null,"input_tokens":19542,"output_tokens":137,"peak_rss_mb":8419.2890625,"verdict":{"state":"Pass"},"prompt_ids_sha256":"fa651c50ada370ebabdf777b8b3eac38d443ada3b2e4762702cf325ab1421721"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"3ab4ac140f83056d2501ee334f17e3ee310e217094d568b4fec0c75a1a30dce9","stratum":"32k","engine":"apr","engine_version":"apr 0.69.3 (96eb79f51)","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":69357.750129,"prompt_ms":null,"prompt_tps":null,"decode_tps":null,"input_tokens":22476,"output_tokens":273,"peak_rss_mb":8419.2890625,"verdict":{"state":"Pass"},"prompt_ids_sha256":"a4b4fb3bd5fe96a03ed1ee01a571c200b6a1ab2e777a7638743f0a24ae89a56b"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"bf6a52843bdf826313ab90adf25c9d7a253496a9a3c5a46c08ef62ef5249470d","stratum":"32k","engine":"apr","engine_version":"apr 0.69.3 (96eb79f51)","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":58598.828861,"prompt_ms":null,"prompt_tps":null,"decode_tps":null,"input_tokens":21254,"output_tokens":221,"peak_rss_mb":8419.2890625,"verdict":{"state":"Pass"},"prompt_ids_sha256":"29750ae08524616eb9b008167fc5e1577e4b6f505976a9986f03dac0a1eb099b"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"4c7ad86de1bf01701febc9297c4271016e389826ac06ac0fdc766c598c4ba789","stratum":"32k","engine":"apr","engine_version":"apr 0.69.3 (96eb79f51)","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":51219.777793,"prompt_ms":null,"prompt_tps":null,"decode_tps":null,"input_tokens":19598,"output_tokens":200,"peak_rss_mb":8419.2890625,"verdict":{"state":"Pass"},"prompt_ids_sha256":"36e696fa27b6f800c61cc29b6ce76dc253720bc7ae8a736483bb1a628c4bf41c"} diff --git a/evidence/prm-001/s1-v2/96eb79f51/rows-llama_cpp.jsonl b/evidence/prm-001/s1-v2/96eb79f51/rows-llama_cpp.jsonl new file mode 100644 index 0000000000..e96d6b1b7d --- /dev/null +++ b/evidence/prm-001/s1-v2/96eb79f51/rows-llama_cpp.jsonl @@ -0,0 +1,200 @@ +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"f1ca076d355345060d79de6ab291b725d78bfd3596f2cfc01444d40d11e2ea79","stratum":"2k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":3867.459457,"prompt_ms":280.022,"prompt_tps":3506.8673175679055,"decode_tps":67.01770915040254,"input_tokens":982,"output_tokens":241,"peak_rss_mb":1188.0625,"verdict":{"state":"Fail"},"prompt_ids_sha256":"27e16faa8fbdff4c4d6e0ff52b176400b21425330e38ee63f75e165299d93d1b"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"7788e95d4d3ecd6288d7d51fa0ed8459158b9b7ff7b94bbbffd244bbb7e1dad1","stratum":"2k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":2194.916114,"prompt_ms":199.945,"prompt_tps":3455.9503863562477,"decode_tps":67.63545419530848,"input_tokens":691,"output_tokens":132,"peak_rss_mb":1377.234375,"verdict":{"state":"Fail"},"prompt_ids_sha256":"10e34de14fde098ef400a5795d6d775f2af564296e76433ce573ff85c38f81a0"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"366eed3dd5e0055ebfdd837af58bd800debdd62e4c0a7f7da2dffa84d06ada9f","stratum":"2k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":4964.099141000001,"prompt_ms":355.717,"prompt_tps":3930.090493285393,"decode_tps":66.26937333390869,"input_tokens":1398,"output_tokens":302,"peak_rss_mb":1554.0390625,"verdict":{"state":"Pass"},"prompt_ids_sha256":"513376357643c1fb6950a0630d0a17b9cd1678c7d4961730e6cdd82a5e6755c4"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"f9d7c31e37b0ed40fbfd87af8984c8860ec6c047dc68a8b55bcb88ee67ee6452","stratum":"2k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":2351.9013019999998,"prompt_ms":203.197,"prompt_tps":3508.9100724912273,"decode_tps":67.06015054280127,"input_tokens":713,"output_tokens":140,"peak_rss_mb":1758.140625,"verdict":{"state":"Pass"},"prompt_ids_sha256":"13634b8349a0b7acd08e20bc05d86f438f3a09e1be22555abc534924c2845212"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"f8879c54b81ffd2aaeaf6398391a818b5f56d1c37ffd4a71100bf19864c1b902","stratum":"2k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":1809.030485,"prompt_ms":254.328,"prompt_tps":3735.333899531314,"decode_tps":67.29950635812087,"input_tokens":950,"output_tokens":101,"peak_rss_mb":1935.78125,"verdict":{"state":"Pass"},"prompt_ids_sha256":"6526b0ed8df35ece7bfb8f728415890c02540d9548fa313c47f20bb0514294f2"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"296c5544001c3a5e623d3bc13a3b59728bad8d086d6f5baaa8ec44409d3e5cea","stratum":"2k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":2118.798391,"prompt_ms":102.323,"prompt_tps":3146.8975694614114,"decode_tps":68.0921999105074,"input_tokens":322,"output_tokens":134,"peak_rss_mb":2118.19921875,"verdict":{"state":"Fail"},"prompt_ids_sha256":"1f540bb4461f766244ab8dbc1d7a1c48162b6cb270d8ae0eb239e075d14fcb7f"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"275cbe4c206c2eb59aca3ca92f0bbd7f171bc949a205afe8e44882e4e56fdc5c","stratum":"2k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":4615.4543300000005,"prompt_ms":469.173,"prompt_tps":4058.2045428871647,"decode_tps":66.64456573375666,"input_tokens":1904,"output_tokens":274,"peak_rss_mb":2234.6484375,"verdict":{"state":"Pass"},"prompt_ids_sha256":"e482de36d4b6c8e06ae0882d309b33305eabf0479dedcbcc690ff717f0dc969c"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"2d012d3f8e232bfdbed32db1d29426b65b9c7725d2a001308ac4fb59f8cdafa9","stratum":"2k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":3099.95154,"prompt_ms":456.374,"prompt_tps":4003.295542690863,"decode_tps":66.42768330719208,"input_tokens":1827,"output_tokens":171,"peak_rss_mb":2453.515625,"verdict":{"state":"Pass"},"prompt_ids_sha256":"aee8112d87fb3291454282615ad62f23da8c7bfa3b2e162d7e909ec867ad11b3"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"e777a873cb9229ece5b23335a8dd1bb421f1b5e298f76024a7af842e2cfcadfb","stratum":"2k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":2302.27568,"prompt_ms":471.129,"prompt_tps":4009.517563130268,"decode_tps":66.4244464868053,"input_tokens":1889,"output_tokens":117,"peak_rss_mb":2666.74609375,"verdict":{"state":"Pass"},"prompt_ids_sha256":"dd8abf75e8df1fb4ba891ef3ac0728486da2bed010880eb742d6d79e669fe432"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"a7a62e12ad053746008e769e95d9c06f39e8b741a52ad4f650d05d1ca720c754","stratum":"2k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":3475.676849,"prompt_ms":455.059,"prompt_tps":4012.666489400275,"decode_tps":66.44586195616618,"input_tokens":1826,"output_tokens":196,"peak_rss_mb":2880.2109375,"verdict":{"state":"Pass"},"prompt_ids_sha256":"7081fe352adf097f8a152f710ba69d4a79e4044130c1c3982d8f715f175e3c32"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"66ec2c8042874218dcf97952a6a557fe1223cb59db4a28bf7615cd2a6a4f269b","stratum":"2k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":2064.654588,"prompt_ms":112.97,"prompt_tps":3213.2424537487827,"decode_tps":67.70155873510032,"input_tokens":363,"output_tokens":128,"peak_rss_mb":3092.83203125,"verdict":{"state":"Pass"},"prompt_ids_sha256":"7f8167b3b624a3f6f0dd6b3c48781ebe5c7aa79d9a20f2016a592cbdb64df348"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"5f449a13b53a4663c67c7b0cea0588720e0a2e63e79561ba6adcd58d9bd87d77","stratum":"2k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":5842.679356,"prompt_ms":436.578,"prompt_tps":3994.704268194917,"decode_tps":66.0421270415459,"input_tokens":1744,"output_tokens":355,"peak_rss_mb":3210.01171875,"verdict":{"state":"Fail"},"prompt_ids_sha256":"4be9a7ed9292e82aa712abd073d04ef245be3607a90208a80f7195e73fbdfd62"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"3b86db746737d9338f2ac5c3c0aa7184acd7f05314b46fb8ae1a8bbdd83ac4ad","stratum":"2k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":2154.683682,"prompt_ms":193.991,"prompt_tps":3386.7550556469114,"decode_tps":66.94468190915606,"input_tokens":657,"output_tokens":127,"peak_rss_mb":3428.58984375,"verdict":{"state":"Pass"},"prompt_ids_sha256":"8278ed707fc9d1928b2ec8f690a552e0a16913c5847057eb444ca2509a21b839"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"a5f04841ad4bdf65c809d00511d1b4669ed6f52b6ec0be2eac5ee0417b29297f","stratum":"2k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":3210.683311,"prompt_ms":426.408,"prompt_tps":3822.62996941896,"decode_tps":66.72117405674322,"input_tokens":1630,"output_tokens":182,"peak_rss_mb":3604.1953125,"verdict":{"state":"Pass"},"prompt_ids_sha256":"f02afc68f2366821fc07f38c65009e232df057bd81d998875a7600a85ff58c1c"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"f1206fea7a5282d2607d846f5d4f0a9fe7b6982f8d0ab75a42632d7a8ca4e487","stratum":"2k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":3086.690352,"prompt_ms":268.986,"prompt_tps":3736.253931431376,"decode_tps":66.31447123864866,"input_tokens":1005,"output_tokens":183,"peak_rss_mb":3811.83984375,"verdict":{"state":"Pass"},"prompt_ids_sha256":"f76934c956f6886a06b4e394e263a24c39df6734ad36fed9be55913886901237"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"993b00de597f00fefd6e145197fc5659223b89690cc0a671c636d955b4c961c5","stratum":"2k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":5293.305445999999,"prompt_ms":416.507,"prompt_tps":3807.85917163457,"decode_tps":66.33191575721936,"input_tokens":1586,"output_tokens":320,"peak_rss_mb":3999.8359375,"verdict":{"state":"Pass"},"prompt_ids_sha256":"79f32211f992adecbdc1e0848ba2e99956661eb798fbac366156478f0670d2d2"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"47015b6bbe7f1a4b0b27cd8396c3430142487ab219124adf8c7f331e365c7ec3","stratum":"2k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":2929.113264,"prompt_ms":425.151,"prompt_tps":3904.495108796639,"decode_tps":67.468870153055,"input_tokens":1660,"output_tokens":164,"peak_rss_mb":4210.19140625,"verdict":{"state":"Pass"},"prompt_ids_sha256":"c8ceab8a8499f6119d42fc491d582972ddcbb9d0eeb513d16f0a8bc73f03a953"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"9af203f89adb6dc0348f46161c0da6bdf7ebea3752a638220571ad3a65854634","stratum":"2k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":3509.99676,"prompt_ms":437.8,"prompt_tps":3908.1772498857927,"decode_tps":67.20704249558139,"input_tokens":1711,"output_tokens":202,"peak_rss_mb":4418.2890625,"verdict":{"state":"Pass"},"prompt_ids_sha256":"40d56ab8f83a8a2a5627f8c21de53e8cc3086149ca9a2d3f7bf948f43294f577"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"34d4c5a7a0e46bbd1b00935ef3000cc8ddba8dd64cbdffa9e01f1207863f945c","stratum":"2k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":5125.584895,"prompt_ms":370.937,"prompt_tps":3917.1072176676903,"decode_tps":66.92750575084749,"input_tokens":1453,"output_tokens":314,"peak_rss_mb":4629.1484375,"verdict":{"state":"Fail"},"prompt_ids_sha256":"6ea38a67c9dcd29895861eb72204a5cb6730ab44d894a6ab601f1753ff7c9df4"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"167d286e0f8a0bae32d26b9d8abc056df02e030188cf395ff255acc3cb1d640d","stratum":"2k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":3555.605921,"prompt_ms":254.536,"prompt_tps":3708.709180626709,"decode_tps":66.93840767783536,"input_tokens":944,"output_tokens":217,"peak_rss_mb":4835.15625,"verdict":{"state":"Fail"},"prompt_ids_sha256":"2fcc6f03b34f1893afd08fe8ec1dcf30568ee3397ef74662c46ef4085107c1d6"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"fab75467c60ebd40cc94c13296129957a26a7eb9865a58175dd09db9610f1631","stratum":"2k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":3536.772497,"prompt_ms":302.74,"prompt_tps":3478.2321463962476,"decode_tps":66.96281921049572,"input_tokens":1053,"output_tokens":213,"peak_rss_mb":5022.29296875,"verdict":{"state":"Pass"},"prompt_ids_sha256":"e857ce0443d4fb4bf8d286c247650ecf93b401aff929197f55c9aa7777881371"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"c274c3708f67fd97e467895e6c780665af6a13a28c8a727d25ea1863e3328dc8","stratum":"2k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":3425.316753,"prompt_ms":415.49,"prompt_tps":3716.0942501624586,"decode_tps":67.59012667884194,"input_tokens":1544,"output_tokens":200,"peak_rss_mb":5212.63671875,"verdict":{"state":"Pass"},"prompt_ids_sha256":"6624743530f806ebfeef29164bc538770adccaf112ec5c254583759b9530f2af"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"11ca5b1e87d2bbf217c1fd81b2c2ea63a96390dea93e9d9258aa7dff541f8de8","stratum":"2k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":4535.274361,"prompt_ms":387.779,"prompt_tps":3948.125091869338,"decode_tps":67.45043251670072,"input_tokens":1531,"output_tokens":276,"peak_rss_mb":5418.06640625,"verdict":{"state":"Pass"},"prompt_ids_sha256":"7f03bbbdd56a316389c60433ff343e431f7bf8984965b1d7c775d0a2b663d42c"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"7ba9cfea210f85f585654545dc13dc487fe37b56924c0996577c87a463100bac","stratum":"2k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":1608.4639750000001,"prompt_ms":92.951,"prompt_tps":2947.789695646093,"decode_tps":68.45995229710523,"input_tokens":274,"output_tokens":101,"peak_rss_mb":5623.65625,"verdict":{"state":"Pass"},"prompt_ids_sha256":"4a21bcf8590e6ea0af00d401671ef07b3715194ed9f4846feab7f8ce28239f10"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"b1c9fb8c5a26aa63dcdb57e6ec6ce35a2c5a73287516f57082f77944c007add1","stratum":"2k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":2204.101573,"prompt_ms":214.509,"prompt_tps":3692.152776806567,"decode_tps":67.76973896328728,"input_tokens":792,"output_tokens":133,"peak_rss_mb":5737.609375,"verdict":{"state":"Pass"},"prompt_ids_sha256":"2f02b9baa0667bc81604dd82b8f81abd390a46442f75a67fed1f9ca6ef54d929"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"9d82e15d5df69ff6b6a2f1d4dd81ba144dca2978ee4f352de1b134cf86a4bfdd","stratum":"2k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":4456.172925,"prompt_ms":421.373,"prompt_tps":3832.708787701158,"decode_tps":67.24780185321845,"input_tokens":1615,"output_tokens":268,"peak_rss_mb":5917.609375,"verdict":{"state":"Fail"},"prompt_ids_sha256":"7e2fd107b311a51bce7d7d6826cbde8412778be86ccc57cd60587b5bf24a54f4"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"4dca9a17db342792f25605430639922e342855e3341653071266173c29d9a859","stratum":"2k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":4109.987260999999,"prompt_ms":312.673,"prompt_tps":3780.307221921944,"decode_tps":66.86863321319359,"input_tokens":1182,"output_tokens":250,"peak_rss_mb":6127.52734375,"verdict":{"state":"Fail"},"prompt_ids_sha256":"cb47cabf619ed11eda1601028c79ced363020fe66316b9895d5602b2f527c0d8"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"e8e5551522f3e72418df3e0fa2227dded865de9949b65a4b85becc6cd3acbc35","stratum":"2k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":1563.001739,"prompt_ms":118.439,"prompt_tps":3242.1752969883237,"decode_tps":66.86585909183081,"input_tokens":384,"output_tokens":93,"peak_rss_mb":6321.59375,"verdict":{"state":"Pass"},"prompt_ids_sha256":"0655ead49567576a1c526b6f710c1807619174567ee1f131159a9ae03c68b8ed"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"e0cf54bdaa7fbbbd164db0ebf846b77b0b78b65b8bfd4a392fcc3171c64a2504","stratum":"2k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":1971.861317,"prompt_ms":183.359,"prompt_tps":3299.5380646709464,"decode_tps":67.78005945115383,"input_tokens":605,"output_tokens":119,"peak_rss_mb":6438.78125,"verdict":{"state":"Pass"},"prompt_ids_sha256":"a7da9728eaf7dc43519a3329e56fbddffb918e60cd11e3b60c2fdd066013112c"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"7492d83b3314d90dae86dcb3530baaa502e8d425427daa63a8e9fc03de96b069","stratum":"2k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":2138.402421,"prompt_ms":336.298,"prompt_tps":3850.751416898108,"decode_tps":67.70081102099768,"input_tokens":1295,"output_tokens":118,"peak_rss_mb":6612.4609375,"verdict":{"state":"Pass"},"prompt_ids_sha256":"3710b42517c7bd8dcd003e4e02e46f2547485a103ea66db61273afabb7a3c2bf"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"0d9bcac1ca95dc9fb8293713b22add78150aaaacfa6a269e8065ea5605feee17","stratum":"2k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":5195.651847,"prompt_ms":256.086,"prompt_tps":3764.3604101747064,"decode_tps":64.65180882995463,"input_tokens":964,"output_tokens":315,"peak_rss_mb":6807.37890625,"verdict":{"state":"Fail"},"prompt_ids_sha256":"5ddc4f9e03e618df9e68b2df8ad867702dff23019b64b9d2ef32f96ad0949717"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"3f51712000a76356352961b38d5b137cdf5179679655ed3fbfef2acad4eeb0e8","stratum":"2k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":2321.7207860000003,"prompt_ms":208.398,"prompt_tps":3301.375253121431,"decode_tps":64.85963784736758,"input_tokens":688,"output_tokens":133,"peak_rss_mb":6998.12890625,"verdict":{"state":"Pass"},"prompt_ids_sha256":"d7e97aaa3ac6609d18fe9c3ba27bfcd91550e16482bf475e2d4f19eb26acd77a"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"b7e5b398d98a7f3dc848eee0e437e07104d372e9365e6f3ce0c101e6667ed890","stratum":"2k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":1953.1391780000001,"prompt_ms":105.93,"prompt_tps":2907.5804776739355,"decode_tps":66.72165128798018,"input_tokens":308,"output_tokens":120,"peak_rss_mb":7172.7734375,"verdict":{"state":"Pass"},"prompt_ids_sha256":"b05019b1e8f3ab2be02e8b81733a49d390f98ae5a16b0e8f13d09593ea04b9e6"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"903ef35fb09bf1cb357547ce61b9b7828b90c24845aa5f9fecd8923e022be75f","stratum":"2k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":2796.852886,"prompt_ms":260.443,"prompt_tps":3682.187657184106,"decode_tps":66.80658487555262,"input_tokens":959,"output_tokens":167,"peak_rss_mb":7288.625,"verdict":{"state":"Fail"},"prompt_ids_sha256":"ec9c7f9e8c70ed80adfbcb818a9db16b1472d69cc4b7d5152bd8e139204963b5"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"34186bc542a9bc6191d88846aadf532b79f67209e4b17474d1d4b0a1cf87cc8c","stratum":"2k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":1915.8627689999998,"prompt_ms":184.288,"prompt_tps":2854.2281646119113,"decode_tps":67.0481627306801,"input_tokens":526,"output_tokens":113,"peak_rss_mb":7474.75390625,"verdict":{"state":"Pass"},"prompt_ids_sha256":"717745ed7fb623d700dffd2b1f7f0d7ae241a0fc248f74ed4b935313a1a249fc"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"c9a02d51ce74d1c65af0fa596da511b8d2a571aa8f65f8d380ed9838b4c5243e","stratum":"2k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":4109.279859,"prompt_ms":274.091,"prompt_tps":3695.8528371964057,"decode_tps":67.10667069483519,"input_tokens":1023,"output_tokens":254,"peak_rss_mb":7695.71875,"verdict":{"state":"Fail"},"prompt_ids_sha256":"0c2f2e99acdaace2521543f52e497a9e01039e659d7c95ffb47efa75d6a0e742"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"d09cb9713ee02abec5dce5dd4cd0b58237e449a0032fdaf96dcc6ac95a0a6101","stratum":"2k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":2089.047439,"prompt_ms":173.115,"prompt_tps":2738.0642925223115,"decode_tps":66.82514816559491,"input_tokens":484,"output_tokens":123,"peak_rss_mb":7934.828125,"verdict":{"state":"Pass"},"prompt_ids_sha256":"6ab504c554d5d36c516e432f36c59eac000d1d9af5d19121bcc80d31a4dd1ace"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"a889b6ddfa18809d32510029c8f832ddf1dc03f3209feee377d47043c757104e","stratum":"2k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":1915.560953,"prompt_ms":187.662,"prompt_tps":3165.265210857819,"decode_tps":67.60876257854991,"input_tokens":604,"output_tokens":113,"peak_rss_mb":8106.34765625,"verdict":{"state":"Pass"},"prompt_ids_sha256":"909814ad448169a742c52875b82d6ef865c02703afda46a7450ffbe8394e794a"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"c1d14d2cd0dbbdd90638158ee715c79fc76baa8ff2c6322160dac0bb5bd27771","stratum":"2k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":2668.438874,"prompt_ms":123.919,"prompt_tps":2598.4715822432395,"decode_tps":67.89252653703394,"input_tokens":332,"output_tokens":168,"peak_rss_mb":8328.00390625,"verdict":{"state":"Fail"},"prompt_ids_sha256":"82eb2fb253fce96176b306689444000ece53e63e156d9fa4863a938a3e705b71"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"955705510e7439ac4b8e27505f87bcb7d3c71b677cb11bddc4a91d5c44e16a04","stratum":"2k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":3195.497377,"prompt_ms":457.02,"prompt_tps":3899.1729027176057,"decode_tps":65.58865256684545,"input_tokens":1792,"output_tokens":177,"peak_rss_mb":8496.21875,"verdict":{"state":"Fail"},"prompt_ids_sha256":"51eeb6fcedffb81fd02fc9c5f05805cb72211d244fb56aad4117e964b187a2d0"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"bf94290e1f98fc502cb4f962bce2883c3eaa9ca2fe6cb67f6b87c7083d3ec0e1","stratum":"2k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":4432.0211580000005,"prompt_ms":366.334,"prompt_tps":3450.4031839796467,"decode_tps":62.400446655828695,"input_tokens":1274,"output_tokens":248,"peak_rss_mb":8758.78125,"verdict":{"state":"Fail"},"prompt_ids_sha256":"05a06916a4b3576cd25ee421c0588937e59d599607237eeab47bec5ef1db27eb"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"258be4b71904b9f67bdda4ea10d76b9be53c374b04a9d486facd87a97fab0756","stratum":"2k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":2538.240788,"prompt_ms":352.026,"prompt_tps":3417.3612176373335,"decode_tps":63.75136010890458,"input_tokens":1213,"output_tokens":134,"peak_rss_mb":9007.36328125,"verdict":{"state":"Pass"},"prompt_ids_sha256":"d1fbddde370dc12c4aec16fbd2fc2fb32418489c008dd55dd3d29e40a5fbd855"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"1bf59f2a1774a0efe188bbfa1ac4a7af14b2059017de009fc98df2a6b8382017","stratum":"2k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":5480.120895,"prompt_ms":475.805,"prompt_tps":3743.130063786636,"decode_tps":66.24939738528917,"input_tokens":1791,"output_tokens":326,"peak_rss_mb":9250.65234375,"verdict":{"state":"Fail"},"prompt_ids_sha256":"6c5a3de9322024e61926b8105522954447661224770b9881c91a10be3aa69a7c"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"305cdbe50000c485ff10ec6127fbe0744109a126be589ccbb6d243834c3c13ef","stratum":"2k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":2636.717755,"prompt_ms":277.438,"prompt_tps":3705.332362545866,"decode_tps":66.31765718602323,"input_tokens":1038,"output_tokens":152,"peak_rss_mb":9328.9296875,"verdict":{"state":"Fail"},"prompt_ids_sha256":"0d2fc4877c6ceccc5bcecc9e02a8dc8468a8265069ac9e54235fb1f170a59c3e"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"7680438d94dac7e6c61561a71575435fcd22d1543b508ec9434d00f686511896","stratum":"2k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":4691.636993,"prompt_ms":253.505,"prompt_tps":3585.728092148084,"decode_tps":67.27064841558108,"input_tokens":919,"output_tokens":294,"peak_rss_mb":9390.66015625,"verdict":{"state":"Fail"},"prompt_ids_sha256":"0ab5ac2510b12bcacb54e31107686f33fa3f624fb80ca84c81fbced4c8e24f46"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"d928b82fd55ca6c509650e822de0f3946d5ccb3d5fbbabba30784e0f6483e2ff","stratum":"2k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":4701.527512000001,"prompt_ms":373.606,"prompt_tps":3795.442257351327,"decode_tps":66.84776622598685,"input_tokens":1428,"output_tokens":282,"peak_rss_mb":9425.71875,"verdict":{"state":"Fail"},"prompt_ids_sha256":"ef93f1ea44b6a4051e70202045ed0b6322e8987557045343e51ca8cdc8074df0"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"64cce1452c38a191c5f249f069acb21eb4098bd2620cddec04b346c3075849dc","stratum":"2k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":3834.732791,"prompt_ms":189.336,"prompt_tps":3184.8142983901635,"decode_tps":67.21383283464087,"input_tokens":613,"output_tokens":237,"peak_rss_mb":9424.265625,"verdict":{"state":"Fail"},"prompt_ids_sha256":"337e04b1920ed7b30ceedf4ff49f4bf841d2ce7657bafdb2a9165eea27d39ea6"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"69b831cc2059acf0f511c8d788337f36e8ef890f6b86b91f6ffa159b8369d1e5","stratum":"2k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":2805.0629320000003,"prompt_ms":488.295,"prompt_tps":3950.480754461954,"decode_tps":67.53326907982567,"input_tokens":1939,"output_tokens":152,"peak_rss_mb":9432.28125,"verdict":{"state":"Pass"},"prompt_ids_sha256":"d93d068e263876cd1cb37558b0015f0fdcf11be3da9eefe2c4aff454b8a9e616"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"14d125f8a319774d2ddc89c330d806c3371695a96ec1cc8757cb63f20c9afe37","stratum":"2k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":2937.1208170000004,"prompt_ms":188.39,"prompt_tps":3020.3301661447,"decode_tps":67.32900145384997,"input_tokens":579,"output_tokens":178,"peak_rss_mb":9430.68359375,"verdict":{"state":"Pass"},"prompt_ids_sha256":"04222af6320ccf0f9b9bc386e01fc9ed98736d6df5ee61bd98894f8facc87347"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"0b54a849b9e7541ec803e57fab1147aee8d209935102b9230c0a024229ccdac7","stratum":"2k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":2217.244353,"prompt_ms":187.664,"prompt_tps":2957.4132492113567,"decode_tps":67.40462634245387,"input_tokens":565,"output_tokens":131,"peak_rss_mb":9430.68359375,"verdict":{"state":"Pass"},"prompt_ids_sha256":"6ea89d378295c73f8cfc8d17f5f2fcab76f6abb4e8c59463e8a4a4025885fed7"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"ae0f31c8b5b43715e0f5f31727de5648fe149c06faef682adb8898f4dddc0d59","stratum":"8k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":3862.5239389999997,"prompt_ms":540.878,"prompt_tps":3901.0645653918255,"decode_tps":66.82744442261553,"input_tokens":2159,"output_tokens":217,"peak_rss_mb":9430.68359375,"verdict":{"state":"Pass"},"prompt_ids_sha256":"a81f7ea1e1114b3b88a0717effd69745ef46e82f2efdcdbd465afae931ebf1f0"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"6324ab9b418859326df39301f2fc9ca30815280479d95ad375d91e6252a4b82b","stratum":"8k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":5657.261002,"prompt_ms":859.748,"prompt_tps":4098.875484444278,"decode_tps":66.22921094003685,"input_tokens":3573,"output_tokens":310,"peak_rss_mb":9430.68359375,"verdict":{"state":"Pass"},"prompt_ids_sha256":"3ae7c4e045aeaa9df4ea1cf4961254d625990b616146ceda3ec66666473b7cff"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"187723585ce31ae0a9dfb2bd1c94ee372724ba456ade6c3ef28816fb1e369e4d","stratum":"8k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":3371.066753,"prompt_ms":555.155,"prompt_tps":3993.479298574272,"decode_tps":67.06559391858237,"input_tokens":2266,"output_tokens":179,"peak_rss_mb":9430.68359375,"verdict":{"state":"Pass"},"prompt_ids_sha256":"eb266f1f82f6cd93fd1590494e67349a0bed128c01781f180d09e71b40998f5d"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"dd16771d607cda7d13b2528bec93be7261e9d739a6d40c855b0794500da92522","stratum":"8k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":5308.778663,"prompt_ms":1192.676,"prompt_tps":4115.954374867944,"decode_tps":65.48444744373211,"input_tokens":4958,"output_tokens":263,"peak_rss_mb":9466.64453125,"verdict":{"state":"Fail"},"prompt_ids_sha256":"51ee3f91d5eaa6d044852fa83079495964304a501fbc737ecc01becf473718b6"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"40ce4b133ac399054d67393d53e91be97d3cafaa33622094f90a35fdea53c703","stratum":"8k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":9830.875328,"prompt_ms":1665.624,"prompt_tps":4137.788600548503,"decode_tps":63.86336090247553,"input_tokens":6941,"output_tokens":512,"peak_rss_mb":9465.7578125,"verdict":{"state":"Pass"},"prompt_ids_sha256":"212c625213d37b2c9f816b691908099bd9f9a58e680ca2d15863dcdabb735b51"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"bc9a159757fe4720bfdc9073b5c2d8879c8adec829ab9308b1b1031768b5c3f0","stratum":"8k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":7112.382621,"prompt_ms":1562.373,"prompt_tps":4141.776643605592,"decode_tps":63.84251178974033,"input_tokens":6520,"output_tokens":341,"peak_rss_mb":9465.7578125,"verdict":{"state":"Pass"},"prompt_ids_sha256":"966d327b207b9dda0ee074cf93126d6091ceef81459965ec9b5b1a03824d5f5f"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"33b63a5283685481a92f5226d573249f59330892462fc75a3c5ff14e9c61ba82","stratum":"8k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":4665.051439,"prompt_ms":1632.919,"prompt_tps":4128.802469687719,"decode_tps":62.900927152598705,"input_tokens":6791,"output_tokens":179,"peak_rss_mb":9465.7578125,"verdict":{"state":"Pass"},"prompt_ids_sha256":"8d3561cb81010d0287b8d36013ce4022187e868e05fb5a2acba13d93ddd0ba75"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"df48ffe5f40ed18ea677fa829b51ee4eaaee5c8caa5a7462bbe212d83396c540","stratum":"8k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":6159.164308,"prompt_ms":832.546,"prompt_tps":4076.651620451002,"decode_tps":66.76898069341735,"input_tokens":3443,"output_tokens":343,"peak_rss_mb":9465.7578125,"verdict":{"state":"Pass"},"prompt_ids_sha256":"513d5ac32c370db50606bba64bf9369a630e11ff0935f649326b6b1082750baf"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"cbc1450d830a7bda671630905c4b86039bf483e305a0d8fecf6971de02ad0406","stratum":"8k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":6152.232332,"prompt_ms":1634.876,"prompt_tps":4116.520151987062,"decode_tps":64.45083172647435,"input_tokens":6779,"output_tokens":281,"peak_rss_mb":9465.7578125,"verdict":{"state":"Fail"},"prompt_ids_sha256":"c9b299004f9b7cc8d7af8cf6262f62e552c1752be74d07b00d51c812552f258f"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"62b4bd91655d82632f2ad6754eac98e25881dcdfeb3eff15172402b9da179800","stratum":"8k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":2618.689475,"prompt_ms":938.591,"prompt_tps":4117.8745587801295,"decode_tps":66.24765027865418,"input_tokens":3914,"output_tokens":97,"peak_rss_mb":9465.7578125,"verdict":{"state":"Pass"},"prompt_ids_sha256":"1d6f64f5e542ef9606b341f0aa9e816aad85f379d21ff095d0138038b8b4ba4a"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"a6c9bb4b1ea341e343c764aed2b694ca7f4c6ab69461291cf6e20a2d1fe53a47","stratum":"8k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":5022.727672,"prompt_ms":727.724,"prompt_tps":4111.448845991063,"decode_tps":66.52049960294059,"input_tokens":3041,"output_tokens":275,"peak_rss_mb":9465.7578125,"verdict":{"state":"Fail"},"prompt_ids_sha256":"a5e089a434b2f460a61b44fe52bddb2e49e8dc5668c0e6de4a0f0bad9f8fe848"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"1239ceb996f617d8cda28ed536075072f460a875c5a4a25a2ef422a9d923043e","stratum":"8k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":3681.0139529999997,"prompt_ms":1031.844,"prompt_tps":4119.808808308232,"decode_tps":66.05006514389046,"input_tokens":4300,"output_tokens":165,"peak_rss_mb":9465.7578125,"verdict":{"state":"Pass"},"prompt_ids_sha256":"f276cbcdd3b7b2349836fbdb1d397d0d986373840f6ddaddc46c581e5f7288df"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"90134b8a8a941ee22156f752e986d85a39c92f11186fff94174ce25e7fc2c65d","stratum":"8k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":4119.5735700000005,"prompt_ms":777.222,"prompt_tps":3979.557964133799,"decode_tps":66.30474540863206,"input_tokens":3142,"output_tokens":212,"peak_rss_mb":9465.7578125,"verdict":{"state":"Fail"},"prompt_ids_sha256":"1fb374c347b9be341a38517cceab0992299ad2aaea74befbaf3906626c37d34b"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"b3c917322e600c6a75b1fdabd254bbb864628227545b9f51c05fedcb33835e55","stratum":"8k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":4613.73171,"prompt_ms":668.354,"prompt_tps":4033.790476304473,"decode_tps":66.56264154505945,"input_tokens":2745,"output_tokens":252,"peak_rss_mb":9465.7578125,"verdict":{"state":"Fail"},"prompt_ids_sha256":"a24ccc6f60289e1cfdec88d1bedae95fd3e07227a8da2a84b31e646d817ee93c"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"38f0a202313d9e2120e55b92747ced425cac793e09a4db44c212449b1cd8566e","stratum":"8k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":9037.904912,"prompt_ms":1094.154,"prompt_tps":4148.410552810665,"decode_tps":65.71046579976003,"input_tokens":4588,"output_tokens":512,"peak_rss_mb":9465.7578125,"verdict":{"state":"Fail"},"prompt_ids_sha256":"95b34da10969f03fa2337931389f1835b5efc1eb5f79585e13cb239ef9503d00"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"e12a747f914c805b9dcbc4349d5ed7fe706e6d0334848a677837cef3d8b95400","stratum":"8k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":4187.200698,"prompt_ms":595.975,"prompt_tps":4057.2171651495446,"decode_tps":66.5266106442577,"input_tokens":2467,"output_tokens":229,"peak_rss_mb":9465.7578125,"verdict":{"state":"Pass"},"prompt_ids_sha256":"0b4a3d5a626c23d2a0a7f227b0d4edfddd53b4da5a6bb9963171c2d1ab882b65"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"a74db9b8ff37392fa217b05b5e70a2564d3c8a6960f02eebe3db8016184e114a","stratum":"8k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":6454.4316659999995,"prompt_ms":1444.265,"prompt_tps":4093.431607080418,"decode_tps":64.589729739292,"input_tokens":5961,"output_tokens":315,"peak_rss_mb":9465.7578125,"verdict":{"state":"Pass"},"prompt_ids_sha256":"ad6c508530582a081bce003291bb7ee3c17effe3ed0527e73172c39694fbc89b"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"27565a444276a49af4557cade6d92398cbccd183c6366e78a5453683923bfe6b","stratum":"8k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":5455.559936000001,"prompt_ms":1900.281,"prompt_tps":4118.338287863742,"decode_tps":63.6675180578119,"input_tokens":7875,"output_tokens":214,"peak_rss_mb":9465.7578125,"verdict":{"state":"Pass"},"prompt_ids_sha256":"72815a6948e2f27d022cec776bb31fd4a99af1123e5c14a25c6fcb175a28e00a"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"e0d0c14ea617d00ecec166c8e598898b3d1b757d639df1729ddf1e3af37872cd","stratum":"8k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":2877.447849,"prompt_ms":682.262,"prompt_tps":4071.7495624847934,"decode_tps":66.13181291208386,"input_tokens":2827,"output_tokens":131,"peak_rss_mb":9474.9296875,"verdict":{"state":"Pass"},"prompt_ids_sha256":"5019b3cebd26be737bb7fe45f6ddb10faa0e2c3bb6fd157fa1eee1deea81d19d"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"7269a7b0e6abf53a633d344677435b768b4342ad507d8833da2aa1b5b90747ee","stratum":"8k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":5442.73069,"prompt_ms":1115.071,"prompt_tps":4116.329812182364,"decode_tps":65.45754826235385,"input_tokens":4639,"output_tokens":274,"peak_rss_mb":9473.48828125,"verdict":{"state":"Pass"},"prompt_ids_sha256":"b76967bf52a3d1d50f5eeb855e43df852e8ff8be2ed8bb4909ea519ca09fa086"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"f87c2ffe3aafeec5f8308352f5190b3c8ec03ed0eafd6d94b000ef39bb3d922f","stratum":"8k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":6674.707880999999,"prompt_ms":1204.28,"prompt_tps":4090.4108679044743,"decode_tps":64.71640644408397,"input_tokens":4975,"output_tokens":342,"peak_rss_mb":9473.48828125,"verdict":{"state":"Fail"},"prompt_ids_sha256":"805c7c9887566084d332a0ff4e3decc7ee4a100a69194e545f3460c67c1fb92a"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"c7323e9dc8cc06122998a5fb83ccb487d2538b7444f9852de80f63983e93dc44","stratum":"8k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":6506.296108,"prompt_ms":926.366,"prompt_tps":4057.7914128972784,"decode_tps":65.35264768683012,"input_tokens":3808,"output_tokens":355,"peak_rss_mb":9473.48828125,"verdict":{"state":"Fail"},"prompt_ids_sha256":"b5cb29cd4d1d2b02437000ebf0115b5483e5b7bf00187bc0d72bb47ba8be7019"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"3bed1317707a01f00daf828a5434d3681278c66100d991520153baa4dc7ebd61","stratum":"8k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":5431.38774,"prompt_ms":503.102,"prompt_tps":4007.1397052685143,"decode_tps":65.65831662949515,"input_tokens":2065,"output_tokens":315,"peak_rss_mb":9473.48828125,"verdict":{"state":"Fail"},"prompt_ids_sha256":"92f0703d894d6fb99af6cf992d9c3d76abd4fad16e00adf056499d5bf79df42a"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"a974a820da6ee897219e5353ad22358cd5cec02a3534baa60388a3f554850d96","stratum":"8k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":5239.489643,"prompt_ms":1050.284,"prompt_tps":4086.5137429495257,"decode_tps":65.32145703468792,"input_tokens":4341,"output_tokens":265,"peak_rss_mb":9473.48828125,"verdict":{"state":"Fail"},"prompt_ids_sha256":"4017ef5b163960754cdb6c87147d51d9110da0dcbb8d7f37fc5b1b1af7c2f8d4"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"1c50664db77045b861287a6f03ded90d514b8d0a32f04336caa5c9c90777b18c","stratum":"8k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":3083.734689,"prompt_ms":549.08,"prompt_tps":3948.422816347345,"decode_tps":66.21628417682263,"input_tokens":2217,"output_tokens":159,"peak_rss_mb":9473.48828125,"verdict":{"state":"Pass"},"prompt_ids_sha256":"ad1b4023aa37427996cc325caba661c60a9cf8343640671dcb1983cfbc5c89b8"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"7e09255545a53208daf6239daa699089bbd30c3ec111d44a59cbe007a8004a57","stratum":"8k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":3534.258393,"prompt_ms":951.09,"prompt_tps":4069.01555057881,"decode_tps":65.68001616481418,"input_tokens":3919,"output_tokens":158,"peak_rss_mb":9473.48828125,"verdict":{"state":"Pass"},"prompt_ids_sha256":"7979c0667fd9abec64e255baa8d0377e7856b52e4e008e1f1b7c35c4471ccdf2"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"1aee819c753ccc3dd6976511c879d33fd99478a25bfe8c5639d006aebe6d00ad","stratum":"8k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":3540.424146,"prompt_ms":1168.783,"prompt_tps":4070.045508875472,"decode_tps":65.40626537893007,"input_tokens":4806,"output_tokens":146,"peak_rss_mb":9473.48828125,"verdict":{"state":"Pass"},"prompt_ids_sha256":"93d0c8edbd6a7cca000f9f23b65b7bd49a90c3e64f4d169cf4705ff14a6bbaf9"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"e92967237c1b1abeb4a548c343928a0afda2c3df6c49e94cec87b7146276b045","stratum":"8k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":5813.007051,"prompt_ms":973.913,"prompt_tps":4083.526968014596,"decode_tps":66.0300430285096,"input_tokens":4026,"output_tokens":310,"peak_rss_mb":9473.48828125,"verdict":{"state":"Fail"},"prompt_ids_sha256":"96c30189e094a2ed3603508cd4222ab22c468239354545620eebf61674389fb1"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"5ef3cfe455ce3ddedb5fd853a62ccbda685b16f06e7ec8818c46f639474fd302","stratum":"8k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":5695.563828,"prompt_ms":686.706,"prompt_tps":4027.9246140269634,"decode_tps":66.0943416675151,"input_tokens":2815,"output_tokens":323,"peak_rss_mb":9473.48828125,"verdict":{"state":"Pass"},"prompt_ids_sha256":"bc74840b53b9b199026a3b937a88491510f44071bb5e597f5a05fc460493e8e5"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"08db5d324448f2faa678eca074285b6275f42db2c667f594ede9fa4adfc9c566","stratum":"8k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":5428.6877190000005,"prompt_ms":1393.326,"prompt_tps":4054.327558661792,"decode_tps":64.22236351142195,"input_tokens":5698,"output_tokens":251,"peak_rss_mb":9473.48828125,"verdict":{"state":"Pass"},"prompt_ids_sha256":"759d47a6fcd52490cc43e3f9f91db6587e8b59bf2ed44a06dae3647f1cab6df2"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"bc995711883be2b01bb079a4433bb34b0eb50a9cda6440856b8c0665d7a4afb5","stratum":"8k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":3054.519312,"prompt_ms":684.724,"prompt_tps":4010.374983204911,"decode_tps":65.93907778704792,"input_tokens":2795,"output_tokens":145,"peak_rss_mb":9473.48828125,"verdict":{"state":"Pass"},"prompt_ids_sha256":"6a64782f6b63fcb21b6814ed2634f945270c9fb500f9ec47abffff02026b3b33"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"9f29c99424409b984aeea5524ec870e016e2e16c617351a938ed700a057ce9ba","stratum":"8k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":5971.877196,"prompt_ms":1819.371,"prompt_tps":4086.027533691589,"decode_tps":63.64057394848246,"input_tokens":7483,"output_tokens":257,"peak_rss_mb":9473.48828125,"verdict":{"state":"Pass"},"prompt_ids_sha256":"f79d101859efe06916d34257165ec054aa373990f5d969fc6c2513c9ee1dd354"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"3d765c54312ef5c279e895fb042a9c08ef9bcd6b44e1098b9db17f68d379f845","stratum":"8k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":4865.479389,"prompt_ms":848.08,"prompt_tps":4085.699462314876,"decode_tps":66.14721494079956,"input_tokens":3514,"output_tokens":254,"peak_rss_mb":9492.30859375,"verdict":{"state":"Pass"},"prompt_ids_sha256":"afa4aed5cdc97dd82a2f3c6f1c3cf83de33f5a873bda726cf8cbedecc68e184a"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"b66cef24141dfcf0cd4776c23e13ac2a297f86fb966ea2f474d3ba2b22ac4689","stratum":"8k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":5559.342332,"prompt_ms":546.45,"prompt_tps":3960.1061396285113,"decode_tps":67.01700525672344,"input_tokens":2213,"output_tokens":327,"peak_rss_mb":9490.70703125,"verdict":{"state":"Pass"},"prompt_ids_sha256":"a7ed93b0333e5ba437c439c1d28332f7c2c21e02f641095bc38eadd1a02cd167"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"bf69f8045076d4b114aef30013614158379bdcec1dddb96497b724fd4bd16df2","stratum":"8k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":4302.261093,"prompt_ms":1148.369,"prompt_tps":4099.727526605125,"decode_tps":65.63547530465242,"input_tokens":4757,"output_tokens":197,"peak_rss_mb":9490.70703125,"verdict":{"state":"Pass"},"prompt_ids_sha256":"ef2a1c7ad19234288174165afe4ff5dc308ed4a5d5448495732bf50390f28fde"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"0ba74445847c37901a423c32a811b0f15026d75d3dd2225fd67ab1a474f97ae2","stratum":"8k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":3766.3690970000002,"prompt_ms":1334.789,"prompt_tps":4108.514529262678,"decode_tps":65.35119778450486,"input_tokens":5533,"output_tokens":147,"peak_rss_mb":9490.70703125,"verdict":{"state":"Pass"},"prompt_ids_sha256":"5fc6cf167f33d8897f20f6457b0677a778f646264bdb552be4e00fb52b0e6b9f"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"a08ed5ccc3331e0a7179c7d49737607bccb9a9cfbc16c6533afacfa6072b133f","stratum":"8k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":2360.498574,"prompt_ms":569.612,"prompt_tps":4004.480242691516,"decode_tps":67.08536063260276,"input_tokens":2330,"output_tokens":111,"peak_rss_mb":9490.70703125,"verdict":{"state":"Pass"},"prompt_ids_sha256":"baf2d30022d1dbc62e43697b26bd1999ee610fcd6b4275293cd352e091390a04"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"b0bdb087e1d95680866dad2b47061fa38e925929fe937f39b834992f44838399","stratum":"8k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":4966.567414,"prompt_ms":1484.171,"prompt_tps":4108.017202869481,"decode_tps":64.35087328624645,"input_tokens":6146,"output_tokens":216,"peak_rss_mb":9490.70703125,"verdict":{"state":"Pass"},"prompt_ids_sha256":"62861c84c3f9994a8f69de8848af2f7ea98e8e6de26bf95f24f760b6c5975b69"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"6e56cee326693c4915a7feef43eee6a597f75ff2e54f970e8f3197135869aed1","stratum":"8k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":6650.669677,"prompt_ms":1689.137,"prompt_tps":4139.983908942851,"decode_tps":63.65842897279377,"input_tokens":7042,"output_tokens":305,"peak_rss_mb":9490.70703125,"verdict":{"state":"Pass"},"prompt_ids_sha256":"6383d5bff555ad33b2339b6120920cf9522aa807a1285f41b3efcface1525924"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"c81288783ae98e2b5a143cdfc4738ce6c817ae09e2b01a0979e18a0de5f90bb6","stratum":"8k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":5052.518025,"prompt_ms":1789.706,"prompt_tps":4111.289787261148,"decode_tps":63.52869487427154,"input_tokens":7407,"output_tokens":197,"peak_rss_mb":9497.58203125,"verdict":{"state":"Pass"},"prompt_ids_sha256":"3c18df6c64b92c1d238f0b099ce92e3e85ec2f7c4a8a113995df1d4036e878bd"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"0354fce0d010d6d69a1539589a7d6046495172ee6865f418b0a39c852edae583","stratum":"8k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":6120.294568,"prompt_ms":972.928,"prompt_tps":4105.13419286936,"decode_tps":66.2229592968503,"input_tokens":4043,"output_tokens":327,"peak_rss_mb":9495.875,"verdict":{"state":"Pass"},"prompt_ids_sha256":"04371fb2a9cfff6184614b3943d1f0afebf2446e3ae21acce139cb914a34632b"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"b45b72bc1a202cb0504dfd27dd16f60af1ec09570fe31e7ecd927168837a02cf","stratum":"8k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":4815.434168999999,"prompt_ms":1434.966,"prompt_tps":4145.045945339472,"decode_tps":65.10473724604458,"input_tokens":5997,"output_tokens":209,"peak_rss_mb":9495.875,"verdict":{"state":"Pass"},"prompt_ids_sha256":"b4ac6d8b05c4c10cf4f5c3388011ec14c3513f17aefa6dc582e2dff4d2968405"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"b6dabc525c8486ab7bab78b2968c241327371b5c234ca1b8a87adb93bb8768ab","stratum":"8k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":4872.402064,"prompt_ms":1158.332,"prompt_tps":4133.529937876187,"decode_tps":65.37420907865852,"input_tokens":4837,"output_tokens":231,"peak_rss_mb":9495.875,"verdict":{"state":"Pass"},"prompt_ids_sha256":"f6a335774a9b2c8389c1c66b5157283cfdff4ebf3dead387979335582189e2cd"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"fd684a94b71725b926c4fb5be284b7c83518a537c466c552d7dbc8c9c1f64bec","stratum":"8k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":3676.6798339999996,"prompt_ms":559.024,"prompt_tps":4001.6171040957097,"decode_tps":66.86910694780204,"input_tokens":2286,"output_tokens":198,"peak_rss_mb":9495.875,"verdict":{"state":"Pass"},"prompt_ids_sha256":"15033c21d4c89645a7f23d359d8c331c1cdae370c57dc97d3aa3ba5c4b935347"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"c5a7b5a62a110bae58ca259b02ea467d3dcdbde7ad3308a657e8028e63322e39","stratum":"8k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":4483.133879,"prompt_ms":961.184,"prompt_tps":4113.676465692313,"decode_tps":65.50389837114628,"input_tokens":4003,"output_tokens":222,"peak_rss_mb":9495.875,"verdict":{"state":"Pass"},"prompt_ids_sha256":"6906e62e5d443f9ed949a4eb821a1ea194d8a3d072af1e52d8d0df23e1fe9f62"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"b71b016e3bf4e4aa82e2c46d8b1c2f137099cf683949b4c65e5900bad8de95ce","stratum":"8k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":2277.6171139999997,"prompt_ms":507.891,"prompt_tps":3949.6663654209265,"decode_tps":66.50522371075964,"input_tokens":2055,"output_tokens":110,"peak_rss_mb":9495.875,"verdict":{"state":"Pass"},"prompt_ids_sha256":"24e56a79ebb675cd95f21cf12c693fbac4af6e2b009309f49d8f85d8b61164e5"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"611ff958e756f16b6143ffe425e3351e94a71034d0306f0e00ebd25696e66a31","stratum":"8k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":3017.732688,"prompt_ms":863.369,"prompt_tps":4072.4186298095015,"decode_tps":65.28125340006528,"input_tokens":3565,"output_tokens":133,"peak_rss_mb":9495.875,"verdict":{"state":"Pass"},"prompt_ids_sha256":"ac1e1de733551d5421552bcc206f001ac1b3efa35aa1a05a6ab785fd99ff12d8"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"1f97a6ca5af77396cc4febaacda124d2f993c18d3e3b96e3e0b28ff92f20610d","stratum":"8k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":4466.232699,"prompt_ms":1897.278,"prompt_tps":4125.910910261964,"decode_tps":62.5835915769946,"input_tokens":7877,"output_tokens":152,"peak_rss_mb":9495.875,"verdict":{"state":"Pass"},"prompt_ids_sha256":"c42007b3ae5fcfb14a8d8a6f5d25753226fc0910d9bc95d65e8b75cc5e4b05f1"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"3811dbd8d38716269bb87eca6a6039f05923f1c129d3d2b09659f4cff15dfc2e","stratum":"8k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":4361.573592,"prompt_ms":1097.399,"prompt_tps":4119.741315601709,"decode_tps":64.86191907840013,"input_tokens":4570,"output_tokens":200,"peak_rss_mb":9495.875,"verdict":{"state":"Pass"},"prompt_ids_sha256":"ea94bd384af8194fc5c38a4dbcdc751afcf318dc8834c3f4560913718900fb35"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"f65d88532cd6ca202557458343fb035dbe140810881382b7ce5d8c668afda27d","stratum":"8k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":3722.74851,"prompt_ms":954.868,"prompt_tps":4095.8540866381527,"decode_tps":65.52091412341397,"input_tokens":3960,"output_tokens":173,"peak_rss_mb":9495.875,"verdict":{"state":"Pass"},"prompt_ids_sha256":"d01aacddaa6f22bdfb64eca2e82ca55373391a16f47162da338cdbbad3508144"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"6bcf0a0fb95dfe5e90aa366595a006bc443f7a3d9de3c86ba176541bdbbc01cc","stratum":"16k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":6953.609653,"prompt_ms":3897.796,"prompt_tps":4029.713202024939,"decode_tps":59.090728457369146,"input_tokens":15756,"output_tokens":172,"peak_rss_mb":9495.875,"verdict":{"state":"Pass"},"prompt_ids_sha256":"e898c162c9392f2d3bd457479d776fac6602c99eba6b04a3d80e8cecc0c2cc92"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"c70e109e3c965254163267ebeee325ed9dfebe4479eb874c38a592a3488af34e","stratum":"16k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":6940.291999,"prompt_ms":2705.045,"prompt_tps":4090.135284255899,"decode_tps":61.62212346272536,"input_tokens":11113,"output_tokens":243,"peak_rss_mb":9495.875,"verdict":{"state":"Pass"},"prompt_ids_sha256":"6c1223de3236d024d3bf9a1a600bf90f1709debccf3ee61c2bd92011cb7c1968"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"9ec99261cd227d8f5e528bab7391b88aaafa4483233ccb0731a46b4f64bee8dc","stratum":"16k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":6907.338987,"prompt_ms":3336.169,"prompt_tps":4054.0512186283136,"decode_tps":60.11204520911553,"input_tokens":13574,"output_tokens":199,"peak_rss_mb":9495.875,"verdict":{"state":"Pass"},"prompt_ids_sha256":"404ea4c6921bceb338fc1491d0dbe4aa8a2711122ca4646724bc17b6bcaaa069"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"9444f47e49f1779a103b611560b50ba89a8f20df19b976c9d4e06c2b01e103a5","stratum":"16k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":10189.509805,"prompt_ms":2158.3,"prompt_tps":4092.5728582680817,"decode_tps":62.58776805308681,"input_tokens":8882,"output_tokens":486,"peak_rss_mb":9495.875,"verdict":{"state":"Fail"},"prompt_ids_sha256":"8f31bc51da4630c662e6ec7e0c67c440f57a03802d09a5f5f286399b72e3c666"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"5e5b0935bd95c58dc95d3caebb22a4f41d0573018dff4ce1633bb7668afdd1a2","stratum":"16k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":7873.263081,"prompt_ms":3578.478,"prompt_tps":4046.692476522142,"decode_tps":59.71569333417951,"input_tokens":14530,"output_tokens":240,"peak_rss_mb":9495.875,"verdict":{"state":"Pass"},"prompt_ids_sha256":"b191956bebc847f0567b83979f08d07982cecd6a68b6d78eba5b6a707c96236e"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"df963cc72999db8bd18665c41caca7b3cd058ec66923e7f5f3c161415d0cd4a2","stratum":"16k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":5849.8686800000005,"prompt_ms":2833.908,"prompt_tps":4084.8185615058783,"decode_tps":61.2179916691272,"input_tokens":11625,"output_tokens":165,"peak_rss_mb":9511.69921875,"verdict":{"state":"Pass"},"prompt_ids_sha256":"9b005fd461f84a7451b7d950c5dac4626187c3e983e8ef186cb0ac942803b1d9"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"675cb632f8d8a2bb1d48f14a4d927d717d45d10dd3c0cacbe19f6e2cf94455a7","stratum":"16k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":6901.277658999999,"prompt_ms":2177.121,"prompt_tps":4076.943817086877,"decode_tps":62.57826503786097,"input_tokens":8925,"output_tokens":279,"peak_rss_mb":9510.41015625,"verdict":{"state":"Pass"},"prompt_ids_sha256":"d017282343b77dda0b7ab50f3cb163775efea0802dee5ceeed7c6495c76cd8bf"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"c4cf0db8bffc8083c5691e577702ca73a932cb4cde56dd1c9dac360353ee38b1","stratum":"16k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":11322.907527,"prompt_ms":2723.342,"prompt_tps":4089.827865908872,"decode_tps":60.98001204086735,"input_tokens":11187,"output_tokens":512,"peak_rss_mb":9510.41015625,"verdict":{"state":"Fail"},"prompt_ids_sha256":"dc38252b026854e36f129b97bd87960e52b6fcb44f4e545c4860459f962308d2"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"aa3e63ab8090ebf9be4f8d697c9f96d692a2763af91a2ad7bf96d1a26c384025","stratum":"16k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":9924.452185,"prompt_ms":2412.414,"prompt_tps":4036.20605750091,"decode_tps":62.105515607476505,"input_tokens":9786,"output_tokens":449,"peak_rss_mb":9510.41015625,"verdict":{"state":"Pass"},"prompt_ids_sha256":"a2e6135129fe41dfbb6cefed737ccaecbb1acd94d176936ba19a8d4d00969e76"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"ea0520ed2859350cd1372043b3a06ff52b3e2f75fb0195b3d0521d1f23b3e7d6","stratum":"16k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":4662.685694000001,"prompt_ms":2120.054,"prompt_tps":4103.19737138771,"decode_tps":62.853744610730935,"input_tokens":8748,"output_tokens":144,"peak_rss_mb":9510.41015625,"verdict":{"state":"Pass"},"prompt_ids_sha256":"c132e4a7a5ce9599d799aadfe7b680427fb4469c2031225692c05d86ceefb87c"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"54246f96c1844cc9afe81e68e68bd564d60308d58851d0af96c22aeb7059b962","stratum":"16k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":8855.00959,"prompt_ms":2689.161,"prompt_tps":4072.2738430313393,"decode_tps":61.07142971938778,"input_tokens":11000,"output_tokens":362,"peak_rss_mb":9510.41015625,"verdict":{"state":"Pass"},"prompt_ids_sha256":"51ccafe7aeaa0580788c3daf75e8b77c84449cb6f48369075c3a4e317d97ec7a"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"014e7869f90798986ecf651ca3a959bdf86f6369b6ce3d5ff71ecf0fd7cf7417","stratum":"16k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":7763.834532,"prompt_ms":2949.094,"prompt_tps":4055.8218897057873,"decode_tps":60.614150450010825,"input_tokens":12010,"output_tokens":278,"peak_rss_mb":9510.41015625,"verdict":{"state":"Pass"},"prompt_ids_sha256":"9bf4cad8197210b509c2cc01811950c93a723c31ef9acd1dd2d4eb9ced1a48e1"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"2cdf5aadfe826822cbeef1cfad68e8d2d21d6bb99bc84ba02b50ad8224102206","stratum":"16k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":6937.187818,"prompt_ms":2323.249,"prompt_tps":4105.457486476913,"decode_tps":62.96160875839478,"input_tokens":9587,"output_tokens":276,"peak_rss_mb":9510.41015625,"verdict":{"state":"Pass"},"prompt_ids_sha256":"3163837c3c6ce35fc9144f2561444c6a631999d70734909762d7308610035870"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"6aa76be5cbd8ad7332c2acf93046bc20e866b9847dd276b88d4b66abe11949c8","stratum":"16k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":5999.334581,"prompt_ms":2225.718,"prompt_tps":4095.756964718801,"decode_tps":62.87645491115128,"input_tokens":9165,"output_tokens":221,"peak_rss_mb":9510.41015625,"verdict":{"state":"Pass"},"prompt_ids_sha256":"5ef3f3955e3349beecc7b3ead0a588b47ee06838c7a297707417f2fd7c975fc4"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"720dc2dbf89b8f4f37b09af101e95d2b2c11697ff7a5d59d50f131286980e9ab","stratum":"16k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":5447.740785999999,"prompt_ms":3176.226,"prompt_tps":4066.776104723027,"decode_tps":59.629779920631755,"input_tokens":12966,"output_tokens":119,"peak_rss_mb":9510.41015625,"verdict":{"state":"Pass"},"prompt_ids_sha256":"34405e806691b42d660adc2a2cfea20841a3de09c5c209d1aa96f85455b602c6"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"ebb26bc757178bfd19484adb04721a053df335e8059aed67cf8e11eb4d53cca1","stratum":"16k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":7494.7566400000005,"prompt_ms":3354.659,"prompt_tps":4065.0927560744626,"decode_tps":59.32692047486516,"input_tokens":13686,"output_tokens":229,"peak_rss_mb":9510.41015625,"verdict":{"state":"Fail"},"prompt_ids_sha256":"2a4712db6c149f45728ae2b1ed0ed2762692034e6140466b328757593cef7e3d"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"1ee0546fe5e55ded3962edb4c5d8eb0632d49a0d3fcbcb6de9ec2d9936a567f2","stratum":"16k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":5677.533266,"prompt_ms":2215.614,"prompt_tps":4109.018989769878,"decode_tps":62.407838675423406,"input_tokens":9153,"output_tokens":200,"peak_rss_mb":9510.41015625,"verdict":{"state":"Pass"},"prompt_ids_sha256":"4125241c2b414624ad34f56859fa91c135d8d1ffcc58e335cf7051aadd0439c5"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"1c2d4d2a2013dc86953f03eddb0ce4b67ac53e05931b7b18ff6d527d2c880640","stratum":"16k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":7748.241364,"prompt_ms":2707.836,"prompt_tps":4104.753759090285,"decode_tps":61.77746882109869,"input_tokens":11164,"output_tokens":298,"peak_rss_mb":9510.41015625,"verdict":{"state":"Pass"},"prompt_ids_sha256":"e4d7241584af87981056a070fd2ec4e3eee56ef2016c62d5f7f71ab382b92dc9"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"f51557a52249a78f34a126d0646c5108cec49aa8bf9829fbfa487f75f82c7984","stratum":"16k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":5847.774438,"prompt_ms":2036.887,"prompt_tps":4119.521603309364,"decode_tps":63.37067790853104,"input_tokens":8440,"output_tokens":226,"peak_rss_mb":9510.41015625,"verdict":{"state":"Pass"},"prompt_ids_sha256":"44aebbb6cf4173813f2621076bab1a2a912ba2f0a714ee96b98f86cfe61ab844"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"69a9d1096c603ec6d3706769831302fbf6a198a02c189f771a730704506c2941","stratum":"16k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":6584.750290999999,"prompt_ms":3510.195,"prompt_tps":4083.818705228627,"decode_tps":60.727772928971106,"input_tokens":14384,"output_tokens":173,"peak_rss_mb":9510.41015625,"verdict":{"state":"Pass"},"prompt_ids_sha256":"dbd2749149ab437d0166e1de2f12680147b380a10ddb9b038ba1e6759f59d1a0"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"36528c9baa7523120f3683b655e9bd996e7ec070c288fa94b92aa17f033e18c4","stratum":"16k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":5849.272231999999,"prompt_ms":2060.482,"prompt_tps":4133.984184283095,"decode_tps":64.10357758329513,"input_tokens":8567,"output_tokens":224,"peak_rss_mb":9510.41015625,"verdict":{"state":"Pass"},"prompt_ids_sha256":"036ef65a29008fd891473edcdf5366c0e021f70e465cd8fd70aee4f55c392c39"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"7d6b1ea94587f39adc61be730808d638df69dc46fb23c0292d839974de14c4aa","stratum":"16k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":5316.456994,"prompt_ms":2164.247,"prompt_tps":4113.6709442129295,"decode_tps":63.73521620093774,"input_tokens":8952,"output_tokens":185,"peak_rss_mb":9510.41015625,"verdict":{"state":"Pass"},"prompt_ids_sha256":"5640f5c62747622fad6026aa1b84af1b301adde703da627336351517c166ff00"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"25c07e0a68726a522357ec60ea143c91cea4227fe4139c3db6f0c52ac274bb8d","stratum":"16k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":7210.906196,"prompt_ms":3556.34,"prompt_tps":4061.760124172604,"decode_tps":60.78270419300339,"input_tokens":14494,"output_tokens":207,"peak_rss_mb":9510.41015625,"verdict":{"state":"Pass"},"prompt_ids_sha256":"3e4d4daca07a12047b37ab6fa0e74f224e28c54cd87dad01f94457face129e0c"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"cc6f12cb2a5dc9e2cd0cff0eda9338245b06a7b29be77caf7c7ad415dd948bb4","stratum":"16k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":6580.562202,"prompt_ms":2763.868,"prompt_tps":4080.151439938521,"decode_tps":62.51335970993801,"input_tokens":11326,"output_tokens":218,"peak_rss_mb":9510.41015625,"verdict":{"state":"Pass"},"prompt_ids_sha256":"7ef7e31a465dc80e0aa2ce7fa43085eba91d4ec00561981cadf511afc94587fc"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"88ef36ab5267cf425c39c87c20a1e697a4b612b9d36b17c39ca3b51cd24f9a9e","stratum":"16k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":7072.983422,"prompt_ms":2891.378,"prompt_tps":4076.257064970405,"decode_tps":62.49971533530464,"input_tokens":11835,"output_tokens":248,"peak_rss_mb":9510.41015625,"verdict":{"state":"Pass"},"prompt_ids_sha256":"537725b34a695cc189b0d1338ef38a956b5d0250fa0c9c4bd1b5740c766399ce"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"43d42945260ed2fee02c5b9fda7974cf8f66fd65dff63de4e17ea8faf547e878","stratum":"16k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":8836.42298,"prompt_ms":2352.499,"prompt_tps":4114.348188883396,"decode_tps":63.56309680691746,"input_tokens":9728,"output_tokens":397,"peak_rss_mb":9510.41015625,"verdict":{"state":"Pass"},"prompt_ids_sha256":"d9b5939d45f4b6cadbb25f9ce529b2543a1f3da2d3ab3decf3c7abe8c21fed68"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"059c0fd951a748e2ae74924c143d7c8e8d3c0c277645ba032d7f3fe28458b002","stratum":"16k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":9477.452508,"prompt_ms":3766.81,"prompt_tps":4050.376844066996,"decode_tps":59.64770528324975,"input_tokens":15306,"output_tokens":327,"peak_rss_mb":9510.41015625,"verdict":{"state":"Fail"},"prompt_ids_sha256":"698b52cf76abfa36ca28d8f350ef8107c942ea080b91406c1ac66d4b38c385b0"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"0882349afbe288f0c5a3e2112311c022c01875743aabcc1d12afe9ce23093012","stratum":"16k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":7165.476712000001,"prompt_ms":3371.8,"prompt_tps":4068.450086007473,"decode_tps":61.272827379705106,"input_tokens":13767,"output_tokens":213,"peak_rss_mb":9510.41015625,"verdict":{"state":"Pass"},"prompt_ids_sha256":"ff993db7ae7f149d4e3a1d6d7840460f0cd8a26c0eea77c35f6196552a40d4e5"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"bcef4d3206e948203a70d22840d2a5bbbd067648e038d4939ac7913c2d0d977d","stratum":"16k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":7417.447786000001,"prompt_ms":2923.495,"prompt_tps":4113.911602380029,"decode_tps":62.42509878237827,"input_tokens":12076,"output_tokens":264,"peak_rss_mb":9510.41015625,"verdict":{"state":"Pass"},"prompt_ids_sha256":"39836f7486ad00c5fda9e448cf60e59f2860a0307cbf1871f34c46333a1c992c"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"f44f0554c5d06d6d9bd8a87e7c9adef7b5acc16f8dd84ee8990ad95cdb0d7a86","stratum":"16k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":7332.451222,"prompt_ms":2726.48,"prompt_tps":4083.653648660544,"decode_tps":62.956871302730434,"input_tokens":11183,"output_tokens":273,"peak_rss_mb":9510.41015625,"verdict":{"state":"Pass"},"prompt_ids_sha256":"1f020914b5d4e264ef245a02a23947d4b1fd4c90b001bd77a260391e94464e4f"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"d49cd0c3fad3e8f5c978cbcb184e89f2cd360cdf37c1a97d192e6ca1e88aa974","stratum":"16k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":6976.992275,"prompt_ms":3388.739,"prompt_tps":4047.2281872401504,"decode_tps":61.450599862770325,"input_tokens":13764,"output_tokens":206,"peak_rss_mb":9537.60546875,"verdict":{"state":"Pass"},"prompt_ids_sha256":"2ad00e24d8988d50c0203eb5a7be9e633ca84a757788f60888e7586e09d1d1c8"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"6472e6c3bf01349b337af3cf2ddd2fc277b3af86eeb21bf5ca250923f3760dbb","stratum":"16k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":5527.475358000001,"prompt_ms":2128.704,"prompt_tps":4116.589248669613,"decode_tps":63.59321215836994,"input_tokens":8812,"output_tokens":196,"peak_rss_mb":9537.60546875,"verdict":{"state":"Pass"},"prompt_ids_sha256":"27a05690c12bac021035e63d41f559f99635ca5e4f2488ee991d136a21e34989"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"5a466ceed2939ee8936626418aadce90a670d794ce8d1289af467fe01430718d","stratum":"16k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":7998.152018,"prompt_ms":3561.323,"prompt_tps":4068.7126666129416,"decode_tps":60.556811504358045,"input_tokens":14539,"output_tokens":254,"peak_rss_mb":9537.60546875,"verdict":{"state":"Pass"},"prompt_ids_sha256":"3f65be738f0bac2a6d1a61af3c99be128d0245394c8f562f5629e4378a22d17b"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"c8cd063621db62fd4dde1e4e2fe07d67562b8d073e89e6bd35c00b88b2c6d546","stratum":"16k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":6622.61696,"prompt_ms":3225.717,"prompt_tps":4074.132975707416,"decode_tps":61.28070251427344,"input_tokens":13191,"output_tokens":192,"peak_rss_mb":9537.60546875,"verdict":{"state":"Pass"},"prompt_ids_sha256":"7d874c65f58ba21b178b10b378237380b9c35c43a22721cead3d56854f952312"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"1c7345312237ba7892931e444d0b663807feae59083cbc849d13f6e4748d2bd2","stratum":"16k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":6099.543836,"prompt_ms":3583.265,"prompt_tps":4066.961276935979,"decode_tps":60.17565862937043,"input_tokens":14622,"output_tokens":134,"peak_rss_mb":9537.60546875,"verdict":{"state":"Pass"},"prompt_ids_sha256":"6dc5aeb4272351770a57d1673d2f6fceb604770bb1e63363de5dde21a07ecbae"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"0dc43a5ae0a19646ed73c5f0aa0495eda3a3b460efd42e35ab5a0d1fe2002192","stratum":"16k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":6946.149087,"prompt_ms":2101.141,"prompt_tps":4137.27588962378,"decode_tps":63.01326341064356,"input_tokens":8742,"output_tokens":287,"peak_rss_mb":9537.60546875,"verdict":{"state":"Fail"},"prompt_ids_sha256":"5a5c9a9d4cc57194ef9db7c284d1261f917f9c29a19a53489aa80dac05004407"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"5189d6142784fae84f7d1634633cf93f49b3d2528950f68906f1c5ca9e8dfc5a","stratum":"16k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":9782.886692,"prompt_ms":3997.949,"prompt_tps":4039.3211619257777,"decode_tps":59.002223680136495,"input_tokens":16198,"output_tokens":328,"peak_rss_mb":9537.60546875,"verdict":{"state":"Pass"},"prompt_ids_sha256":"490004b669067449f5c7e62e6e309d0ee30ed55781d9c2b9d140e4ac5dbb38c8"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"28c93295ce9295f699d1ab17eecef2d2ef643f6ec8434694ee70fa20ad70c0b9","stratum":"16k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":5651.255816,"prompt_ms":3352.734,"prompt_tps":4036.407302219621,"decode_tps":60.29823507065948,"input_tokens":13582,"output_tokens":121,"peak_rss_mb":9537.60546875,"verdict":{"state":"Pass"},"prompt_ids_sha256":"8c0ae16658161f3d1698b96183289aed7fd75bcda9877263efda1062664d7497"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"0f1bc13a198826247cef2fe67851805b3a7046ac9d291b02b47581866c6d896c","stratum":"16k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":6059.209978,"prompt_ms":2439.023,"prompt_tps":4080.7323260174257,"decode_tps":62.394006279576274,"input_tokens":10002,"output_tokens":206,"peak_rss_mb":9537.60546875,"verdict":{"state":"Pass"},"prompt_ids_sha256":"6a1e197e1c7efd9196871bec225a755aca2bd8ddfe47e0d46af2f3e0b14f1419"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"4e7c51716440d0055d37c6d118eca15cfb84381cb2984b437ddbf7920cee4651","stratum":"16k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":4590.068313,"prompt_ms":2509.312,"prompt_tps":4059.2799938788007,"decode_tps":61.10951547376263,"input_tokens":10235,"output_tokens":111,"peak_rss_mb":9537.60546875,"verdict":{"state":"Pass"},"prompt_ids_sha256":"f45b53c01e8955c24c49d0f24260bd34e4258513c7701ab5e462911006f37e99"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"d2107848512e9694b1e838662ca844b67f962cdc8d3826265b75f6b002d2982d","stratum":"16k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":6323.299934,"prompt_ms":2082.76,"prompt_tps":4106.5701280992525,"decode_tps":63.219274810205285,"input_tokens":8602,"output_tokens":255,"peak_rss_mb":9537.60546875,"verdict":{"state":"Pass"},"prompt_ids_sha256":"5b324a8a6e5df50a926268dd114f6374ee8d731ed147592868a109e4fa8cbc33"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"f0466d3c53c99fb39ebfe065673d3b9bdd70fdf30ee02898eb6e886201e8d4a3","stratum":"16k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":8015.988069000001,"prompt_ms":3508.039,"prompt_tps":4025.040770641375,"decode_tps":59.65829134442416,"input_tokens":14169,"output_tokens":256,"peak_rss_mb":9537.60546875,"verdict":{"state":"Pass"},"prompt_ids_sha256":"8600d8498d868371fc947edc7db322d590a85502079ad419abb0df4a4727df72"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"f570aee8b67fbb453024af27fe758441e49aa16bc6b09147772a0d43a92115ec","stratum":"16k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":5637.9935049999995,"prompt_ms":3027.654,"prompt_tps":4047.027830789119,"decode_tps":59.79052214711282,"input_tokens":12302,"output_tokens":137,"peak_rss_mb":9537.60546875,"verdict":{"state":"Pass"},"prompt_ids_sha256":"8b2497a464c07d512e5a9659dcb4f07a47d97fc06c111ae926ed313355ce34d3"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"409826d3610e79d93d92ba45ad55d943307796f6b4218ad327e872390681ef26","stratum":"16k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":9466.520773999999,"prompt_ms":4056.007,"prompt_tps":3998.513809271039,"decode_tps":57.83885037540355,"input_tokens":16267,"output_tokens":296,"peak_rss_mb":9537.60546875,"verdict":{"state":"Pass"},"prompt_ids_sha256":"cc546fdfce8b6b26c4668049fff094491f2f8bee3cf437f37a5eb735bdb27740"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"b53f7c2d4a456e341e0f1e013eede2c53dbe5230bb2ef59c8437a9dbd9b1126f","stratum":"16k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":7719.696202,"prompt_ms":3281.623,"prompt_tps":4054.7009818007737,"decode_tps":59.8676850852247,"input_tokens":13355,"output_tokens":246,"peak_rss_mb":9537.60546875,"verdict":{"state":"Pass"},"prompt_ids_sha256":"58c397b59aee26d82ba43edafd279603a863fc48170cd0fb176fdf91ffc3a98f"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"fd326fe7d4bacc97fcf64369c18bd23e1a718b755ec9d720b6010b7bea96f970","stratum":"16k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":4969.119675,"prompt_ms":2025.862,"prompt_tps":4124.170353163246,"decode_tps":62.25329772696232,"input_tokens":8404,"output_tokens":166,"peak_rss_mb":9537.60546875,"verdict":{"state":"Pass"},"prompt_ids_sha256":"450e403e1bc170e1470b9b1d2a97c64aa904b0167319be75d1fa671f5a99f4dd"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"fa7720559e61059c92e06f6156fc0da7ae0a6367cd814f36465e85f7d04800e7","stratum":"16k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":9879.578428,"prompt_ms":3912.593,"prompt_tps":4034.153309582673,"decode_tps":58.47790419393374,"input_tokens":15833,"output_tokens":335,"peak_rss_mb":9537.60546875,"verdict":{"state":"Pass"},"prompt_ids_sha256":"7d9efdebce284cc4c5ce1d3a24fd62794d1ac2228510aa0c228e9ee283ab2cd9"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"f8f4e9f89c28ef4f440267322b74bbf68eed96ea240d6a883bec0d50f2dbe576","stratum":"16k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":6153.750053,"prompt_ms":2299.402,"prompt_tps":4131.50897494218,"decode_tps":62.34774138156875,"input_tokens":9549,"output_tokens":219,"peak_rss_mb":9537.60546875,"verdict":{"state":"Pass"},"prompt_ids_sha256":"5d5cfe1a4a40e808aca7241c0e6a96d796456776e3fc689c00288b819bcc28cd"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"ca612af4633a006928c0400dcf7a90a769c47290edc85241d3ba156583b2f6b7","stratum":"16k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":5028.767685,"prompt_ms":2232.18,"prompt_tps":4124.667365535038,"decode_tps":62.379583714980576,"input_tokens":9256,"output_tokens":159,"peak_rss_mb":9537.60546875,"verdict":{"state":"Pass"},"prompt_ids_sha256":"3a050649a54bd19a4c5b2fc11ca233aacbd3f9ea410506f99d0d65a893a023cd"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"f80dddb74b021666f2f30078ee85f6a231fb529e1ed67f051a9b662ffb17b4e6","stratum":"16k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":6653.427907,"prompt_ms":2232.869,"prompt_tps":4122.49890163731,"decode_tps":62.277447946832304,"input_tokens":9254,"output_tokens":261,"peak_rss_mb":9537.60546875,"verdict":{"state":"Pass"},"prompt_ids_sha256":"29ad40656fdbdd17ccccd5a79690a1c5856bd2c5e066028c0776ebd8fcc59873"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"fa49d36e7eb9eed429d0d549464654396d8fe8fc5a0be7b4e667bddb37922c36","stratum":"32k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":8516.28718,"prompt_ms":6384.795,"prompt_tps":3858.228807659447,"decode_tps":54.907140937655576,"input_tokens":24683,"output_tokens":105,"peak_rss_mb":9537.60546875,"verdict":{"state":"Pass"},"prompt_ids_sha256":"10163b836e6b21a8d1943f429085e148470b19b5c857f7822b5661509f4d11d9"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"3b51ad3f76c868adaa04840a77eed635ade302cec8c85bf0d3795526ab237fa7","stratum":"32k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":8645.989842,"prompt_ms":5038.111,"prompt_tps":4001.4997684648074,"decode_tps":56.99461562798837,"input_tokens":20209,"output_tokens":177,"peak_rss_mb":9537.60546875,"verdict":{"state":"Pass"},"prompt_ids_sha256":"2d0742ae98d285c41c4c84ef725a4fdd40c2b06112ad0f5f6709cd4fd9da985b"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"f8815530d0c7dc219ea3341c95432e757c3ab7fb0308bd2146f2d90b840f726b","stratum":"32k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":7100.824191,"prompt_ms":4165.795,"prompt_tps":4048.446935098823,"decode_tps":58.85041597435691,"input_tokens":16914,"output_tokens":151,"peak_rss_mb":9537.60546875,"verdict":{"state":"Fail"},"prompt_ids_sha256":"dd1693371bba23714bd583154ef11f1d6a58342d9feb0fbc8b3f89281a3c430a"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"717524cb0e8f36fac7a56af480fe9ecfe30ed9e797cefb9908a1c663d247b0c7","stratum":"32k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":8080.288732,"prompt_ms":4761.256,"prompt_tps":4004.195531599225,"decode_tps":58.600543192564494,"input_tokens":19114,"output_tokens":171,"peak_rss_mb":9537.60546875,"verdict":{"state":"Pass"},"prompt_ids_sha256":"fe2fbf0e6c29ac51fe639ae75e221ab52400551682caf9dcc5f560088f5991f0"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"03c15ba04bbacf6b92b3ab9bd18cdca26071fbb4a09dc4de9bef2ad078ba580f","stratum":"32k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":9363.195019,"prompt_ms":5560.145,"prompt_tps":3965.364212623951,"decode_tps":57.07866676003246,"input_tokens":22097,"output_tokens":195,"peak_rss_mb":9537.60546875,"verdict":{"state":"Pass"},"prompt_ids_sha256":"1d1e1149ebcc7a12c4464cd97630fc968780b0fa6703886e8550b6cd7ab8f27c"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"351ae6cc9897a62bc51dc629ead6161e8242c49a7cfbeabdd8f05e9c0c3cdaea","stratum":"32k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":10050.933352,"prompt_ms":4872.835,"prompt_tps":4004.0346122944857,"decode_tps":58.295379508220186,"input_tokens":19560,"output_tokens":276,"peak_rss_mb":9537.60546875,"verdict":{"state":"Fail"},"prompt_ids_sha256":"d4c0dfa77150a8a80c4404832981c8f2c1f4eca29ffbec592e3d21b0ba30e535"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"7bb9e2edd88d122d0553e7776b5ac605cc7da369d4410dba027a27df9290f060","stratum":"32k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":8749.079531,"prompt_ms":5319.913,"prompt_tps":3991.6066296572894,"decode_tps":57.63655735157659,"input_tokens":21284,"output_tokens":172,"peak_rss_mb":9537.60546875,"verdict":{"state":"Pass"},"prompt_ids_sha256":"162822cb75db0d188fb71a133b6480a9768b7b1582914953aaf3b0494125830a"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"d667fffd336a3dd90c6245ba9527ce0fb0448c554d576a089e347230a6ecb2c0","stratum":"32k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":8063.780111,"prompt_ms":4764.902,"prompt_tps":4016.2420968993697,"decode_tps":58.10660672118419,"input_tokens":19186,"output_tokens":167,"peak_rss_mb":9537.60546875,"verdict":{"state":"Pass"},"prompt_ids_sha256":"544ecff0b9504efcc6965bfe96d614b95a0c9a33f99d1f021276789c0ac5f294"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"38a1dbfd07591ad9a6e4b27e7916e661614e6cd7231082e0e4e2601297e5fe7f","stratum":"32k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":8550.237493999999,"prompt_ms":4855.904,"prompt_tps":4011.611432186468,"decode_tps":58.03477915499548,"input_tokens":19529,"output_tokens":193,"peak_rss_mb":9537.60546875,"verdict":{"state":"Pass"},"prompt_ids_sha256":"3fa656a12db52afb3add603a41a8269708e92167a90663f8d3cc7548b477f08c"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"3d8a6474dcfe17a53ec35178358b968c69405032d23e581b7a700fe39982960a","stratum":"32k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":8167.5187479999995,"prompt_ms":4745.905,"prompt_tps":4001.13360886912,"decode_tps":58.46883998467535,"input_tokens":19038,"output_tokens":182,"peak_rss_mb":9537.60546875,"verdict":{"state":"Pass"},"prompt_ids_sha256":"64b5124f7d02001c4cf5434c7f64ad4222b626650acc205dded43af97165d1eb"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"bf2811a3edb86d54d69166c93ec73ebcf6ff64b5556d9d7d9f090726e8fee9a2","stratum":"32k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":9291.083892999999,"prompt_ms":5456.066,"prompt_tps":3976.4914867232183,"decode_tps":56.68656708726381,"input_tokens":21745,"output_tokens":199,"peak_rss_mb":9537.60546875,"verdict":{"state":"Pass"},"prompt_ids_sha256":"ddea598580c58fd52253867387479e86617b6fc8bef87e6b9a6bf70c78966051"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"b515045eff1517e48974319c1c62e87bc8ef658b739d6e365b934ee04f6667f0","stratum":"32k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":12399.037201000001,"prompt_ms":7797.677,"prompt_tps":3826.0112595071582,"decode_tps":52.64293748543982,"input_tokens":29883,"output_tokens":222,"peak_rss_mb":9537.60546875,"verdict":{"state":"Pass"},"prompt_ids_sha256":"cce8cd881a53898e5c7a62a4f4ca518a141c8988a66f1f84e53c4982ca7f1560"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"8f82c371a972bb3e1d51e43968dccb02cfdaa9e94eaabb04b2b6202addef14fa","stratum":"32k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":9722.466473,"prompt_ms":6408.665,"prompt_tps":3751.4833432547966,"decode_tps":55.5284695644914,"input_tokens":24091,"output_tokens":142,"peak_rss_mb":9537.60546875,"verdict":{"state":"Pass"},"prompt_ids_sha256":"4db639b147d20927495dfc8f015d002c34a1a8dbb361f9a395d0f540f44c1f25"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"304c3c29ecc274958e1042ea4f3e6fb4108bdc048767ad229c669ae8025e23f7","stratum":"32k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":8719.175781,"prompt_ms":4848.917,"prompt_tps":3951.1915753558983,"decode_tps":57.5694798856555,"input_tokens":19208,"output_tokens":201,"peak_rss_mb":9537.60546875,"verdict":{"state":"Pass"},"prompt_ids_sha256":"7d297d57bb08bd16d5a0bfd093357da4f17a4f2954bc5aebe4e292401877ef3a"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"c9d4c24c8671432ccf62fc9a693dd62fc3430da536f4b20f53abb64ba65e010b","stratum":"32k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":8775.016889999999,"prompt_ms":5051.59,"prompt_tps":3652.711324553259,"decode_tps":56.946101914851575,"input_tokens":18501,"output_tokens":184,"peak_rss_mb":9537.60546875,"verdict":{"state":"Pass"},"prompt_ids_sha256":"2b348c36ba02c98acdd3b0264d0bf7c6abc97c4aa1e0b5d5a57e9a76895b8ff7"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"7173ebdf8174e7e3ac987fc47c136b866b83cdf52d03a1c1fce2fb4ed8192ded","stratum":"32k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":14014.510967,"prompt_ms":8294.827,"prompt_tps":3784.889063991329,"decode_tps":51.66889587354785,"input_tokens":31444,"output_tokens":274,"peak_rss_mb":9537.60546875,"verdict":{"state":"Pass"},"prompt_ids_sha256":"0f80c00a890f530a76c0f115c409d1e0b9ef5735899f43cac2e7a695fba889c4"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"0f963ad6b87e0d3342e5334a4115c16eaf148f32c1e0adc968706c77b5191039","stratum":"32k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":12366.853047999999,"prompt_ms":7421.26,"prompt_tps":3795.9861263451216,"decode_tps":53.55579444964295,"input_tokens":28220,"output_tokens":234,"peak_rss_mb":9537.60546875,"verdict":{"state":"Pass"},"prompt_ids_sha256":"4c4b6ae7e6b58fe247abcafa33c758ba870176b098d102db06af0ac0686fd4b0"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"74f801a07d6f9f7366f719b4b2302c99b8c74f8eb9f099cb0088470f7ed180e1","stratum":"32k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":8469.893875,"prompt_ms":5805.578,"prompt_tps":3912.616452659838,"decode_tps":56.61430908399417,"input_tokens":22764,"output_tokens":125,"peak_rss_mb":9537.60546875,"verdict":{"state":"Pass"},"prompt_ids_sha256":"70c486262966bf7809098d46926cd98aaf3f6670ceb98cb3f34cb817ffbf760e"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"430c7c0f69397873ade236dd3f5db7663f2f1fb4f40eb879109629b59e0b3ce3","stratum":"32k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":11253.19017,"prompt_ms":5548.901,"prompt_tps":3929.4267459448274,"decode_tps":56.53649440231598,"input_tokens":21853,"output_tokens":298,"peak_rss_mb":9556.44921875,"verdict":{"state":"Pass"},"prompt_ids_sha256":"d344c43778457ceeaf9fcafd3565a371a4836ae8c8fd3ac08de51d30e2f26f57"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"8003dd98e1763cde69c838d89b0bab41aaa2622c580b9e3de39b8b3c1e182f85","stratum":"32k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":8908.934066,"prompt_ms":6351.454,"prompt_tps":3921.936614828667,"decode_tps":54.42072341147509,"input_tokens":24959,"output_tokens":120,"peak_rss_mb":9562.99609375,"verdict":{"state":"Pass"},"prompt_ids_sha256":"83323d291c7ce06a7d2db0bdee88fcedd25ebe222a3c6653fc9c2257e2379150"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"b931d437202ef5adb659042b0e7255fd1754602c9d52039ef36d831ad232674b","stratum":"32k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":8501.712049,"prompt_ms":4907.3,"prompt_tps":4006.072585739613,"decode_tps":58.051253922774265,"input_tokens":19708,"output_tokens":186,"peak_rss_mb":9560.734375,"verdict":{"state":"Pass"},"prompt_ids_sha256":"0adbf65326e98c30c97a86ac1cd456158dddac02a101d1cc351a815782a5cdba"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"084e31008f5a675e5153246d96a5142e0adc8215e66f598266bbb76a10be6db2","stratum":"32k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":12493.09165,"prompt_ms":7797.62,"prompt_tps":3864.512505097709,"decode_tps":53.32755849975203,"input_tokens":30183,"output_tokens":231,"peak_rss_mb":9560.734375,"verdict":{"state":"Pass"},"prompt_ids_sha256":"008c43328b72985f0930804bf4dba749371420359c750afa424dd5e6d3047e5a"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"14b3251ddf800d7bdfdb7a775c2281eba042c909e8a46700689825014ded27de","stratum":"32k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":9866.019241,"prompt_ms":4423.402,"prompt_tps":4050.728376032746,"decode_tps":59.70053388746408,"input_tokens":17967,"output_tokens":291,"peak_rss_mb":9560.734375,"verdict":{"state":"Pass"},"prompt_ids_sha256":"cc964f6596ba0bf32e87225ab3a6e88f89eac8e4ba6bc237cab69ecee8f5ee60"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"ca55757f48d1d3f049d77fd86f25cc77c581af66809d307a89d2a5b7bb14d278","stratum":"32k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":9627.976267999999,"prompt_ms":5268.323,"prompt_tps":3958.185555441456,"decode_tps":57.07169484506954,"input_tokens":20902,"output_tokens":224,"peak_rss_mb":9560.734375,"verdict":{"state":"Pass"},"prompt_ids_sha256":"36511a899244f7383295611d61080169d7b333ce858c223384ee57588db0cc22"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"5bc59b96abf0b69ee35dd697a3d38ea685e54eb0cd1544a880e08b0f64c3905f","stratum":"32k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":8306.359154000002,"prompt_ms":4649.895,"prompt_tps":4005.2517314907104,"decode_tps":58.244139410987934,"input_tokens":18673,"output_tokens":193,"peak_rss_mb":9560.734375,"verdict":{"state":"Pass"},"prompt_ids_sha256":"01878ef6e12e6090d8bf4d912ac05afd77064731e6d2c22d029d8e95916d2ef8"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"4608b8ddaa9de754634659da66dee0e3d59c7097dfbebe7ef8ff664f7a8e2a3c","stratum":"32k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":12172.25981,"prompt_ms":5092.086,"prompt_tps":3943.766857040513,"decode_tps":57.01893313674805,"input_tokens":20131,"output_tokens":381,"peak_rss_mb":9560.734375,"verdict":{"state":"Pass"},"prompt_ids_sha256":"ea75c65f1593e8994329bf6520bd50cfd827175845144b2decd4a39d6a816003"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"dc10ee3c8ec1879df8935451288b060dcba94ff3db9fb2d0e714e938b65c15ca","stratum":"32k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":10216.518845999999,"prompt_ms":6901.583,"prompt_tps":3812.023995074753,"decode_tps":52.48206950095499,"input_tokens":26358,"output_tokens":151,"peak_rss_mb":9560.734375,"verdict":{"state":"Pass"},"prompt_ids_sha256":"60fa312565da30886df98a128049dec4b80d7fbab7adb34688ce8d3b13131f87"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"841dd09f216b308d2acb091c53e79c3c22c6dc9c70d99ddd5aaa007e65fb8f0f","stratum":"32k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":7840.328517999999,"prompt_ms":4714.402,"prompt_tps":3875.358953267032,"decode_tps":56.70356204688833,"input_tokens":18319,"output_tokens":145,"peak_rss_mb":9560.734375,"verdict":{"state":"Pass"},"prompt_ids_sha256":"693837d52c6a4861282f0d455277e48fac6d7e207bdaa4e7a172c2acf8726055"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"24bc258f12dfbb5600afabdff2efa2edeaf9efb4c7e13dae0bb627e6c0add649","stratum":"32k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":10964.886054,"prompt_ms":5963.427,"prompt_tps":3819.951179078741,"decode_tps":54.64582378471359,"input_tokens":22829,"output_tokens":254,"peak_rss_mb":9560.734375,"verdict":{"state":"Pass"},"prompt_ids_sha256":"0662c740da8b7967cc8500d02856b1bb61b045eb9df620e43bddc27d9f806dfc"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"78a121fb8ef6ffe62c572f12c3113443d88b19491cd0db0981294e8ed29d38bb","stratum":"32k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":9444.746093,"prompt_ms":4830.465,"prompt_tps":3848.6977961749026,"decode_tps":56.40845241387715,"input_tokens":18640,"output_tokens":238,"peak_rss_mb":9560.734375,"verdict":{"state":"Pass"},"prompt_ids_sha256":"ba43eb4396f1eb5beb8d13c853dad39cd3e1cf8d77a0732070175d726fd6b01c"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"c0b75e2f0a6bcd1a3341d0cbb4e96aa16449934b09e961f790c377d972340d08","stratum":"32k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":9835.550787,"prompt_ms":5589.164,"prompt_tps":3810.587773055148,"decode_tps":55.82273007220264,"input_tokens":21347,"output_tokens":214,"peak_rss_mb":9560.734375,"verdict":{"state":"Pass"},"prompt_ids_sha256":"957b6f511e3f28dc0e7f71f4f420374158278789df3325c0cb389818c1e26247"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"21a87ab49fd3587febc4c89bb82872bb6967bf0a9492abc590112d5f097f1bd5","stratum":"32k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":10191.866788000001,"prompt_ms":6170.205,"prompt_tps":3797.4427105744458,"decode_tps":54.06420164697489,"input_tokens":23480,"output_tokens":200,"peak_rss_mb":9560.734375,"verdict":{"state":"Pass"},"prompt_ids_sha256":"7b8e789020dabaf591806d9d9bf0c6232547b067051bbd4cb26f58de384c2ce8"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"0534621eb0e2215a3c13b22ebf197a2a472f4306bf24f0e380154a527dbc1463","stratum":"32k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":9597.427445,"prompt_ms":5944.119,"prompt_tps":3837.911051242413,"decode_tps":54.35443752216237,"input_tokens":22862,"output_tokens":169,"peak_rss_mb":9560.734375,"verdict":{"state":"Pass"},"prompt_ids_sha256":"e49800ebb35f74199fe8a55797fc6e015fb00fd0fff65e78dd6f91889e9f5ff3"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"add564d1303cf5966124647b302705301dcd4528a1caad315f4d86aa3485e61a","stratum":"32k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":10953.84792,"prompt_ms":6146.866,"prompt_tps":3804.7030795855967,"decode_tps":54.666391512496055,"input_tokens":23436,"output_tokens":241,"peak_rss_mb":9560.734375,"verdict":{"state":"Pass"},"prompt_ids_sha256":"d501682a3b448ff13e405df0b3c77e50f0da19852d59188afedbf391ff330a0c"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"756c7e34ba781b2a93fb5ba02710dfea9f422786e59c39eb6e2b38e11ad4bf4a","stratum":"32k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":10525.505151000001,"prompt_ms":4832.904,"prompt_tps":3885.8624131578026,"decode_tps":57.09220218226321,"input_tokens":18829,"output_tokens":299,"peak_rss_mb":9560.734375,"verdict":{"state":"Pass"},"prompt_ids_sha256":"d6675b3bdd9d86978cd61242f476ccbc7623c2446ae06132244a6f42efbc7255"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"40d8b0a4203d03441d9dcc41d1b76704f65ec7ea5ebd165cac2457a566b70162","stratum":"32k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":8025.339755999999,"prompt_ms":4548.729,"prompt_tps":3909.66355656712,"decode_tps":57.18036746391347,"input_tokens":17833,"output_tokens":176,"peak_rss_mb":9560.734375,"verdict":{"state":"Pass"},"prompt_ids_sha256":"0200f2f9a1370d6f38cae62051efcd98e40658a6fc1ec422e8b37f6701ce5412"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"3db9e737af60c9d2537ac8d8b93aeaffc7b6cc68294a49978e5c870ffb36eb4f","stratum":"32k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":8955.034314,"prompt_ms":5083.05,"prompt_tps":3870.9042799106833,"decode_tps":56.19146023992915,"input_tokens":19725,"output_tokens":202,"peak_rss_mb":9566.1953125,"verdict":{"state":"Pass"},"prompt_ids_sha256":"fffd8b1255cac61466278bf8094821380a784a6aefa5c63410027ecce55429ed"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"c354c595f4a8ea1b09659b55a5ce312a495e4eb6400c77183cd34cca3cb8327b","stratum":"32k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":8595.033956,"prompt_ms":4822.834,"prompt_tps":3870.545824301645,"decode_tps":56.67790470411207,"input_tokens":18716,"output_tokens":185,"peak_rss_mb":9563.9453125,"verdict":{"state":"Pass"},"prompt_ids_sha256":"05dadf955da5b93f28985b714df19e2ef0206facca769a7ffa72479563a4975a"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"c42dfb333a42fbec1745318326627fa05d41440da37c5d2d57304daf1e4bb3b6","stratum":"32k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":8203.874284,"prompt_ms":5218.12,"prompt_tps":3837.397376833036,"decode_tps":55.49554767673954,"input_tokens":20073,"output_tokens":152,"peak_rss_mb":9563.9453125,"verdict":{"state":"Pass"},"prompt_ids_sha256":"80097ce1ced3c4a30308e27f670ccbf58abb86ff4f9385521228aac8db4ed868"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"8136fdc9902571f25cbf93bf14de71435aea774d50ab0fb50ba2196f422389af","stratum":"32k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":9696.474744,"prompt_ms":5383.867,"prompt_tps":3823.4599777446206,"decode_tps":55.3358287280761,"input_tokens":20634,"output_tokens":214,"peak_rss_mb":9563.9453125,"verdict":{"state":"Pass"},"prompt_ids_sha256":"a3c875c6ba4ab5c35526ae9bc83a120f426b85c20fe3a644c37dd3cb99b85be3"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"a267f9989c8533bce8be2d300c85a8509c45a80b1fe16836591ae91341138b2a","stratum":"32k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":12742.475853,"prompt_ms":7867.873,"prompt_tps":3713.964371311027,"decode_tps":50.671452303828694,"input_tokens":29270,"output_tokens":229,"peak_rss_mb":9563.9453125,"verdict":{"state":"Pass"},"prompt_ids_sha256":"52b26517747f4815b31f168051fd64d5054550a410ec132c26ad9f221325ae4b"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"60d2f019ddec1435b5b1230d70553a3833405b30dae55c8dbc8f6bcc1f0f6487","stratum":"32k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":8116.781141,"prompt_ms":4918.9,"prompt_tps":3863.872003903312,"decode_tps":55.798900761655,"input_tokens":19055,"output_tokens":145,"peak_rss_mb":9563.9453125,"verdict":{"state":"Pass"},"prompt_ids_sha256":"926956c06005c8e58a90a81567de45b1b9f4cf512f4714410a8c2bdff97eaf09"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"4be2763e70a83f3b538045b385bf920c45f0ab971c6641e4573f745b32673998","stratum":"32k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":10530.318944999999,"prompt_ms":4397.431,"prompt_tps":3876.5815768342927,"decode_tps":56.438272947801785,"input_tokens":17096,"output_tokens":331,"peak_rss_mb":9563.9453125,"verdict":{"state":"Fail"},"prompt_ids_sha256":"2209238bca60b627f187342a3fa75e2cc850f42325091d4f4918c6c267a9a260"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"4a66cbc27f5a915eaa9fed54adca1621f3d27e77b95dee234328d0114d722209","stratum":"32k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":13397.462179,"prompt_ms":7277.965,"prompt_tps":3721.644717994659,"decode_tps":52.91975087764428,"input_tokens":27135,"output_tokens":303,"peak_rss_mb":9563.9453125,"verdict":{"state":"Pass"},"prompt_ids_sha256":"7d7c836939bcc09cc2d8280e28b6a392d04a2145d05417cf9d72dd860d4e04fe"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"f552e516d8a086ee988e3c5488a31467a2bb6b35b277aecfb98ad15c96d6d3da","stratum":"32k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":9646.609543,"prompt_ms":4989.003,"prompt_tps":3819.199948366437,"decode_tps":56.838639736698056,"input_tokens":19103,"output_tokens":234,"peak_rss_mb":9563.9453125,"verdict":{"state":"Pass"},"prompt_ids_sha256":"c2c806c56c4d144111f554ef8f5f933837eea605a07ee70ebb09c16490c92ca1"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"5c2a849b4a443f183000bbd6996f84b95d3df274e267f4a0b9be650a53080e75","stratum":"32k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":9763.336777999999,"prompt_ms":6359.029,"prompt_tps":3718.020471364417,"decode_tps":55.30467988875666,"input_tokens":23692,"output_tokens":165,"peak_rss_mb":9563.9453125,"verdict":{"state":"Pass"},"prompt_ids_sha256":"000dbf6068631c787dc66205d5a9ad463a2c755eb612acc7ece318bff11f5bfc"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"a2b4707bfaf04f8599408ccfb45a44703a000f71a094a68ba1759061b2a43672","stratum":"32k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":8963.141489,"prompt_ms":5156.601,"prompt_tps":3780.2032773138744,"decode_tps":57.27501346697111,"input_tokens":19542,"output_tokens":196,"peak_rss_mb":9563.9453125,"verdict":{"state":"Pass"},"prompt_ids_sha256":"fa651c50ada370ebabdf777b8b3eac38d443ada3b2e4762702cf325ab1421721"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"3ab4ac140f83056d2501ee334f17e3ee310e217094d568b4fec0c75a1a30dce9","stratum":"32k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":11190.229583,"prompt_ms":5954.12,"prompt_tps":3766.635539760704,"decode_tps":55.99195872521656,"input_tokens":22476,"output_tokens":271,"peak_rss_mb":9563.9453125,"verdict":{"state":"Pass"},"prompt_ids_sha256":"a4b4fb3bd5fe96a03ed1ee01a571c200b6a1ab2e777a7638743f0a24ae89a56b"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"bf6a52843bdf826313ab90adf25c9d7a253496a9a3c5a46c08ef62ef5249470d","stratum":"32k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":10271.290129,"prompt_ms":5626.246,"prompt_tps":3768.942915044952,"decode_tps":56.151664694617025,"input_tokens":21254,"output_tokens":237,"peak_rss_mb":9563.9453125,"verdict":{"state":"Pass"},"prompt_ids_sha256":"29750ae08524616eb9b008167fc5e1577e4b6f505976a9986f03dac0a1eb099b"} +{"schema":"review-replay-receipt-v1","replay_version":"review-replay-v2","set_sha":"a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632","diff_sha256":"4c7ad86de1bf01701febc9297c4271016e389826ac06ac0fdc766c598c4ba789","stratum":"32k","engine":"llama_cpp","engine_version":"llama.cpp d1d3c3396aa13a5f239109a822666c4870490ad5","apr_tag":"untagged-96eb79f51","cell":"gx10-cuda","gguf_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","wall_ms":9343.915381,"prompt_ms":5158.409,"prompt_tps":3789.734392910683,"decode_tps":56.77595727462611,"input_tokens":19598,"output_tokens":214,"peak_rss_mb":9563.9453125,"verdict":{"state":"Pass"},"prompt_ids_sha256":"36e696fa27b6f800c61cc29b6ce76dc253720bc7ae8a736483bb1a628c4bf41c"} diff --git a/evidence/prm-001/s1-v2/96eb79f51/speed-block.json b/evidence/prm-001/s1-v2/96eb79f51/speed-block.json new file mode 100644 index 0000000000..a69b03b37b --- /dev/null +++ b/evidence/prm-001/s1-v2/96eb79f51/speed-block.json @@ -0,0 +1,57 @@ +{ + "replay_version": "review-replay-v2", + "set_sha": "a4910a65ea53e520841b164876d34dead653771226f8a740e6d3408b08b30632", + "apr_tag": "untagged-96eb79f51", + "cell": "gx10-cuda", + "gguf_sha256": "00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4", + "n_items": 200, + "p50_s": 26.607406881, + "p95_s": 69.357750129, + "p95_ci": [ + 59.489092600000006, + 82.555173866 + ], + "queue_budget_p95_s": 601.590268351, + "queue_budget_lane": "antigravity", + "within_budget": true, + "ttft_ms": { + "apr": null, + "llama_cpp": 1900.281, + "ratio": null + }, + "prefill_tps": { + "2k": { + "apr": null, + "llama_cpp": 3695.8528371964057, + "ratio": null + }, + "8k": { + "apr": null, + "llama_cpp": 4090.4108679044743, + "ratio": null + }, + "16k": { + "apr": null, + "llama_cpp": 4076.943817086877, + "ratio": null + }, + "32k": { + "apr": null, + "llama_cpp": 3864.512505097709, + "ratio": null + } + }, + "decode_tps": { + "apr": null, + "llama_cpp": 63.52869487427154, + "ratio": null + }, + "peak_rss_mb": { + "apr": 8419.2890625, + "llama_cpp": 9566.1953125, + "ratio": 0.8801084221538572 + }, + "parse_rate": 1.0, + "verdict_identity_vs_prev_tag": null, + "prompt_ids_sha": "f3c201362a6287c9c665509b8e8832e437131eddb206e0202010debe131dc0fa" +} diff --git a/scripts/check_append_only_ledgers.sh b/scripts/check_append_only_ledgers.sh index ba71e0565a..3bfc402396 100755 --- a/scripts/check_append_only_ledgers.sh +++ b/scripts/check_append_only_ledgers.sh @@ -29,6 +29,13 @@ usage() { printf 'check_append_only_ledgers.sh [--self-test|--help]\n'; } # attr_union PATH -> 0 if git resolves merge=union for it attr_union() { [ "$(git check-attr merge -- "$1" 2>/dev/null | sed 's/.*: //')" = union ]; } +# attr_golden PATH -> 0 if .gitattributes declares it `audit-golden`: a +# single-writer file under docs/audits/ that is NOT a ledger (#4354: the review +# corpus, the #4026 oracle captures). Declared, never inferred; never union. +# Only the bare form declares it: `audit-golden=true`, `=false` and +# `-audit-golden` are NOT golden (R5), so such a file must still be union. +attr_golden() { [ "$(git check-attr audit-golden -- "$1" 2>/dev/null | sed 's/.*: //')" = set ]; } + measure() { local rc=0 n_ledger=0 n_other=0 f printf '=== append-only ledgers must be union-merged; goldens must NOT be ===\n' @@ -36,11 +43,18 @@ measure() { while IFS= read -r f; do [ -n "$f" ] || continue n_ledger=$((n_ledger + 1)) - if attr_union "$f"; then + if attr_golden "$f"; then + if attr_union "$f"; then + printf 'FAIL %s is declared audit-golden AND union-merged. Pick one.\n' "$f" + rc=1 + else + printf 'ok golden %s\n' "$f" + fi + elif attr_union "$f"; then printf 'ok union %s\n' "$f" else printf 'FAIL %s is an append-only ledger and is NOT union-merged.\n' "$f" - printf ' Add it to .gitattributes, or move it out of docs/audits/.\n' + printf ' Add it to .gitattributes, declare it audit-golden, or move it out of docs/audits/.\n' rc=1 fi done < <(git ls-files "$LEDGER_GLOB") @@ -79,6 +93,10 @@ self_test() { "$(attr_union docs/audits/impl-estimates.jsonl && echo yes || echo no)" yes _row 'R2 a golden does NOT resolve merge=union' \ "$(attr_union crates/aprender-contrastive-data/tests/goldens/golden_corpus_train.jsonl && echo yes || echo no)" no + # R1b discriminates attr_golden: a predicate that always says golden would turn + # every ledger into golden+union, and nothing else here would notice. + _row 'R1b docs/audits/impl-estimates.jsonl (a real ledger) is NOT audit-golden' \ + "$(attr_golden docs/audits/impl-estimates.jsonl && echo yes || echo no)" no # R3 IS THE MECHANISM, not the declaration: a real two-sided append, merged # in a throwaway repo, with and without the attribute. An attribute that is @@ -102,6 +120,44 @@ self_test() { _row 'R3c and the result is the UNION (base+MAIN+SIDE), not one side' "$(cat "$t/lines" 2>/dev/null)" 3 rm -rf "${t:?}" + # R4: a declared golden under docs/audits/ resolves golden and NOT union (the + # real tree), and a golden that is ALSO union is caught (a throwaway tree). + _row 'R4a docs/audits/review-corpus/corpus-v1.jsonl is audit-golden, not union' \ + "$(attr_golden docs/audits/review-corpus/corpus-v1.jsonl && ! attr_union docs/audits/review-corpus/corpus-v1.jsonl && echo yes || echo no)" yes + # R4b-R4d run MEASURE itself on a throwaway tree, so the golden+union FAIL is + # the decision under test and not just the two predicates it reads. R4d is the + # control: the same tree with the golden declared alone PASSES, so R4c's FAIL + # comes from the both-declared branch and from nothing else in the scan. + t="$(mktemp -d)" || return 1 + ( + cd "$t" || exit 1 + git init -q . && mkdir -p docs/audits/g && : > docs/audits/g/x.jsonl + printf 'docs/audits/**/*.jsonl merge=union\ndocs/audits/g/*.jsonl audit-golden\n' > .gitattributes + git add -A + { attr_golden docs/audits/g/x.jsonl && attr_union docs/audits/g/x.jsonl && echo caught || echo missed; } > both + measure > both.out 2>&1; printf '%s\n' "$?" > both.rc + grep -c 'declared audit-golden AND union-merged' both.out > both.named + printf 'docs/audits/g/*.jsonl audit-golden\n' > .gitattributes + measure > solo.out 2>&1; printf '%s\n' "$?" > solo.rc + ) + _row 'R4b a golden that is ALSO union-merged is seen as both' "$(cat "$t/both" 2>/dev/null)" caught + _row 'R4c measure FAILs that tree (rc 1)' "$(cat "$t/both.rc" 2>/dev/null)" 1 + _row 'R4d ... naming the file as declared audit-golden AND union' "$(cat "$t/both.named" 2>/dev/null)" 1 + _row 'R4e control: the golden declared alone, measure PASSes (rc 0)' "$(cat "$t/solo.rc" 2>/dev/null)" 0 + rm -rf "${t:?}" + + # R5: only the bare attribute declares a golden. The value forms are NOT golden, + # so a file carrying one is still held to union (the stricter reading). + t="$(mktemp -d)" || return 1 + ( + cd "$t" || exit 1 + git init -q . && mkdir -p docs/audits + printf 'docs/audits/t.jsonl audit-golden=true\ndocs/audits/f.jsonl audit-golden=false\ndocs/audits/u.jsonl -audit-golden\n' > .gitattributes + for f in t f u; do attr_golden "docs/audits/$f.jsonl" && echo golden || echo not; done | tr '\n' ' ' > forms + ) + _row 'R5 audit-golden=true, =false and -audit-golden are NOT golden' "$(cat "$t/forms" 2>/dev/null)" 'not not not ' + rm -rf "${t:?}" + printf '\n%s row(s), %s\n' "$rows" "$( [ "$fails" -eq 0 ] && echo '0 red / FALSIFIER GREEN' || echo RED )" return "$fails" } diff --git a/scripts/transcript_retention_audit.sh b/scripts/transcript_retention_audit.sh new file mode 100755 index 0000000000..5ac5463988 --- /dev/null +++ b/scripts/transcript_retention_audit.sh @@ -0,0 +1,66 @@ +#!/usr/bin/env bash +# PRA-001 §1 G14 / T0: read-only transcript-retention audit, one row per fleet host. +# +# make transcript-retention-audit # the five fleet hosts +# HOSTS="gx10 mini" make transcript-retention-audit +# +# Row: host cleanupPeriodDays files_older_30d files_total oldest_mtime_utc +# cleanupPeriodDays: the value in ~/.claude/settings.json, `unset` (Claude Code's default of 30 applies) or +# `no-settings`. Files are regular files under ~/.claude/projects. A host with no ~/.claude/projects reports 0 0 -. +# +# Nothing is written on any host. Fails closed: a host that cannot be reached, or whose row does not parse, +# prints `UNREACHABLE` and makes the exit status 1, so a missing row can never read as a compliant one. +# Portable to macOS (mini): no `find -printf`, no jq, `stat` probed for GNU or BSD form. +set -euo pipefail + +HOSTS="${HOSTS:-lambda-labs intel gx10 mini yoga}" +SELF="$(uname -n)" + +# Runs on the audited host under sh. Prints exactly one line: cpd older total oldest_epoch +PROBE=' +p="$HOME/.claude/projects"; s="$HOME/.claude/settings.json" +if [ -f "$s" ]; then + cpd=$(tr -d "\n " < "$s" | sed -n "s/.*\"cleanupPeriodDays\":\([0-9][0-9]*\).*/\1/p") + [ -n "$cpd" ] || cpd=unset +else + cpd=no-settings +fi +if [ -d "$p" ]; then + older=$(find "$p" -type f -mtime +30 | wc -l | tr -d " ") + total=$(find "$p" -type f | wc -l | tr -d " ") + if stat -c %Y / >/dev/null 2>&1; then fmt="-c %Y"; else fmt="-f %m"; fi + oldest=$(find "$p" -type f -exec stat $fmt {} + 2>/dev/null | sort -n | head -1) + [ -n "$oldest" ] || oldest=- +else + older=0; total=0; oldest=- +fi +echo "ROW $cpd $older $total $oldest" +' + +to_utc() { + if [ "$1" = "-" ]; then + printf '%s' "-" + else + date -u -d "@$1" +%Y-%m-%dT%H:%MZ + fi +} + +rc=0 +printf '%s\t%s\t%s\t%s\t%s\n' host cleanupPeriodDays files_older_30d files_total oldest_mtime_utc +read -r -a hosts <<<"$HOSTS" +for h in "${hosts[@]}"; do + if [ "$h" = "$SELF" ]; then + out=$(sh -s <<<"$PROBE" 2>/dev/null) || out="" + else + out=$(timeout 60 ssh -o BatchMode=yes -o ConnectTimeout=10 "$h" sh -s <<<"$PROBE" 2>/dev/null) || out="" + fi + row=$(printf '%s\n' "$out" | sed -n 's/^ROW //p' | tail -1) + if [ -z "$row" ]; then + printf '%s\tUNREACHABLE\t-\t-\t-\n' "$h" + rc=1 + continue + fi + read -r cpd older total oldest <<<"$row" + printf '%s\t%s\t%s\t%s\t%s\n' "$h" "$cpd" "$older" "$total" "$(to_utc "$oldest")" +done +exit "$rc"