diff --git a/Cargo.lock b/Cargo.lock index 5e1d7bff28..a6dcfe1003 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1307,6 +1307,20 @@ dependencies = [ "zstd", ] +[[package]] +name = "aprender-review-experiment" +version = "0.69.3" +dependencies = [ + "half", + "regex", + "serde", + "serde_json", + "sha2 0.10.9", + "toml 0.8.23", + "ureq 2.12.1", + "zstd", +] + [[package]] name = "aprender-serve" version = "0.69.3" diff --git a/Cargo.toml b/Cargo.toml index e8541d3fab..8a14186ab6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -79,6 +79,8 @@ members = [ "crates/aprender-zram-adaptive", "crates/aprender-zram-cli", "crates/aprender-zram-generator", + # --- REX-001 (#4354): review-lane experiment harness + frozen analysis --- + "crates/aprender-review-experiment", # --- APR-MONO Phase 3b: presentar sub-crates (enabled) --- "crates/aprender-present-core", "crates/aprender-present-terminal", diff --git a/Makefile b/Makefile index f55faeb772..ed57842caa 100644 --- a/Makefile +++ b/Makefile @@ -1417,3 +1417,18 @@ oracle: oracle-check: oracle @git diff --exit-code tests/oracle/differential.json \ || { echo "FAIL: tests/oracle/differential.json differs from a fresh run — commit it"; exit 1; } + +# PRA-001 §1 G14 (T0): read-only, one row per fleet host; UNREACHABLE fails the target. +transcript-retention-audit: + @bash scripts/transcript_retention_audit.sh +.PHONY: transcript-retention-audit + +# PRM-C13 (was PRA T13), lane-independence-v1: κ_err scoreboard of the shadow lane vs every +# counted lane on matured gold SPLIT rows. With MANIFEST and CANDIDATE it gates (exit 12 = S-14). +SPLIT ?= val +MIN_N ?= 30 +lane-kappa: + @test -n "$(ROWS)" || { echo "usage: make lane-kappa ROWS=trace.jsonl [SPLIT=val] [MIN_N=30 | MANIFEST=m.json CANDIDATE=c.jsonl]"; exit 2; } + cargo run -q -p aprender-review-experiment --example rex -- lane-kappa "$(ROWS)" --split "$(SPLIT)" \ + $(if $(MANIFEST),--manifest "$(MANIFEST)" --candidate "$(CANDIDATE)",--min-n "$(MIN_N)") +.PHONY: lane-kappa diff --git a/README.md b/README.md index df038597eb..37869ea11c 100644 --- a/README.md +++ b/README.md @@ -40,8 +40,8 @@ publishing — all backed by YAML provable contracts that fail CI on drift. | Metric | Count | Source of truth | |-------:|------:|---| -| Workspace crates | **79** workspace crates | `cargo metadata --no-deps` (NOT `ls crates/` — 4 are `exclude`d, 1 has no Cargo.toml) | -| Provable contracts | **1837** provable contracts | `contracts/census.json` `.n_files` — the set `pv lint` walks (`pv census`, ONT-001 ONT-1; regenerated by `make contracts`, written by `make readme-sync`, guarded by `scripts/check_readme_claims.sh`) | +| Workspace crates | **81** workspace crates | `cargo metadata --no-deps` (NOT `ls crates/` — 4 are `exclude`d, 1 has no Cargo.toml) | +| Provable contracts | **1858** provable contracts | `contracts/census.json` `.n_files` — the set `pv lint` walks (`pv census`, ONT-001 ONT-1; regenerated by `make contracts`, written by `make readme-sync`, guarded by `scripts/check_readme_claims.sh`) | | CLI commands | **111** CLI commands | `contracts/apr-cli-commands-v1.yaml` §`commands` (parse the list; `apr --help` prints 112 because it lists `help` itself, and `grep -c '^ - name:'` gives 117 — other same-indent `name:` keys exist in the file) | | Book CLI chapters | **113** chapters | `ls book/src/cli/*.md` | | Book lib chapters | **72** chapters | `ls book/src/lib/*.md` (parity with `pub mod`) | @@ -321,7 +321,7 @@ falsification_tests: prediction: apr validate bad-model.apr exits non-zero ``` -The tree carries 1837 contracts across inference, training, quantization, attention, FFN, +The tree carries 1858 contracts across inference, training, quantization, attention, FFN, tokenization, model formats, CLI safety — and this README itself. ## Migration from old crates diff --git a/contracts/agent-trace-terms-v1.yaml b/contracts/agent-trace-terms-v1.yaml new file mode 100644 index 0000000000..7c6d58f929 --- /dev/null +++ b/contracts/agent-trace-terms-v1.yaml @@ -0,0 +1,48 @@ +metadata: + version: 1.0.0 + kind: schema + description: | + Provider-terms tagging for agent-trace-v1 rows (PRA-001 T12, spec §2.10, §9 P-TERMS; + agent-trace ruling 2026-09-25 items 1 and 7). Every row carries provider + {anthropic, google, local}, an exact model_id (never a family alias; a local model is + its weights sha256), access_channel {anthropic-api, claude-code-cli, antigravity, + gemini-api, vertex, apr-serve} consistent with the provider, and terms_ref {https url, + YYYY-MM-DD effective_date, RFC 3339 UTC fetched_at}. Any pool can be rebuilt without + a provider or channel through the first-class --exclude-provider / --exclude-channel + arguments. A row with a missing or inconsistent tag is REFUSED by every rebuild, since + an untagged row cannot be shown not to come from an excluded source. The public-release + default keeps local rows only (§2.11). This records the governing terms. It is not + legal advice; S-6 governs. Implementation: crates/aprender-review-experiment/src/terms.rs. + references: + - "docs/specifications/research-design-pr-agent-qwen-3.5.md" + - "contracts/training-pool-admission-v1.yaml" + +equations: + rebuild: + formula: "kept(R, X) = { r in R : gaps(r) = {} and provider(r) not in X.providers and channel(r) not in X.channels }" + +falsification_tests: + - id: FALSIFY-TERMS-001 + name: pool_rebuild_excluding_a_provider + prediction: "P-TERMS: a rebuild of a six-channel batch with --exclude-provider google succeeds, keeps 0 google rows, and passes the rest through byte-identical" + test_harness: "cargo test -p aprender-review-experiment --lib falsify_terms_001" + expected_output: "exit 0" + if_fails: "A pool cannot be rebuilt without a provider, so provider terms cannot be honoured later." + - id: FALSIFY-TERMS-002 + name: channel_exclusion_is_narrower_than_provider + prediction: "--exclude-channel antigravity drops only antigravity rows; gemini-api and vertex rows of the same provider stay" + test_harness: "cargo test -p aprender-review-experiment --lib falsify_terms_002" + expected_output: "exit 0" + if_fails: "Terms that differ by channel (App. A.7) cannot be applied separately." + - id: FALSIFY-TERMS-003 + name: identity_case_table + prediction: "rows with a family alias, unknown or empty model_id, unknown provider or channel, a channel of another provider, a non-sha local model, or a missing/non-https/ill-dated terms_ref all have gaps; well-formed rows do not" + test_harness: "cargo test -p aprender-review-experiment --lib falsify_terms_003" + expected_output: "exit 0" + if_fails: "A row with unknown or aliased identity (G-ID) is accepted as tagged." + - id: FALSIFY-TERMS-004 + name: untagged_row_is_refused_never_kept + prediction: "a row with gaps, or a non-JSON line, is refused by every rebuild, including one that excludes nothing, and is never in kept" + test_harness: "cargo test -p aprender-review-experiment --lib falsify_terms_004" + expected_output: "exit 0" + if_fails: "An untagged row slips past an exclusion filter it cannot be checked against." diff --git a/contracts/census.json b/contracts/census.json index f004942d87..2f24ad7ee4 100644 --- a/contracts/census.json +++ b/contracts/census.json @@ -1,8 +1,8 @@ { "schema": "ont.paiml.dev/census/v1alpha1", "git_sha": null, - "n_files": 1837, - "n_parsed": 1837, + "n_files": 1858, + "n_parsed": 1858, "n_parse_errors": 0, "parse_errors": [], "quarantined_n": 0, @@ -15,7 +15,7 @@ "pattern": 95, "pretraining-corpus": 2, "registry": 521, - "schema": 768, + "schema": 789, "tokenizer": 1, "training-loop": 8, "training-precondition-gate": 1 @@ -26,12 +26,25 @@ "pv-contract": 2, "release-evidence": 1 }, + "by_concept": { + "Code": 0, + "Contract": 1793, + "Dataset": 0, + "Doc": 0, + "Entity": 0, + "Kernel": 836, + "Model": 0, + "Proof": 0, + "Statement": 408, + "Symbol": 168, + "Test": 0 + }, "by_anchoring": { - "unanchored": 1829, + "unanchored": 1850, "class": 5, "instance": 3 }, - "id_set_sha256": "2055792eb2f6a0f696509d418e57868758e669988ea6e167867b02ec9983f1d7", + "id_set_sha256": "7298cd3a70dd7bbfa65a539da2e408c456a69bac8e87e41b4dfeede58bb57eb0", "declared_external": [ { "name": "provable-contracts", diff --git a/contracts/contracts.nt b/contracts/contracts.nt index dd58e94bcf..44b1997d75 100644 --- a/contracts/contracts.nt +++ b/contracts/contracts.nt @@ -8999,6 +8999,31 @@ . "contracts/trueno-rag/retrieval-quality-v1.yaml"^^ . "retrieval-quality-v1"^^ . + . + . + "contracts/review-corpus-contamination-v1.yaml"^^ . + "review-corpus-contamination-v1"^^ . + "schema"^^ . + . + . + "contracts/review-corpus-v1.yaml"^^ . + "review-corpus-v1"^^ . + "schema"^^ . + . + . + "contracts/review-experiment-receipt-v1.yaml"^^ . + "review-experiment-receipt-v1"^^ . + "schema"^^ . + . + . + "contracts/rex-cell-admission-v1.yaml"^^ . + "rex-cell-admission-v1"^^ . + "schema"^^ . + . + . + "contracts/rex-prereg-v1.yaml"^^ . + "rex-prereg-v1"^^ . + "schema"^^ . . . "contracts/rmsnorm-kernel-v1.yaml"^^ . diff --git a/contracts/lane-independence-v1.yaml b/contracts/lane-independence-v1.yaml new file mode 100644 index 0000000000..6b6a0c768d --- /dev/null +++ b/contracts/lane-independence-v1.yaml @@ -0,0 +1,53 @@ +metadata: + version: 2.0.0 + kind: schema + description: | + PRM-C13 (was PRA-001 T13) (PRM-001 v3 "PROMETHEUS" H7 §3, S-14; row 2 of §7, was research-design-pr-agent-qwen-3.5 §5 M4). + `make lane-kappa SPLIT=val ROWS=F [MANIFEST=M CANDIDATE=F2]` = `rex lane-kappa ROWS --split val (--min-n N | --manifest M --candidate ROWS2)` (crates/aprender-review-experiment: kappa_probe.rs, examples/rex.rs) + measures κ_err, Cohen's κ on binary per-item error indicators against GOLD, for every + (qwen, counted-lane) pair on matured gold val rows, reusing stats::error_kappa (PRM-001 §3 error + overlap). An error is a parsed verdict that disagrees with the gold outcome; unparsed items and + silver, pending or quarantined rows are excluded, never counted as errors. A pair with fewer than + `min_n` shared items, or with κ undefined, reports `insufficient` (S-14), never a number. + 2.0.0: the gate is v3's δ-free H7, decided by the prereg-locked stats::h7_holds. On I, the + items the shadow lane and EVERY counted lane (voter) scored, κ_err(qwen, V) must be ≤ the + largest voter–voter κ_err for each voter V. v2's baseline + δ rule is gone, and a manifest that + still carries `delta` is refused, since nothing would read it. The voters and min_n come only from + the run manifest, registered before training. Refusal is an andon (non-zero exit), not a warning. + references: + - "docs/specifications/PRM-001-prometheus.md" + - "docs/specifications/review-experiment-protocol.md" + +equations: + error_indicator: + formula: "err(lane, i) = parsed(lane, i) ∧ verdict(lane, i) ≠ gold(i)" + kappa_err: + formula: "κ_err(a, b) = (p_o − p_e) / (1 − p_e) over items i with gold(i) ∧ matured(i) ∧ split(i) = val ∧ parsed(a, i) ∧ parsed(b, i)" + gate: + formula: "pass ⇔ |I| ≥ min_n ∧ h7_holds([κ_err_I(qwen, V) : V ∈ voters], [κ_err_I(V_i, V_j) : i < j]) = Some(true), i.e. ∀ V: κ_err_I(qwen, V) ≤ max_{i voter–voter max) is refused naming that voter; with identical voters (ceiling 1) the same copy passes, since the bound is inclusive" + test_harness: "cargo test -p aprender-review-experiment --lib falsify_lind_003" + expected_output: "exit 0" + if_fails: "A silver-trained Qwen that copies the voters' blind spots could occupy the dissent lane (R13, PRA-H1)." + - id: FALSIFY-LIND-004 + name: voters_and_min_n_are_pre_registered_and_delta_is_gone + prediction: "a manifest with no block, min_n 0, no voters, a registration at or after training start, a non-Z timestamp, or a v2 `delta` is refused" + test_harness: "cargo test -p aprender-review-experiment --lib falsify_lind_004" + expected_output: "exit 0" + if_fails: "The runner could pick the voters after seeing κ, or believe a δ that the δ-free gate never reads." diff --git a/contracts/review-b2-loop-v1.yaml b/contracts/review-b2-loop-v1.yaml new file mode 100644 index 0000000000..6046539ce9 --- /dev/null +++ b/contracts/review-b2-loop-v1.yaml @@ -0,0 +1,47 @@ +metadata: + version: 1.0.0 + kind: schema + description: | + PROMETHEUS (PRM-001, was REX-001) REX-12 B2 loop, spec §5.3 B2a..B2d. Each B2 tier waits + on one apr verb accepting qwen35 (B2a distill, B2b finetune, B2c merge, B2d quantize). + `rex b2 status` reads the in-tree refusal ledger (evidence/verbs/refusals.json, + refusal-receipt-v1) and writes one row per tier: ready only on a positive acceptance + receipt, verb_refused{removed_by} on a ledger row, and unledgered when neither exists. + A missing refusal row is not acceptance, so an unmeasured verb is NotRun and never + silently ready. It exits 11 while any row is NotRun. Ready tiers feed the REX-11 §5.4 + gate (review-champion-challenger-v1). `rex b2 teacher-receipt` receipts the 27B + teacher's top-k logits at m = 1 over the train pool (every non-test item): sha, count, + k, the size of the sealed index, and 0 test hits. Any test item, unknown item, + duplicate row, wrong-k or non-finite position, sealed diff sha or hunk in any string + field, or an empty sealed index refuses the whole file. At HEAD all four rows are + unledgered; the qwen35 fine-tune/distill verbs are EPIC #4002 (0.75.0). + references: + - "docs/specifications/review-experiment-protocol.md" + - "contracts/refusal-receipt-v1.yaml" + - "https://github.com/paiml/aprender/issues/4367" + +equations: + row: + formula: "ready(v) ⇔ accepted(v) ∧ ¬refused(v); accepted(v) ∧ refused(v) ⇒ error; ¬accepted(v) ∧ ¬refused(v) ⇒ unledgered" + teacher: + formula: "receipt ⇔ |sealed| > 0 ∧ ∀ row: split(id) = dev ∧ once(id) ∧ ∀ pos: |pos| = k ∧ finite ∧ scan(file) = ∅" + +falsification_tests: + - id: FALSIFY-RXB-001 + name: no_b2_row_is_ready_without_an_acceptance_receipt + prediction: "on the in-tree ledger no B2 row is ready; a ledger row reads verb_refused with its removed_by; no row plus no acceptance reads unledgered; accepted and refused at once is an error" + test_harness: "cargo test -p aprender-review-experiment --lib falsify_rxb_001" + expected_output: "exit 0" + if_fails: "A B2 tier whose verb nobody measured could read as ready and enter §5.4." + - id: FALSIFY-RXB-002 + name: the_ledger_is_refusal_receipt_v1_or_rejected + prediction: "removed_by accepts v., never and unscheduled; tbd, a bare or patch version, a sha and empty are rejected; duplicate verbs, exit 0 and a foreign schema reject the file" + test_harness: "cargo test -p aprender-review-experiment --lib falsify_rxb_002" + expected_output: "exit 0" + if_fails: "A decorative removed_by would read as a validated refusal." + - id: FALSIFY-RXB-003 + name: the_teacher_dataset_is_receipted_with_zero_test_hashes + prediction: "two dev rows at k = 4 are receipted with sha, count 2, m 1 and 0 test hits; a test item, a duplicate, an unknown id, wrong k, NaN, a sealed diff in a string field, a sealed diff sha, no rows and an empty index are each refused" + test_harness: "cargo test -p aprender-review-experiment --lib falsify_rxb_003" + expected_output: "exit 0" + if_fails: "Distillation could train the student on the sealed test split." diff --git a/contracts/review-champion-challenger-v1.yaml b/contracts/review-champion-challenger-v1.yaml new file mode 100644 index 0000000000..e3aaea497d --- /dev/null +++ b/contracts/review-champion-challenger-v1.yaml @@ -0,0 +1,65 @@ +metadata: + version: 1.0.0 + kind: schema + description: | + PROMETHEUS (PRM-001, was REX-001) REX-11 champion/challenger, spec §5.3–§5.4. A + challenger (B1a prompt version, B1b retrieval few-shot, a B2 checkpoint) replaces the + serving champion only if all five §5.4 gates hold on the current sealed test version: + (1) McNemar exact one-sided p < ALPHA on per-item correctness, (2) precision point + estimate not lower, (3) H1 and H2 hold, (4) 0 contamination hits, (5) p95 ≤ the §4 + queue budget. Every gate fails closed: unmeasured is failed. Each decision spends one + of the version's 20 evaluations (§2.2); refusals (budget spent, no executed row (R-6), + no champion rows, challenger == champion) spend nothing. Promotion is a pin bump of + (weights, prompt, adapter) shas; the rollback target is the row's champion_before. + Demotion: two consecutive gold escapes where the champion said PASS and a voter said + FAIL. B1b retrieves from non-test gold only (R-2), never the query itself, and every + challenger prompt passes the contamination guard, which scans each fenced block alone + because a gold `- defect at …` line after a fence joins the last hunk and hides it + from a whole-text scan. + references: + - "docs/specifications/review-experiment-protocol.md" + - "https://github.com/paiml/aprender/issues/4366" + +equations: + promote: + formula: "promote ⇔ used < 20 ∧ ran ∧ p_McNemar(b,c) < α ∧ Δprecision ≥ 0 ∧ H1 ∧ H2 ∧ hits = 0 ∧ p95 ≤ budget" + demote: + formula: "demote ⇔ ∃ consecutive gold escapes e_i, e_{i+1}: champion_passed ∧ a_voter_failed" + +falsification_tests: + - id: FALSIFY-RXC-001 + name: all_five_gates_promote_and_each_alone_rejects + prediction: "b=20, c=0 with every gate measured green promotes; flipping any single gate (or leaving it unmeasured) rejects with exactly that gate's reason; more correct but less precise is rejected on gate 2" + test_harness: "cargo test -p aprender-review-experiment --lib falsify_rxc_001" + expected_output: "exit 0" + if_fails: "A challenger could be promoted with an unmeasured or failing gate." + - id: FALSIFY-RXC-002 + name: the_evaluation_budget_counts_and_refusals_spend_nothing + prediction: "the 21st decision on one test version is refused and spends nothing; a rotated version has its own budget; a challenger that never ran and the champion itself are refused; promotion moves the pin and records the rollback sha" + test_harness: "cargo test -p aprender-review-experiment --lib falsify_rxc_002" + expected_output: "exit 0" + if_fails: "The sealed test set could be over-fitted by unbounded challengers, or a never-run challenger counted." + - id: FALSIFY-RXC-003 + name: two_consecutive_gold_escapes_demote + prediction: "two consecutive gold champion-PASS/voter-FAIL escapes demote; a silver row neither breaks nor extends the run; a caught gold escape breaks it" + test_harness: "cargo test -p aprender-review-experiment --lib falsify_rxc_003" + expected_output: "exit 0" + if_fails: "A champion that keeps missing gold defects would keep serving." + - id: FALSIFY-RXF-001 + name: a_test_item_in_the_retrieval_pool_is_an_error + prediction: "select() on a pool containing a Split::Test item errors naming the item and R-2" + test_harness: "cargo test -p aprender-review-experiment --lib falsify_rxf_001" + expected_output: "exit 0" + if_fails: "Few-shot retrieval could show the model a sealed test diff." + - id: FALSIFY-RXF-002 + name: retrieval_ranks_by_similarity_and_never_returns_the_query + prediction: "the most similar dev item ranks first; a dev query retrieves neither its own id nor a byte-identical diff" + test_harness: "cargo test -p aprender-review-experiment --lib falsify_rxf_002" + expected_output: "exit 0" + if_fails: "A dev evaluation becomes open-book." + - id: FALSIFY-RXF-003 + name: a_rendered_prompt_carrying_a_test_hunk_is_refused + prediction: "a clean rendered prompt passes; the same prompt with a sealed item as an example is refused naming it, although the whole-text scan alone misses it; an empty index is refused" + test_harness: "cargo test -p aprender-review-experiment --lib falsify_rxf_003" + expected_output: "exit 0" + if_fails: "A B1 challenger prompt could carry a sealed test diff past the contamination guard." diff --git a/contracts/review-corpus-contamination-v1.yaml b/contracts/review-corpus-contamination-v1.yaml new file mode 100644 index 0000000000..9fd8ad0388 --- /dev/null +++ b/contracts/review-corpus-contamination-v1.yaml @@ -0,0 +1,70 @@ +metadata: + version: 1.1.0 + kind: schema + description: | + PROMETHEUS (PRM-001) sealed-test contamination check, spec §2.2 and rule R-2. A sealed + test item leaks when a training file carries its diff sha256 literally, or any of + its hunks (header stripped, >= 4 lines, at least one change) — including inside + JSON string values, so a diff stored in a JSONL field is seen unescaped. Training + roots: docs/audits/review-corpus/train/ (contamination::TRAIN_ROOTS). Any hit is + a hard FAIL (S-3). + v1.1 (PRA-001 T7, gate G-CON, spec §2.9.4): the exact checks miss a sealed diff that + was re-indented or had identifiers renamed, so a sealed item is also hit by CLUSTER. + Tokens drop all whitespace and blank lines, keep a +/- line marker, keep Rust + keywords and literals, and replace every identifier with $ (MOSS normalisation). + Diff header lines are skipped. A sealed item's sketch is the 64 smallest 12-token + shingle hashes (bottom-k MinHash; items under 32 shingles have none). A text holds + the item when >= 0.7 of that sketch is among its own shingles (containment, so a + diff inside a longer prompt counts). Sketches live in test-sketch-v1.txt, written by + `rex corpus-build` beside the prereg-locked manifest, which is unchanged. + False-positive measurement, 2026-09-25: 391 real .rs commit diffs from origin/main, + all 152,490 ordered pairs. At k=5, 469 pairs cluster; at k=12, 7 do (0.005%), all + inside one family of templated "beat" test files that are copies with renamed + identifiers, so over-refusing them fails safe. k=16 gives the same 7. + references: + - "docs/specifications/review-experiment-protocol.md" + - "docs/audits/review-corpus/test-manifest-v1.txt" + - "https://github.com/paiml/aprender/issues/4357" + - "https://github.com/paiml/aprender/issues/4354" + +equations: + leak: + formula: "leak(f) = exists t in sealed: t.diff_sha256 in text(f) or hunks(t) ∩ hunks(text(f) ∪ json_strings(f)) != {} or |sketch(t) ∩ shingles(s)| / |sketch(t)| >= 0.7 for some s in text(f) ∪ json_strings(f)" + +falsification_tests: + - id: FALSIFY-RCC-001 + name: planted_jsonl_leak_is_caught + prediction: "a sealed diff inside a JSONL training record is a hit" + test_harness: "cargo test -p aprender-review-experiment --lib falsify_rcc_001" + expected_output: "exit 0" + if_fails: "Escaped diffs in JSONL pass the check; B2 teacher sets could train on test items." + - id: FALSIFY-RCC-002 + name: rebased_copy_and_sha_literal_leak + prediction: "a copy with other hunk line numbers, and a bare diff sha, are hits; an unrelated diff is not" + test_harness: "cargo test -p aprender-review-experiment --lib falsify_rcc_002" + expected_output: "exit 0" + if_fails: "The check keys on line numbers or misses sha references." + - id: FALSIFY-RCC-003 + name: train_roots_are_clean + prediction: "no file under the training roots carries a sealed test item" + test_harness: "cargo test -p aprender-review-experiment --lib falsify_rcc_003" + expected_output: "exit 0" + if_fails: "S-3: a sealed test item is in training data. STOP; rotate the leaked items out." + - id: FALSIFY-RCC-004 + name: perturbed_sealed_item_is_a_cluster_hit + prediction: "a sealed diff re-indented with tabs, re-spaced, with blank lines added, a partial identifier rename and a re-based header passes the exact checks, and with sketches is a cluster hit and a Refused pool row (also inside a JSONL prompt field, and with one line added as in a backport)" + test_harness: "cargo test -p aprender-review-experiment --lib falsify_rcc_004" + expected_output: "exit 0" + if_fails: "G-CON: a whitespace or rename perturbation smuggles a sealed test item into a training pool." + - id: FALSIFY-RCC-005 + name: unrelated_diffs_are_not_clustered + prediction: "unrelated diffs, including one with the sealed item's loop skeleton, stay below 0.7 containment; a diff under 32 shingles has no sketch" + test_harness: "cargo test -p aprender-review-experiment --lib falsify_rcc_005" + expected_output: "exit 0" + if_fails: "The cluster check refuses ordinary training rows, so the pool shrinks for no reason." + - id: FALSIFY-RCC-006 + name: sketch_file_round_trips_and_fails_closed + prediction: "render/parse round-trips; a wrong scheme line, bad hex or an empty sketch parses to None" + test_harness: "cargo test -p aprender-review-experiment --lib falsify_rcc_006" + expected_output: "exit 0" + if_fails: "A corrupt sketch file is read as sealing nothing." diff --git a/contracts/review-corpus-v1.yaml b/contracts/review-corpus-v1.yaml new file mode 100644 index 0000000000..b6bd17367a --- /dev/null +++ b/contracts/review-corpus-v1.yaml @@ -0,0 +1,44 @@ +metadata: + version: 1.0.0 + kind: schema + description: | + PROMETHEUS (PRM-001, was REX-001) review corpus v1, spec §2.2. 150 items: P (planted + cargo-mutants diffs, markers stripped), R (merged fix PRs with a linked issue, + code hunks reverse-applied), G (merged PRs before the 14-day cutoff, green + ci / gate + workspace-test, not reverted, no regression label). Strata on a + bytes/4 token proxy (S < 2000, M 2000–8000, L > 8000). Seeded (4354), stratified + 30/70 dev/test split. Item metadata: docs/audits/review-corpus/corpus-v1.jsonl. + Sealed test manifest: docs/audits/review-corpus/test-manifest-v1.txt (version + review-corpus-v1@787d2026256cc08b). Diffs: items-v1.tar (sha256 70dffe5625e1849661303b82bb1fdf5254ab1d4560e2fe3eacc147e5d2588bba), outside git. + references: + - "docs/specifications/review-experiment-protocol.md" + - "docs/audits/rex-001/rex-02-receipt.md" + - "https://github.com/paiml/aprender/issues/4357" + +equations: + stratum: + formula: "t = ceil(bytes/4); S if t < 2000, M if t <= 8000, else L" + split: + formula: "per (class, stratum): order by SplitMix64(4354) key in id order; first round(0.3 n) are dev" + corpus_version: + formula: "'review-corpus-v1@' || sha256(test-manifest-v1.txt)[0..16]" + +falsification_tests: + - id: FALSIFY-RCV-001 + name: mutation_markers_never_reach_a_prompt + prediction: "a sanitised P diff names neither cargo-mutants nor the replacement" + test_harness: "cargo test -p aprender-review-experiment --lib sanitize_removes_every_mutation_marker" + expected_output: "exit 0" + if_fails: "P items leak their answer; every P verdict is void." + - id: FALSIFY-RCV-002 + name: split_is_seeded_stratified_and_order_free + prediction: "the same seed gives the same split whatever the input order; a different seed moves it; each cell is 30% dev" + test_harness: "cargo test -p aprender-review-experiment --lib split_is_stratified_seeded_and_stable" + expected_output: "exit 0" + if_fails: "The split is not reproducible; the sealed manifest cannot be re-derived." + - id: FALSIFY-RCV-003 + name: comment_only_hunks_are_not_defects + prediction: "R items keep only hunks that change code; defect locations point at code lines" + test_harness: "cargo test -p aprender-review-experiment --lib comment_only_hunks_are_not_the_fix" + expected_output: "exit 0" + if_fails: "R labels name comments; localization is measured against the wrong line." diff --git a/contracts/review-experiment-receipt-v1.yaml b/contracts/review-experiment-receipt-v1.yaml new file mode 100644 index 0000000000..cc576d04b4 --- /dev/null +++ b/contracts/review-experiment-receipt-v1.yaml @@ -0,0 +1,54 @@ +metadata: + version: 1.0.0 + kind: schema + description: | + PROMETHEUS (PRM-001, was REX-001) receipt, spec §2.3/§2.4. One JSONL row per + (item, cell, arm) written by the REX-03 harness (crates/aprender-review-experiment: + harness.rs) against a resident `apr serve`. A row is admissible only if it + deserializes with every required field (no unknown fields), carries no empty or + `unknown` identity field, carries 64-hex shas for the item, prompt, request, prereg, + apr binary and weights (hosted arms say `hosted`), names the locked prereg sha and + the corpus version under analysis, is greedy, and, when it ran, carries token + counts, timings, raw-output path+sha and a load snapshot. The scorer (score.rs) + scores the EXPECTED item set: an item without exactly one admissible row whose raw + output exists with the recorded sha and re-parses to the recorded verdict is + NotRun{Inadmissible}. Unparsed and NotRun are never correct and never a FAIL (R-6). + Signed receipts are verified with minisign; unsigned results are labelled + exploratory. + references: + - "docs/specifications/review-experiment-protocol.md" + - "docs/audits/rex-001/analysis-plan.md" + - "docs/audits/rex-001/rex-03-receipt.md" + - "https://github.com/paiml/aprender/issues/4358" + +equations: + correct: + formula: "correct(item) = (verdict == FAIL and class in {P,R}) or (verdict == PASS and class == G)" + recall: + formula: "|FAIL on P∪R| / |P∪R expected| (NotRun and Unparsed stay in the denominator)" + precision: + formula: "|FAIL on P∪R| / |FAIL|" + false_refute: + formula: "|FAIL on G| / |G expected|" + parse_rate: + formula: "|PASS ∪ FAIL| / |executed|" + +falsification_tests: + - id: FALSIFY-RXR-001 + name: missing_model_sha_is_inadmissible + prediction: "a receipt without weights_sha256 (or with `unknown`, or a sha prefix) is inadmissible" + test_harness: "cargo test -p aprender-review-experiment --lib falsify_rxr_001" + expected_output: "exit 0" + if_fails: "Receipts can no longer be tied to the weights that produced them; every cross-cell and cross-model comparison is unanchored." + - id: FALSIFY-RXR-002 + name: not_run_is_never_correct + prediction: "NotRun (any reason, including a missing receipt) is never correct and never a FAIL; it stays in the recall and correctness denominators" + test_harness: "cargo test -p aprender-review-experiment --lib falsify_rxr_002" + expected_output: "exit 0" + if_fails: "A cell that cannot run, or a dropped receipt, would raise a score (R-6)." + - id: FALSIFY-RXR-003 + name: unparsed_is_never_a_pass + prediction: "Unparsed is never correct on a G item, never a FAIL on a defect, and never enters a paired rate" + test_harness: "cargo test -p aprender-review-experiment --lib falsify_rxr_003" + expected_output: "exit 0" + if_fails: "A model that answers garbage would score as a clean reviewer on good diffs." diff --git a/contracts/review-lane-perf-ratchet-v1.yaml b/contracts/review-lane-perf-ratchet-v1.yaml new file mode 100644 index 0000000000..b0435b2830 --- /dev/null +++ b/contracts/review-lane-perf-ratchet-v1.yaml @@ -0,0 +1,45 @@ +metadata: + version: 1.0.0 + kind: schema + description: | + PROMETHEUS (PRM-001, was REX-001) REX-10 perf ratchet, spec §5.1: review p95 on the + primary cell may never rise after 3 recorded tags. Each tag reruns the perf arm on the + primary cell over one fixed item set (full dev split + 30 fixed test items, timing + only) with the new apr tag and with llama.cpp. `rex ratchet record` turns the tag's + warm, non-rerun, admissible receipts into one `review-lane-perf-ratchet-v1` entry + (per-item wall ms, nearest-rank p95 and its bootstrap CI, llama.cpp p95, receipts + sha); `rex ratchet check` compares every tag after the third with the best earlier + tag by a paired bootstrap of the Harrell–Davis p95 ratio and exits 10 on RED. + Unpaired CI overlap (§4 tie rule) and a paired nearest-rank p95 both miss a +10 % + rise at the pilot's n; the only level used is the prereg's ALPHA (R-7). A tag with + no timings did not run and is not a recorded tag (R-6). A growing gap to llama.cpp + is a note to file (§9.1), not an andon. + references: + - "docs/specifications/review-experiment-protocol.md" + - "https://github.com/paiml/aprender/issues/4365" + +equations: + ratchet: + formula: "RED ⇔ ∃ i ≥ 3: lo_{1-α/2}( boot_paired( HD95(t_i) / HD95(t_best( 1 ∨ violations ≠ ∅" + state: + formula: "|tags| ≤ 3 ⇒ arming; |tags| > 3 ∧ ¬RED ⇒ green" + +falsification_tests: + - id: FALSIFY-RXR-001 + name: a_planted_ten_percent_p95_regression_is_red + prediction: "3 tags arm; an unchanged 4th tag is green; the same tag × 1.10 is RED in 10/10 independent runs with ≤ 1/10 false alarms; giving back a gain to the best tag is RED" + test_harness: "cargo test -p aprender-review-experiment --lib falsify_rxr_001" + expected_output: "exit 0" + if_fails: "A release could slow review p95 by 10 % and the ratchet would stay green." + - id: FALSIFY-RXR-002 + name: the_unpaired_overlap_rule_misses_ten_percent + prediction: "the two unpaired p95 CIs of a +10 % tag overlap at n = 130, so the overlap rule cannot be the ratchet's rule" + test_harness: "cargo test -p aprender-review-experiment --lib falsify_rxr_002" + expected_output: "exit 0" + if_fails: "The finding that forced the paired rule no longer holds; re-derive the rule." + - id: FALSIFY-RXR-003 + name: the_ratchet_fails_closed + prediction: "a duplicate tag, a foreign cell, or no common items is RED; record refuses an empty tag, two apr binaries under one tag, and a non-finite timing; a growing llama.cpp gap is noted" + test_harness: "cargo test -p aprender-review-experiment --lib falsify_rxr_003" + expected_output: "exit 0" + if_fails: "A malformed ratchet file or a tag that never ran would read as green." diff --git a/contracts/review-ledger-v1.yaml b/contracts/review-ledger-v1.yaml new file mode 100644 index 0000000000..94ce4ee907 --- /dev/null +++ b/contracts/review-ledger-v1.yaml @@ -0,0 +1,45 @@ +metadata: + version: 1.0.0 + kind: schema + description: | + PROMETHEUS (PRM-001, was REX-001) REX-07 day-0 shadow lane, spec §5.2/§7. The 4B + lane runs inside the quorum rail as a non-voting advisory lane (paiml-implement + advisory-lane.sh, #420), which records `.advisory_lane` in every quorum receipt. + `rex ledger REPO OUT RECEIPT...` (crates/aprender-review-experiment: ledger.rs) turns + receipts into `review-ledger-v1` rows (repo, ticket, head, PR, diff sha, counted lanes + with model ids, verdicts and finding counts, the shadow result, apr tag, weights sha, + outcome=pending until the ASL escape join) and measures shadow coverage. A receipt + with no advisory_lane, or with the lane `off`, is missing; a lane that did not answer + is Unknown{LaneUnavailable}, never a verdict; a shadow not recorded as uncounted, or a + width other than the counted lanes, is a violation. Exit 10 unless coverage holds. + superseded_by: "contracts/review-ledger-v2.yaml (writing); v1 rows stay readable" + references: + - "docs/specifications/review-experiment-protocol.md" + - "https://github.com/paiml/aprender/issues/4362" + - "https://github.com/paiml/paiml-implement/pull/420" + +equations: + coverage: + formula: "holds ⇔ |Q| > 0 ∧ ∀ q ∈ Q: shadow(q) ∈ {Answered, Unknown} ∧ counts(q) = false ∧ width(q) = |counted_lanes(q)|" + unknown: + formula: "shadow(q) ∈ {Verdict, NotRun, Refused} (the paiml-implement#436 wire form since 89d994bc1); the 402aff870 form state ≠ answered ⇒ Unknown{LaneUnavailable} is read-only, see review-ledger-v2" + +falsification_tests: + - id: FALSIFY-RXL-001 + name: a_quorum_without_a_shadow_row_is_missing + prediction: "a receipt with no advisory_lane, or with the lane off, is listed as missing and coverage fails; an empty or unparsable set never holds" + test_harness: "cargo test -p aprender-review-experiment --lib falsify_rxl_001" + expected_output: "exit 0" + if_fails: "Quorums could run without the shadow lane and REX-07's 100 % would still read green." + - id: FALSIFY-RXL-002 + name: gx10_down_is_unknown_and_keeps_the_width + prediction: "an unavailable lane, or an answered state with no verdict, is Unknown{LaneUnavailable}; the width stays 3" + test_harness: "cargo test -p aprender-review-experiment --lib falsify_rxl_002" + expected_output: "exit 0" + if_fails: "A gx10 outage would enter the ledger as a verdict and contaminate H4/H5 agreement." + - id: FALSIFY-RXL-003 + name: a_counted_or_width_changing_shadow_is_a_violation + prediction: "counts=true, a missing counts flag, or a width other than the counted lanes fails coverage" + test_harness: "cargo test -p aprender-review-experiment --lib falsify_rxl_003" + expected_output: "exit 0" + if_fails: "The non-voting lane could start voting before REX-09's promotion gates." diff --git a/contracts/review-ledger-v2.yaml b/contracts/review-ledger-v2.yaml new file mode 100644 index 0000000000..858db39be9 --- /dev/null +++ b/contracts/review-ledger-v2.yaml @@ -0,0 +1,82 @@ +metadata: + version: 1.0.0 + kind: schema + description: | + PRA-001 T2 (spec docs/specifications/research-design-pr-agent-qwen-3.5.md §7), on top of + PRM-001 REX-07. `rex ledger` (crates/aprender-review-experiment: ledger.rs, ledger_v1.rs) + writes `review-ledger-v2` rows. v1 kept findings as a count, so each quorum lost its + (input, output, findings) triple. v2 keeps each counted lane's findings text verbatim + plus its agent-trace-v1 input/output shas. Text never captured is None, never an empty + list. The apr shadow row is the typed paiml-implement#436 wire form (Verdict | NotRun | + Refused). Outcomes are appended as OutcomeJoin lines and never rewrite a row. v1 rows + (both shapes ever written under that name) still parse through read_row, and a + typed-shadow v1 row migrates with its count kept and its text None. Supersedes + review-ledger-v1 for writing; v1 stays for reading. + references: + - "docs/specifications/research-design-pr-agent-qwen-3.5.md" + - "docs/specifications/review-experiment-protocol.md" + - "contracts/review-ledger-v1.yaml" + - "https://github.com/paiml/aprender/issues/4354" + - "https://github.com/paiml/paiml-implement/issues/436" + +equations: + coverage: + formula: "holds ⇔ |Q| > 0 ∧ missing = ∅ ∧ violations = ∅ ∧ identity_gaps = 0 ∧ trace_gaps = 0" + shadow: + formula: "shadow(q) ∈ {Verdict{verdict ∈ {PASS,FAIL}, cell, backend}, NotRun{NoExecutor|Busy|Timeout|ContextOverflow|TrainActive}, Refused{cell, removed_by ∈ {ladder,gpu-proof,parse}}}; any other value ⇒ violation" + trace_gap: + formula: "gap(l) ⇔ findings(l) = None ∨ input_sha256(l) = None ∨ output_sha256(l) = None" + outcome: + formula: "outcome(r, J) = at-max { j ∈ J : j.repo = r.repo ∧ j.head = r.head }.outcome, else r.outcome; r is never rewritten" + migration: + formula: "migrate(v1) = v2 with findings_count = v1.findings ∧ findings = None ∧ shas = None; shadow(v1) legacy ⇒ Err" + +falsification_tests: + - id: FALSIFY-RXL-001 + name: a_quorum_without_a_shadow_row_is_missing + prediction: "no advisory_lane, or the lane off, is missing; an empty set never holds" + test_harness: "cargo test -p aprender-review-experiment --lib falsify_rxl_001" + expected_output: "exit 0" + if_fails: "Quorums could run without the shadow lane and REX-07 would still read green." + - id: FALSIFY-RXL-002 + name: gx10_down_is_notrun_and_keeps_the_width + prediction: "a lane that did not run is NotRun, never a verdict; the width stays the counted lanes" + test_harness: "cargo test -p aprender-review-experiment --lib falsify_rxl_002" + expected_output: "exit 0" + if_fails: "A gx10 outage would enter the ledger as a verdict." + - id: FALSIFY-RXL-003 + name: a_counted_or_width_changing_shadow_is_a_violation + prediction: "counts=true, a missing counts flag, or a width other than the counted lanes fails coverage" + test_harness: "cargo test -p aprender-review-experiment --lib falsify_rxl_003" + expected_output: "exit 0" + if_fails: "The non-voting lane could start voting before promotion." + - id: FALSIFY-RXL-004 + name: a_receipt_without_a_weights_sha_does_not_hold + prediction: "a row with no apr tag or weights sha is an identity gap and coverage fails" + test_harness: "cargo test -p aprender-review-experiment --lib falsify_rxl_004" + expected_output: "exit 0" + if_fails: "Shadow rows could not be tied to the weights that produced them." + - id: FALSIFY-RXL-005 + name: a_free_text_reason_fails_to_parse + prediction: "an apr row outside the #436 wire form (free text, unknown key or value) is a violation" + test_harness: "cargo test -p aprender-review-experiment --lib falsify_rxl_005" + expected_output: "exit 0" + if_fails: "Free-text reasons would re-enter the ledger and split the spelling from receipt-lint." + - id: FALSIFY-RXL-006 + name: a_counted_lane_without_its_trace_does_not_hold + prediction: "a counted lane missing findings text, input_sha256 or output_sha256 is a trace gap; the row is kept and coverage fails" + test_harness: "cargo test -p aprender-review-experiment --lib falsify_rxl_006" + expected_output: "exit 0" + if_fails: "v2 would silently degrade to v1's count-only capture and the training triples would be lost again." + - id: FALSIFY-RXL-007 + name: an_outcome_join_appends_and_never_rewrites + prediction: "the latest join by `at` for (repo, head) wins; other heads are ignored; the row is byte-identical after" + test_harness: "cargo test -p aprender-review-experiment --lib falsify_rxl_007" + expected_output: "exit 0" + if_fails: "Outcome labels could overwrite history, and gold labels would lose their provenance." + - id: FALSIFY-RXL-008 + name: v1_rows_parse_and_migrate_without_invented_text + prediction: "both v1 shapes parse; a typed v1 row migrates with its count kept and findings None (a trace gap); a legacy shadow is refused; a v1 body under the v2 name, or an unknown schema, is an error" + test_harness: "cargo test -p aprender-review-experiment --lib falsify_rxl_008" + expected_output: "exit 0" + if_fails: "Old rows would stop reading, or migration would invent an empty findings list that trains as 'no issues'." diff --git a/contracts/review-replay-v1.yaml b/contracts/review-replay-v1.yaml new file mode 100644 index 0000000000..5c55b8c14a --- /dev/null +++ b/contracts/review-replay-v1.yaml @@ -0,0 +1,66 @@ +metadata: + version: 1.0.0 + kind: schema + description: | + PRM-S1 (PRM-001 v3 "PROMETHEUS" §2.4 review-replay-v1, §9 speed block; Refs #4354, #4447). + crates/aprender-review-experiment/src/replay.rs builds a frozen replay set of real quorum + diffs (INPUTS only, never a lane's output), defines the review-replay-receipt-v1 row, and + folds one run into the §9 speed block. `examples/replay.rs` is the runner, and it + drives `apr serve` and llama.cpp d1d3c3396 `llama-server` with the same GGUF on the same cell. + Strata by composed input tokens: 2k (0,2048], 8k (2048,8192], 16k (8192,16384], + 32k (16384,32768], 50 items each. One item per repo#PR and per diff sha. Sealed test items + (diff sha, hunk fingerprint, near-dup cluster) are excluded. Order within a stratum is + sha256(seed ‖ diff_sha), so the set is frozen by (version, seed, candidates). A short stratum is an + error and is never padded. Every ratio is apr / llama_cpp: TTFT targets ≤ 2.0 by E2, and prefill and + decode target ≥ 1.0 by E6. The queue budget is the p95 of the slowest counted voter, taken from + captured quorum latency. A summary refuses an inconsistent run rather than print a number. + references: + - "docs/specifications/review-experiment-protocol.md" + +equations: + stratum: + formula: "stratum(t) = min{ s ∈ (2k, 8k, 16k, 32k) : 0 < t ≤ upper(s) }, undefined for t = 0 or t > 32768" + ratio: + formula: "ratio(m) = median_apr(m) / median_llama_cpp(m); null if any row of either engine has m null" + queue_budget: + formula: "budget = max over counted voter lanes L of p95(latency_L); within_budget ⇔ p95(apr wall) ≤ budget" + verdict_identity: + formula: "identity = |{ i : verdict_tag(i) = verdict_prev_tag(i) }| / n, only on the same set_sha and the same items" + +falsification_tests: + - id: FALSIFY-REPLAY-001 + name: sealed_items_never_enter_the_set + prediction: "a sealed diff, or the same diff re-posted under another repo#PR, is dropped, and its stratum reports short rather than silently shrinking" + test_harness: "cargo test -p aprender-review-experiment --lib falsify_replay_001" + expected_output: "exit 0" + if_fails: "Sealed test diffs would be replayed and cached by a benchmark run. That is the leak §2.4 forbids." + - id: FALSIFY-REPLAY-002 + name: short_strata_are_refused_not_padded + prediction: "a stratum with fewer than per_stratum eligible items is an error that names it; an out-of-range (>32768) item is never moved into a stratum" + test_harness: "cargo test -p aprender-review-experiment --lib falsify_replay_002" + expected_output: "exit 0" + if_fails: "A 32k stratum filled from 16k inputs would report long-context prefill that was never measured." + - id: FALSIFY-REPLAY-003 + name: one_item_per_pr_and_order_free + prediction: "the set has one item per repo#PR and per diff sha, is identical under reversed candidate order, and changes with the seed" + test_harness: "cargo test -p aprender-review-experiment --lib falsify_replay_003" + expected_output: "exit 0" + if_fails: "One large PR could dominate a stratum, or re-collecting candidates could silently change the frozen set." + - id: FALSIFY-REPLAY-004 + name: ratios_are_apr_over_llama_and_budget_is_slowest_voter + prediction: "TTFT, per-stratum prefill and decode ratios equal apr/llama_cpp medians; the queue budget is the largest voter-lane p95; a single null server timing nulls the figure" + test_harness: "cargo test -p aprender-review-experiment --lib falsify_replay_004" + expected_output: "exit 0" + if_fails: "An inverted ratio would print a scoreboard win for a loss, and a subset median would hide the rows the server could not time." + - id: FALSIFY-REPLAY-005 + name: inconsistent_runs_never_print_a_number + prediction: "mixed set_sha or version, a GGUF, cell or apr_tag mismatch, unequal engine coverage, a duplicate row, any not-run row, or a missing voter latency is refused" + test_harness: "cargo test -p aprender-review-experiment --lib falsify_replay_005" + expected_output: "exit 0" + if_fails: "apr on one GGUF against llama.cpp on another, or apr with its timeouts dropped, would be reported as parity." + - id: FALSIFY-REPLAY-006 + name: verdict_identity_is_same_set_item_by_item + prediction: "identity vs the previous tag is the per-item equal-verdict fraction on the same set_sha and items; another set or missing items refuse" + test_harness: "cargo test -p aprender-review-experiment --lib falsify_replay_006" + expected_output: "exit 0" + if_fails: "A tag could change review verdicts while the speed block reported only faster wall-clock." diff --git a/contracts/rex-cell-admission-v1.yaml b/contracts/rex-cell-admission-v1.yaml new file mode 100644 index 0000000000..c33dd9d6c1 --- /dev/null +++ b/contracts/rex-cell-admission-v1.yaml @@ -0,0 +1,65 @@ +metadata: + version: 1.1.0 + kind: schema + description: | + PROMETHEUS (PRM-001, was REX-001) REX-04 per-cell admission, spec §2.1/§7. One JSONL + row per §2.1 device cell (C1 intel-wgpu, C2 intel-cpu, C3 lambda-cpu, C4 gx10-cuda, + C5a mini-cpu, C5b mini-metal) written by `rex admit` (crates/aprender-review-experiment: + admission.rs). A file is admissible only if every cell has exactly one row, each + row names its declared host and backend, carries the apr tag, the apr and weights + sha256, the model id and the locked prereg sha, and resolves to Admitted (a parity + receipt sha, an oracle, a cosine at or above a threshold with a cited basis), + Refused{removed_by}, or NotRun{reason}. A silent cell rejects the whole file. When + no cell is Admitted, the summary raises S-7 and `rex admission-check` exits 10. + references: + - "docs/specifications/review-experiment-protocol.md" + - "docs/audits/rex-001/rex-04-receipt.md" + - "https://github.com/paiml/aprender/issues/4359" + +equations: + resolved: + formula: "∀ c ∈ CELLS: |rows(c)| = 1 ∧ state(c) ∈ {Admitted, Refused, NotRun}" + admitted: + formula: "Admitted(c) ⇒ cosine(c) ≥ threshold(c) ∧ receipt_sha256(c) is 64-hex ∧ threshold_basis(c) ≠ ∅" + derived: + formula: "parity_from_receipt(R, x) = Ok(p) ⇒ n(R) ≥ x.min_positions ∧ p.cosine = min_i cos_i(R) ≥ x.threshold ∧ p.receipt_sha256 = sha256(R) ∧ sha(R).binary = x.apr_sha256 (when R carries one) ∧ sha(R).weights = x.weights_sha256 ∧ oracle(R) ∈ {llama.cpp@d1d3c3396, apr-parity-gpu-cpu}" + s7: + formula: "S-7 ⇔ |{c : Admitted(c)}| = 0" + +falsification_tests: + - id: FALSIFY-RCA-001 + name: a_silent_cell_rejects_the_file + prediction: "dropping any one of the six rows (or duplicating one) makes the admission file inadmissible" + test_harness: "cargo test -p aprender-review-experiment --lib falsify_rca_001" + expected_output: "exit 0" + if_fails: "A cell could vanish from the report without a refusal or NotRun on record (0 silent cells, §7 REX-04)." + - id: FALSIFY-RCA-002 + name: admitted_needs_a_passing_parity_receipt + prediction: "Admitted below threshold, with a NaN cosine, or without a receipt, basis or oracle is rejected; Refused without removed_by is rejected" + test_harness: "cargo test -p aprender-review-experiment --lib falsify_rca_002" + expected_output: "exit 0" + if_fails: "A cell that diverges from the oracle would be admitted into the hardware ruling." + - id: FALSIFY-RCA-003 + name: no_admitted_cell_raises_s7 + prediction: "a file whose cells are all Refused or NotRun is admissible and raises S-7" + test_harness: "cargo test -p aprender-review-experiment --lib falsify_rca_003" + expected_output: "exit 0" + if_fails: "The experiment would proceed to REX-05 with no admissible cell for (A)." + - id: FALSIFY-RCA-004 + name: admitted_is_derived_from_the_receipt + prediction: "`rex admit --parity-receipt R` fills Admitted.parity from R: the minimum cosine over R's positions (never its summary `cosine`), sha256 of R's bytes, R's oracle, and the DECLARED threshold and basis (never R's own); exactly min_positions and exactly the threshold admit" + test_harness: "cargo test -p aprender-review-experiment --lib falsify_rca_004" + expected_output: "exit 0" + if_fails: "An Admitted row would carry a hand-typed cosine or the receipt's self-chosen threshold instead of the measured worst position." + - id: FALSIFY-RCA-005 + name: a_short_or_failing_receipt_does_not_admit + prediction: "fewer than min_positions (64, evidence/parity/thresholds.yaml, I8/CF-4 #1864) positions, exit ≠ 0, a non-passing verdict, failed > 0, any position below threshold, a missing or non-numeric cosine (JSON carries no NaN), an undeclared oracle, an empty basis or a non-finite threshold is refused with the reason named" + test_harness: "cargo test -p aprender-review-experiment --lib falsify_rca_005" + expected_output: "exit 0" + if_fails: "A 7-position smoke check would admit a cell to the hardware ruling that parity evidence requires 64 positions for." + - id: FALSIFY-RCA-006 + name: the_receipt_must_be_the_rows_binary_and_weights + prediction: "a passing receipt whose binary_sha256 or weights sha256 differs from the row's apr_sha256 / weights_sha256 (or is absent) is refused" + test_harness: "cargo test -p aprender-review-experiment --lib falsify_rca_006" + expected_output: "exit 0" + if_fails: "Parity measured on another build or another model would admit this cell." diff --git a/contracts/rex-pilot-projection-v1.yaml b/contracts/rex-pilot-projection-v1.yaml new file mode 100644 index 0000000000..53677d5f51 --- /dev/null +++ b/contracts/rex-pilot-projection-v1.yaml @@ -0,0 +1,44 @@ +metadata: + version: 1.0.0 + kind: schema + description: | + PROMETHEUS (PRM-001, was REX-001) REX-05 pilot projection, spec §2.2/§7. `rex score + --pilot --split dev` (crates/aprender-review-experiment: pilot.rs) projects the + test-split Wilson 95 % half-width on recall at the pilot's point estimate (or at the + §2.2 design p = 0.5 when the pilot has no defect item in its denominator) and applies + the sample-size rule: above 0.12 the corpus grows and the hypotheses do not change. + An uncomputable width (no test defect items) also grows the corpus. The runtime + projection counts the 10 % determinism re-run and is None, never zero, when nothing + warm ran. The spec's "about 0.117" at 70 defects is the Wald width; the frozen + stats.rs Wilson width the rule reads is about 0.114. + references: + - "docs/specifications/review-experiment-protocol.md" + - "https://github.com/paiml/aprender/issues/4360" + +equations: + half_width: + formula: "h = wilson_half_width(round(p · n_def), n_def), p = recall_pilot if defined else 0.5" + rule: + formula: "grow_corpus ⇔ h undefined ∨ h > 0.12" + runtime: + formula: "requests = n_test + ⌈n_test / 10⌉; runtime_s = requests · mean(warm_ms) / 1000, undefined if no warm sample" + +falsification_tests: + - id: FALSIFY-RXP-001 + name: a_wide_projection_grows_the_corpus + prediction: "a projected half-width above 0.12, or one that cannot be computed, sets grow_corpus" + test_harness: "cargo test -p aprender-review-experiment --lib falsify_rxp_001" + expected_output: "exit 0" + if_fails: "The full run would start on a corpus too small to resolve the §2.2 interval." + - id: FALSIFY-RXP-002 + name: the_projection_uses_the_pilot_estimate + prediction: "the rule reads the pilot's recall, not a fixed p; 9/10 at n=50 keeps the corpus and 5/10 grows it" + test_harness: "cargo test -p aprender-review-experiment --lib falsify_rxp_002" + expected_output: "exit 0" + if_fails: "The sample-size decision would ignore the pilot it exists to read." + - id: FALSIFY-RXP-003 + name: runtime_counts_the_rerun_and_nothing_warm_is_unknown + prediction: "145 test items project 160 requests; no warm sample or a non-finite one projects None" + test_harness: "cargo test -p aprender-review-experiment --lib falsify_rxp_003" + expected_output: "exit 0" + if_fails: "The runtime budget would under-count the re-run or report zero seconds for a cell that never ran." diff --git a/contracts/rex-prereg-v1.yaml b/contracts/rex-prereg-v1.yaml new file mode 100644 index 0000000000..401e5400f4 --- /dev/null +++ b/contracts/rex-prereg-v1.yaml @@ -0,0 +1,52 @@ +metadata: + version: 3.0.0 + kind: schema + description: | + PRM-001 pre-registration lock (rule R-1). Spec §2–§5, the analysis code + (crates/aprender-review-experiment/src/stats.rs), the analysis plan and prompt v1 + are frozen by sha256 in a lock file. The prereg sha is carried by every + review-experiment receipt; a tree that no longer matches the lock is a new spec + version and its data is exploratory. The contract id keeps its v1 name. + - rex-prereg-v1 (REX-00, 5f0ae10ac): docs/audits/rex-001/prereg.lock over + review-experiment-protocol.md. SUPERSEDED, never deleted or edited (operator + 2026-09-25); its spec §2–§5 and plan stay byte-frozen. + - rex-prereg-v2 (PRM-00b): docs/audits/prm-001/prereg.lock over PRM-001 v3 + (docs/specifications/PRM-001-prometheus.md) and analysis plan v2. v1 had 0 data rows. + The 17:09Z v2 draft lock (1f4acb3d) over the v2 ruling was a stray draft and is void. + references: + - "docs/specifications/PRM-001-prometheus.md" + - "docs/audits/prm-001/analysis-plan.md" + - "docs/specifications/review-experiment-protocol.md" + - "https://github.com/paiml/aprender/issues/4355" + +equations: + prereg_sha: + formula: "sha256('rex-prereg-v2\\n' || 'spec_s2_s5 ' || H(spec[§2..§6)) || '\\nstats_rs ' || H(stats.rs) || '\\nanalysis_plan ' || H(plan) || '\\nprompt_v1 ' || H(prompt) || '\\n')" + locked: + formula: "prereg.lock[k] == H(component_k) for every k, and prereg.lock[prereg_sha] == prereg_sha" + +falsification_tests: + - id: FALSIFY-REX-PREREG-001 + name: lock_matches_tree + prediction: "the committed v2 lock matches every frozen component" + test_harness: "cargo test -p aprender-review-experiment --lib falsify_rex_prereg_001" + expected_output: "exit 0" + if_fails: "A frozen component changed after PRM-00b. That is a new spec version (R-1): label prior data exploratory and cut v3; never edit the lock to match." + - id: FALSIFY-REX-PREREG-002 + name: planted_section3_edit_is_caught + prediction: "an edit to §3 (hypotheses) changes the prereg sha" + test_harness: "cargo test -p aprender-review-experiment --lib falsify_rex_prereg_002" + expected_output: "exit 0" + if_fails: "The §2–§5 span no longer covers §3; the lock is vacuous." + - id: FALSIFY-REX-PREREG-003 + name: out_of_scope_edit_is_not_a_version + prediction: "an edit to the header (outside §2–§5) leaves the prereg sha unchanged" + test_harness: "cargo test -p aprender-review-experiment --lib falsify_rex_prereg_003" + expected_output: "exit 0" + if_fails: "The span leaks past §6; routine header edits would invalidate data." + - id: FALSIFY-REX-PREREG-004 + name: v1_lock_is_frozen_and_verifies + prediction: "the v1 lock still carries prereg_sha ef51087d… and its spec §2–§5, plan and prompt digests match the in-tree v1 files" + test_harness: "cargo test -p aprender-review-experiment --lib falsify_rex_prereg_004" + expected_output: "exit 0" + if_fails: "A superseded lock or the v1 bytes it froze were edited (operator: never deleted or edited)." diff --git a/contracts/rex-promotion-ladder-v1.yaml b/contracts/rex-promotion-ladder-v1.yaml new file mode 100644 index 0000000000..76c05f0b48 --- /dev/null +++ b/contracts/rex-promotion-ladder-v1.yaml @@ -0,0 +1,48 @@ +metadata: + version: 2.0.0 + kind: schema + description: | + SUPERSEDED by rex-promotion-ladder-v2 (PRM-09, the tie-breaker rung); kept as the v2-spec record. + PROMETHEUS (PRM-001, was REX-001) REX-09 promotion ladder, spec §3 H4/H5, §4.5, R-11. + `rex ladder REPORT_JSON` (crates/aprender-review-experiment: ladder.rs) reads only a + `rex-001-report-v2` report (spec v2) whose prereg_sha is the lock and that is not exploratory, + and returns the 4B lane's rung: shadow by default; tripwire when H4 holds (explicit + `holds` verdict and a finite Holm-adjusted one-sided p ≤ 0.05; the CI is report-only); vote when H4 + and H5 both hold. H5 without H4 stays shadow. The §4.5 hardware ruling mode caps the + rung, and a report without one is shadow. The paiml-implement quorum gate and arbiter + `decide` consume `.mode`; they never re-derive it. + references: + - "docs/specifications/review-experiment-protocol.md" + - "https://github.com/paiml/aprender/issues/4364" + +equations: + ladder: + formula: "mode = min(ruling.mode, H4 ∧ H5 ? vote : H4 ? tripwire : shadow)" + holds: + formula: "holds(H) ⇔ verdict(H) = holds ∧ p_holm(H) finite ∧ 0 ≤ p_holm(H) ≤ 0.05" + +falsification_tests: + - id: FALSIFY-RXG-001 + name: a_failing_h4_keeps_the_lane_in_shadow + prediction: "the planted report with H4 failing, a holds verdict with Holm p > 0.05, NaN, negative or missing, or H4 absent all stay shadow" + test_harness: "cargo test -p aprender-review-experiment --lib falsify_rxg_001" + expected_output: "exit 0" + if_fails: "The 4B lane would enter tripwire without evidence its precision matches the voters (R-11)." + - id: FALSIFY-RXG-002 + name: h4_alone_is_tripwire_never_vote + prediction: "H4 with H5 failing is tripwire; a shadow hardware ruling or no ruling caps both gates passing at shadow" + test_harness: "cargo test -p aprender-review-experiment --lib falsify_rxg_002" + expected_output: "exit 0" + if_fails: "The lane could cast a deciding vote without the recall gate or lengthen the quorum past the queue budget." + - id: FALSIFY-RXG-003 + name: only_a_locked_confirmatory_report_is_read + prediction: "a v1 or foreign schema, a stale prereg sha, an exploratory report or unparsable JSON stays shadow" + test_harness: "cargo test -p aprender-review-experiment --lib falsify_rxg_003" + expected_output: "exit 0" + if_fails: "A promotion could be read from a report outside the pre-registration (R-1)." + - id: FALSIFY-RXG-004 + name: the_ci_is_report_only + prediction: "a CI crossing 0 beside Holm p ≤ 0.05 is vote; a clean CI beside Holm p > 0.05 is shadow" + test_harness: "cargo test -p aprender-review-experiment --lib falsify_rxg_004" + expected_output: "exit 0" + if_fails: "Two H4 decision rules would coexist and could disagree (PRM-001 v2 ruling item 3c)." diff --git a/contracts/rex-promotion-ladder-v2.yaml b/contracts/rex-promotion-ladder-v2.yaml new file mode 100644 index 0000000000..341a4a2ca6 --- /dev/null +++ b/contracts/rex-promotion-ladder-v2.yaml @@ -0,0 +1,84 @@ +metadata: + version: 1.1.0 + kind: schema + description: | + PRM-09 (PRM-001 v3 "PROMETHEUS" §5.4, §7, §9): the promotion ladder with the tie-breaker rung. + Supersedes rex-promotion-ladder-v1. `rex ladder REPORT_JSON [--current RUNG]` + (crates/aprender-review-experiment: ladder.rs, examples/rex.rs) reads only a `prm-001-report-v1` + report (the §9 template: spec PRM-001, version 3) whose prereg_sha is the lock and that is not + exploratory. Anything else, including the older rex-001-report-v2, is read as no evidence (shadow). + Rungs, lowest first: shadow < tripwire < tie-breaker < vote. + tripwire ⇔ H4 ∧ speed.within_budget (replay p95 ≤ the queue budget). + tie-breaker ⇔ tripwire ∧ H5 ∧ H7 ∧ H1 ∧ H2 ∧ lane.availability_7d ≥ 0.95. It decides only a 1–1 split. + vote ⇔ tie-breaker ∧ promotion.tie_breaks_decided ≥ 20 ∧ + promotion.tie_break_outcome_agreement ≥ promotion.voter_outcome_agreement. + H4 and H5 hold on a `holds` verdict with a finite Holm-adjusted p in [0, 0.05]; the CI is report-only. + H1 and H2 (count tests) and H7 (δ-free, outside the Holm family) hold on a `holds` verdict. + hardware_ruling.rung_eligible caps the rung; a report without a ruling is shadow. + A primary cell that is lambda, or is not named, caps the lane at tripwire (R-9, STOP S-4). + The ladder is sequential: from the current rung it moves at most one rung up, and a demotion + drops exactly one rung. The S-4 cap is the only exception and applies at once. + The paiml-implement quorum gate and the arbiter read `.mode`; they never re-derive it. + references: + - "docs/specifications/PRM-001-prometheus.md" + - "contracts/rex-promotion-ladder-v1.yaml" + +equations: + evidence: + formula: "e = H4 ∧ fast ? (H5 ∧ H7 ∧ H1 ∧ H2 ∧ avail ≥ 0.95 ? (n_tb ≥ 20 ∧ agree_tb ≥ agree_voters ? vote : tie-breaker) : tripwire) : shadow" + cap: + formula: "c = min(e, ruling.rung_eligible, primary is lambda or absent ? tripwire : vote); no ruling ⇒ c = shadow" + step: + formula: "mode = primary is lambda or absent ∧ current > tripwire ? tripwire : clamp(c, current − 1, current + 1)" + holds: + formula: "holds(H4|H5) ⇔ verdict = holds ∧ p_holm finite ∧ 0 ≤ p_holm ≤ 0.05; holds(H1|H2|H7) ⇔ verdict = holds" + +falsification_tests: + - id: FALSIFY-RXG-001 + name: a_failing_h4_keeps_the_lane_in_shadow + prediction: "the planted report with H4 failing, a holds verdict with Holm p > 0.05, NaN, negative or missing, or H4 absent all stay shadow" + test_harness: "cargo test -p aprender-review-experiment --lib falsify_rxg_001" + expected_output: "exit 0" + if_fails: "The lane would enter tripwire without evidence its precision matches the voters." + - id: FALSIFY-RXG-002 + name: h4_alone_is_tripwire_never_vote + prediction: "the planted H4-only report is tripwire; H5 failing names H5 and stays tripwire; within_budget false or absent is shadow; rung_eligible caps; a ruling without rung_eligible, or no ruling, is shadow" + test_harness: "cargo test -p aprender-review-experiment --lib falsify_rxg_002" + expected_output: "exit 0" + if_fails: "A lane over the queue budget, or with only H4, would cast deciding votes." + - id: FALSIFY-RXG-003 + name: only_a_locked_confirmatory_report_is_read + prediction: "a rex-001-report-v2 report, a version-2 spec, a stale prereg_sha, an exploratory report and non-JSON are all shadow" + test_harness: "cargo test -p aprender-review-experiment --lib falsify_rxg_003" + expected_output: "exit 0" + if_fails: "An unlocked or exploratory analysis could promote the lane." + - id: FALSIFY-RXG-004 + name: the_ci_is_report_only + prediction: "a CI crossing 0 with Holm p ≤ 0.05 still holds; a clean CI with Holm p > 0.05 does not" + test_harness: "cargo test -p aprender-review-experiment --lib falsify_rxg_004" + expected_output: "exit 0" + if_fails: "The ladder would decide on a statistic the prereg made report-only." + - id: FALSIFY-RXG-005 + name: tie_breaker_needs_h4_h5_h7_h1_h2_and_availability + prediction: "the planted H4+H5+H7 report (H1, H2 holding, availability 0.97) is tie-breaker; without or failing any of H5, H7, H1, H2 it is tripwire and names it (H5 also needs Holm p ≤ 0.05); availability 0.95 is tie-breaker, 0.9499 or absent is tripwire" + test_harness: "cargo test -p aprender-review-experiment --lib falsify_rxg_005" + expected_output: "exit 0" + if_fails: "A lane that is unavailable or nondeterministic would break ties in the quorum." + - id: FALSIFY-RXG-006 + name: lambda_primary_is_refused_above_tripwire + prediction: "a report passing every gate with hardware_ruling.primary lambda-cuda is tripwire with an S-4 reason naming the cell as written (lambda-cuda, never a Rust Debug Some(..)); a ruling without a primary is also tripwire and is named (unnamed)" + test_harness: "cargo test -p aprender-review-experiment --lib falsify_rxg_006" + expected_output: "exit 0" + if_fails: "The operator's desktop GPU would sit on the merge path (R-9)." + - id: FALSIFY-RXG-007 + name: vote_needs_twenty_agreeing_tie_breaks + prediction: "19 decided tie-breaks, tie-break agreement below the voters', no promotion block, or no voter agreement all stay tie-breaker" + test_harness: "cargo test -p aprender-review-experiment --lib falsify_rxg_007" + expected_output: "exit 0" + if_fails: "The lane would vote on ties it never decided, or decided worse than the voters." + - id: FALSIFY-RXG-008 + name: never_skips_a_rung_and_demotes_one + prediction: "full evidence from shadow → tripwire, tripwire → tie-breaker, tie-breaker → vote; no evidence from vote → tie-breaker, tripwire → shadow; a tripwire-only report takes vote to tie-breaker; the lambda cap takes vote or tie-breaker to tripwire at once, never promotes a shadow lane past its evidence" + test_harness: "cargo test -p aprender-review-experiment --lib falsify_rxg_008" + expected_output: "exit 0" + if_fails: "One good report would jump the lane to vote, or one bad report would drop it to shadow." diff --git a/contracts/sparse-logits-v1.yaml b/contracts/sparse-logits-v1.yaml new file mode 100644 index 0000000000..a340c1b041 --- /dev/null +++ b/contracts/sparse-logits-v1.yaml @@ -0,0 +1,50 @@ +metadata: + version: 1.0.0 + kind: schema + description: | + PRA-001 T11 (spec §2.7): the `sparse-logits-v1` teacher-logit blob written beside apr serve + rows (crates/aprender-review-experiment: sparse_logits.rs). Per token it stores the sampled + id, the top-k ids (u32) with log-softmax at T = 1.0 before any sampling filter (f16), the + full-vocab logsumexp (f32), and the retained mass M = Σexp(logprobs) (f16) so that 1 − M is + the true tail mass a sparse forward-KL correction needs. Header (closed JSON): schema, mode + (topk | rs-sample), model_sha256, tokenizer_sha256, apr_tag, backend (served_by), + temperature_of_record, k, n_tokens. Body: one zstd frame of LE columns. Raw column size is + n·(10 + 6k) bytes = 130 B/token at k = 20. Frontier lanes carry no blob (logits_sha = null). + Encoder and decoder share one validator: a blob that exists is a blob that reads. + references: + - "/mnt/nvme-raid0/cop-inbox/handoff/brief-pra.md §2.7 (private)" + - "https://github.com/paiml/aprender/issues/4026" + +equations: + logprob: + formula: "lp_i = z_i − logsumexp(z), z at T = 1.0, lp_i ≤ 0" + mass: + formula: "M = Σ_{i ∈ topk} exp(lp_i) ∈ [0, 1], |M − Σexp(f16(lp_i))| ≤ 2^-8; residual = 1 − M" + raw_bytes: + formula: "raw(n, k) = n · (4 + 6k + 4 + 2)" + +falsification_tests: + - id: FALSIFY-SPL-001 + name: round_trip_is_exact + prediction: "header and every column (incl. ids > 65535) decode bit-identical to what was encoded" + test_harness: "cargo test -p aprender-review-experiment --lib falsify_spl_001" + expected_output: "exit 0" + if_fails: "Stored teacher distributions would differ from the served ones; Qwen ids above u16 would alias." + - id: FALSIFY-SPL-002 + name: residual_mass_is_the_true_tail + prediction: "1 − M equals the full-softmax mass outside the top-k within 1e-3, and survives the round trip" + test_harness: "cargo test -p aprender-review-experiment --lib falsify_spl_002" + expected_output: "exit 0" + if_fails: "A top-k cache without its tail mass is a biased teacher estimate (§2.7)." + - id: FALSIFY-SPL-003 + name: inconsistent_mass_is_refused + prediction: "a mass disagreeing with its own logprobs is refused by encode AND by decode of a forged blob" + test_harness: "cargo test -p aprender-review-experiment --lib falsify_spl_003" + expected_output: "exit 0" + if_fails: "A producer defect (post-filter mass, wrong temperature) would enter the cache silently." + - id: FALSIFY-SPL-004 + name: decoder_is_strict + prediction: "an unknown header key, an n_tokens that mismatches the columns, any truncation, a trailing byte or a bad magic is an error" + test_harness: "cargo test -p aprender-review-experiment --lib falsify_spl_004" + expected_output: "exit 0" + if_fails: "A malformed blob would decode to a plausible but wrong distribution." diff --git a/contracts/trace-admission-prov-v1.yaml b/contracts/trace-admission-prov-v1.yaml new file mode 100644 index 0000000000..1878367c87 --- /dev/null +++ b/contracts/trace-admission-prov-v1.yaml @@ -0,0 +1,48 @@ +metadata: + version: 1.0.0 + kind: schema + description: | + PRM-C13 G-PROV (PRM-001 v3 "PROMETHEUS"): no hosted-model output enters the training pool. + `pool::admit` (crates/aprender-review-experiment: pool.rs, reusing PRM-C12 terms.rs `tags`/`Provider`) + decides each candidate row's eligibility by provenance BEFORE the secret scan, so a hosted row + never reaches the quarantine, whose rows may be released after review. A row is eligible only if + it is local: provider `local` with a weights-sha model_id and a full terms tag (terms.rs), or it + carries a `label_source` in the gold set {outcome, hrq_ruling, sealed_corpus}. A row from a hosted + provider (anthropic, google) is Ineligible(Hosted) whatever else it claims; a row with no or a + malformed provenance tag is Ineligible(Untagged); any other label_source is Ineligible(NotGold). + `label_source` is a provisional gold marker until PRM-C10 fixes the label schema. + references: + - "docs/specifications/PRM-001-prometheus.md" + - "contracts/agent-trace-terms-v1.yaml" + +equations: + eligible: + formula: "eligible(r) ⇔ json(r) ∧ provider(r) ∉ {anthropic, google} ∧ (label_source(r) ∈ {outcome, hrq_ruling, sealed_corpus} ∨ (label_source(r) absent ∧ tags(r) = Ok ∧ provider(r) = local))" + order: + formula: "verdict(r) = Refused(sealed) if sealed hit; else Ineligible(p) if ¬eligible(r); else Quarantined if secret(r); else Admitted" + +falsification_tests: + - id: FALSIFY-TAP-001 + name: planted_hosted_row_is_ineligible + prediction: "anthropic and google rows planted in a pool beside a local row are Ineligible(Hosted); the rebuilt pool holds only the local row" + test_harness: "cargo test -p aprender-review-experiment --lib falsify_tap_001" + expected_output: "exit 0" + if_fails: "Hosted-model outputs would train the shadow lane, breaching the provider terms and making it a copy of the voters." + - id: FALSIFY-TAP-002 + name: hosted_row_is_ineligible_whatever_it_claims + prediction: "a hosted row claiming label_source outcome, or carrying a secret, is still Ineligible(Hosted) and never Quarantined" + test_harness: "cargo test -p aprender-review-experiment --lib falsify_tap_002" + expected_output: "exit 0" + if_fails: "A gold claim or the quarantine's later release would be a path for hosted output into the pool." + - id: FALSIFY-TAP-003 + name: untagged_row_is_ineligible + prediction: "a bare row, a misspelt provider (\"Anthropic\") and a non-JSON row are Ineligible(Untagged)" + test_harness: "cargo test -p aprender-review-experiment --lib falsify_tap_003" + expected_output: "exit 0" + if_fails: "A row with unknown provenance would be admitted by default; admission must fail closed." + - id: FALSIFY-TAP-004 + name: gold_labels_are_eligible_and_only_gold + prediction: "each gold label_source is admitted; quorum_majority is Ineligible(NotGold)" + test_harness: "cargo test -p aprender-review-experiment --lib falsify_tap_004" + expected_output: "exit 0" + if_fails: "Silver (voter-majority) labels would pass as gold and train the shadow lane on the voters' errors." diff --git a/contracts/trace-admission-secret-v1.yaml b/contracts/trace-admission-secret-v1.yaml new file mode 100644 index 0000000000..263461b0bf --- /dev/null +++ b/contracts/trace-admission-secret-v1.yaml @@ -0,0 +1,81 @@ +metadata: + version: 1.0.0 + kind: schema + description: | + Trace admission (PRA-001 T6, gates G-SEC / G-SEC-FN; agent-trace ruling 2026-09-25 + item 6). A captured lane row enters the fine-tune/distill pool only when it is + secret-scan clean AND carries no sealed-test hash (review-corpus-contamination-v1). + The secret scan is the UNION of two in-process Rust scanners, no Go or Python: + (a) builtin detectors (AWS key id/secret, GitHub token, PEM private key, Anthropic, + Slack, JWT, high-entropy assignment), which honour no allowlist; (b) the gitleaks + v8.30.1 ruleset vendored verbatim as data (data/gitleaks/, MIT, sha256-pinned) and + interpreted in-process: keyword prefilter, regex, secret group, entropy floor, + per-rule and global allowlists. Without a file path the scan fails safe: a + path-scoped rule runs on all content, a path-conditioned (AND) allowlist allows + nothing, and the one path-only rule (pkcs12-file) is listed, not dropped. + Any hit QUARANTINES the row byte-for-byte; it is never redacted in place. A + sealed-test hit REFUSES the row and is checked first. An empty sealed index admits + nothing (Unchecked). Canaries are built at run time, so no key literal is in source. + Implementation: crates/aprender-review-experiment/src/{secret,pool}.rs. + Pending on T1/T3 (the trace index does not exist yet): the backfill run + (`make trace-secret-scan-backfill`, G11) and the index's secret_scan record + {scanner_versions, hits, status}; secret::SCANNER_VERSIONS is its version field. + references: + - "contracts/review-corpus-contamination-v1.yaml" + - "docs/specifications/review-experiment-protocol.md" + +equations: + admit: + formula: "pool(R) = { r in R : sealed != {} and leak(r) = false and builtin(r) ∪ gitleaks(r) = {} }" + recall: + formula: "for each canary c labelled (scanner s, rule k): k in scan(c), and scan_s(c) without k = {}" + +falsification_tests: + - id: FALSIFY-TAS-001 + name: every_canary_fires_its_labelled_rule + prediction: "all 20 canaries (8 builtin, 12 gitleaks: AWS, GitHub PAT, private key, JWT, high-entropy generic, …) fire their labelled rule, bare and inside a JSONL row: 100% recall" + test_harness: "cargo test -p aprender-review-experiment --lib falsify_tas_001" + expected_output: "exit 0" + if_fails: "A secret shape the gate claims to catch reaches the pool (G-SEC)." + - id: FALSIFY-TAS-002 + name: removing_a_rule_turns_its_canary_red + prediction: "with a canary's labelled rule removed, its scanner reports nothing on that canary" + test_harness: "cargo test -p aprender-review-experiment --lib falsify_tas_002" + expected_output: "exit 0" + if_fails: "Recall is carried by some other rule, so the per-rule recall claim is untested (G-SEC-FN)." + - id: FALSIFY-TAS-003 + name: aws_doc_key_needs_the_builtin_scanner + prediction: "the AWS documentation key, which gitleaks allowlists (.+EXAMPLE$), is hit by the builtin scanner alone; with that rule removed nothing catches it" + test_harness: "cargo test -p aprender-review-experiment --lib falsify_tas_003" + expected_output: "exit 0" + if_fails: "The union is not load-bearing, or an upstream allowlist is silently trusted." + - id: FALSIFY-TAS-004 + name: planted_aws_key_is_quarantined + prediction: "a lane output carrying the AWS key literal is Quarantined byte-identical and is not in the pool; a clean neighbour is admitted" + test_harness: "cargo test -p aprender-review-experiment --lib falsify_tas_004" + expected_output: "exit 0" + if_fails: "Secrets reach the training pool, or a quarantined row was rewritten in place." + - id: FALSIFY-TAS-005 + name: planted_test_split_diff_is_refused + prediction: "a row carrying a sealed test diff, or only its diff sha, is Refused and not in the pool" + test_harness: "cargo test -p aprender-review-experiment --lib falsify_tas_005" + expected_output: "exit 0" + if_fails: "Sealed test items leak into training data (S-3)." + - id: FALSIFY-TAS-006 + name: scanner_case_tables + prediction: "builtin must-match rows (incl. a JSON-escaped key) fire exactly their rule and must-not-match rows fire none; gitleaks drops a sub-entropy match and does not apply a path-conditioned allowlist; ordinary review rows fire neither scanner" + test_harness: "cargo test -p aprender-review-experiment --lib -- case_table ordinary_review_rows_are_clean" + expected_output: "exit 0" + if_fails: "A key shape is missed, or clean rows are quarantined." + - id: FALSIFY-TAS-007 + name: ruleset_pinned_and_complete + prediction: "the vendored toml matches its sha256; all 222 declared rules are either compiled or listed path-only; no regex fails to compile" + test_harness: "cargo test -p aprender-review-experiment --lib -- vendored_gitleaks_toml_is_sha_pinned every_gitleaks_rule_compiles" + expected_output: "exit 0" + if_fails: "The ruleset drifted from upstream, or a rule was silently dropped." + - id: FALSIFY-TAS-008 + name: empty_sealed_index_admits_nothing + prediction: "with no sealed items indexed, no row is admitted" + test_harness: "cargo test -p aprender-review-experiment --lib empty_sealed_index_refuses_everything" + expected_output: "exit 0" + if_fails: "A missing test manifest silently admits every row." diff --git a/contracts/trace-datacard-v1.yaml b/contracts/trace-datacard-v1.yaml new file mode 100644 index 0000000000..32d04d3640 --- /dev/null +++ b/contracts/trace-datacard-v1.yaml @@ -0,0 +1,89 @@ +metadata: + version: 1.0.0 + kind: schema + description: | + PRA-001 T14 (spec §2.11): the dataset card for one snapshot of the agent-trace-v1 index. + aprender_review_experiment::datacard reads every index/agent-trace-v1/YYYY/MM/DD.jsonl under + the traces root, in path order. It binds them with a sha256sum manifest, and the snapshot id + is the first 16 hex of the manifest's sha256. For that snapshot it generates a Croissant 1.0 + + RAI 1.0 JSON-LD card and a Datasheet (motivation, composition, collection, preprocessing, + uses, distribution, maintenance) under datacard//, regenerated per snapshot and + never hand-edited. + The official validator (mlcroissant) is Python, which S-PY forbids, so datacard::validate is a + structural Croissant 1.0 + RAI checker in Rust. It checks the context, sc:Dataset, + conformsTo, the resources and their sha256, the record set, dataTypes, @id uniqueness, + reference resolution, jsonPath and the ten RAI fields. write_snapshot refuses to write a + card that fails it. + The card carries aggregate counts only. HF publication is gold AND provider local AND + secret-scan clean; provider anthropic and google rows are never public-eligible. + PRM-001 v2 PRM-C14 (was PRA T14) adds the weekly yield/bytes receipt, trace-weekly-receipt-v1 + (weekly.json beside the card). Per ISO 8601 week it records quorums/day, rows, index bytes and + the CAS blobs the rows address: stored bytes, plus raw bytes from the zstd frame header + (RFC 8878). It checks them against the G16 [O] band (50-200 quorums/day, <= 0.45 TB/yr raw) + and reads within | outside | unmeasured. A missing blob or an undeclared raw size is + unmeasured, never within. + references: + - "docs/specifications/research-design-pr-agent-qwen-3.5.md" + - "http://mlcommons.org/croissant/1.0" + - "http://mlcommons.org/croissant/RAI/1.0" + - "https://github.com/paiml/aprender/issues/4354" + +equations: + snapshot: + formula: "id = sha256(manifest)[0..16]; manifest = ⊕_{f ∈ sort(index files)} sha256(bytes(f)) ‖ ' ' ‖ path(f)" + public: + formula: "eligible(row) ⇔ label_tier = gold ∧ provider = local ∧ secret_scan.status = clean" + card: + formula: "write(card) ⇒ validate(card) = ∅; exists(dir/f) ∧ bytes(dir/f) ≠ regen(f) ⇒ refuse" + g16: + formula: "within(w) ⇔ missing(w) + undeclared(w) = 0 ∧ 50 ≤ min_d q(d) ∧ max_d q(d) ≤ 200 ∧ (index(w) + raw(w)) / days(w) · 365 ≤ 0.45 TB" + +falsification_tests: + - id: FALSIFY-TDC-001 + name: the_generated_card_passes_croissant_validation + prediction: "the generated card validates clean. Each planted defect (no conformsTo, empty name, wrong @type, FileObject with no sha256, source naming an unknown resource, duplicate @id, unknown dataType, missing rai:dataLimitations, @context missing rai) is refused with an error that names it" + test_harness: "cargo test -p aprender-review-experiment --lib falsify_tdc_001" + expected_output: "exit 0" + if_fails: "A card that is not Croissant 1.0 + RAI could be published, or the validator passes anything." + - id: FALSIFY-TDC-002 + name: a_snapshot_is_deterministic_and_bound_to_the_index_bytes + prediction: "the same index gives the same snapshot id, card bytes and datasheet; every manifest line's sha equals the file's sha256; the card's manifest sha equals sha256(manifest); one appended day gives a new snapshot id and sha" + test_harness: "cargo test -p aprender-review-experiment --lib falsify_tdc_002" + expected_output: "exit 0" + if_fails: "A card would not identify which index bytes it describes, or would not change when the index does." + - id: FALSIFY-TDC-003 + name: only_gold_local_rows_are_public_eligible + prediction: "gold anthropic, gold google, silver local and pending rows are not public-eligible, nor is a gold local row with a secret-scan hit; composition counts per provider, label_tier and split are exact" + test_harness: "cargo test -p aprender-review-experiment --lib falsify_tdc_003" + expected_output: "exit 0" + if_fails: "The card would count provider anthropic/google rows or unscanned rows as releasable (§2.11)." + - id: FALSIFY-TDC-004 + name: a_hand_edited_card_is_refused_and_regeneration_is_idempotent + prediction: "write_snapshot writes croissant.json, datasheet.md and manifest.sha256 under datacard//; a second run on the same index is a no-op; after a hand edit the next run errors with 'hand-edited' and leaves the edited bytes in place" + test_harness: "cargo test -p aprender-review-experiment --lib falsify_tdc_004" + expected_output: "exit 0" + if_fails: "A hand-edited card would be silently kept or silently clobbered." + - id: FALSIFY-TDC-005 + name: no_row_content_reaches_the_card + prediction: "a sentinel finding text, a diff sha and an output sha planted in the fixture rows appear in neither the card nor the datasheet" + test_harness: "cargo test -p aprender-review-experiment --lib falsify_tdc_005" + expected_output: "exit 0" + if_fails: "Trace content would leak into a card (PRIVATE, §6)." + - id: FALSIFY-TDC-006 + name: a_malformed_index_is_refused_not_skipped + prediction: "a bad JSON line errors with its file:line; a row whose schema is not agent-trace-v1 errors with its schema; an empty index errors 'no index'" + test_harness: "cargo test -p aprender-review-experiment --lib falsify_tdc_006" + expected_output: "exit 0" + if_fails: "A card would describe an index with rows silently dropped." + - id: FALSIFY-TDC-007 + name: the_weekly_receipt_measures_g16_or_says_unmeasured + prediction: "per ISO week the receipt counts rows, distinct quorums/day, index bytes and distinct blobs; a missing blob or an undeclared zstd size reads unmeasured; all blobs measured at 1-2 quorums/day reads outside; 50 quorums/day with tiny blobs reads within; no sha or row text appears in it" + test_harness: "cargo test -p aprender-review-experiment --lib falsify_tdc_007" + expected_output: "exit 0" + if_fails: "G16 could read verified from a week whose bytes were never measured." + - id: FALSIFY-TDC-008 + name: iso_weeks_and_zstd_sizes_match_their_standards + prediction: "iso_week agrees with ISO 8601 across year boundaries (2027-01-01 = 2026-W53, 2024-12-30 = 2025-W01, 2021-01-03 = 2020-W53); zstd_content_size reads 1/2/4-byte FCS (2-byte is +256), skips a dict id, and returns None for no FCS, a non-zstd magic or a truncated header" + test_harness: "cargo test -p aprender-review-experiment --lib falsify_tdc_008" + expected_output: "exit 0" + if_fails: "Rows would land in the wrong week, or raw bytes would be misread." diff --git a/contracts/trace-dedup-v1.yaml b/contracts/trace-dedup-v1.yaml new file mode 100644 index 0000000000..f7256a82ae --- /dev/null +++ b/contracts/trace-dedup-v1.yaml @@ -0,0 +1,57 @@ +metadata: + version: 1.0.0 + kind: schema + description: | + PRM-001 v3 PRM-C8 (was PRA T8), gate G-DUP. `dedup::clusters` + (crates/aprender-review-experiment/src/dedup.rs) groups trace diffs into near-duplicate + clusters so the split guard (PRM-C9, trace-split-guard-v1) can close each cluster to + one split. It reuses the G-CON fingerprint in cluster.rs (identifiers normalised to `$`, + whitespace dropped, 12-token shingles, bottom-64 sketch), so a re-indented, re-pathed or + renamed copy is the same diff here as it is to the contamination check. Candidates come + from an inverted index over sketch hashes (LSH); a candidate pair matches when either + sketch is at least 0.7 contained in the other's shingle set; union-find closes chains. + Diffs too short to sketch merge only on identical normalised tokens. A cluster's id is + its earliest member by (at, id), independent of input order. Recall bound, stated: two + items whose sketches share no hash are never compared. + references: + - "docs/specifications/research-design-pr-agent-qwen-3.5.md" + - "contracts/review-corpus-contamination-v1.yaml" + - "https://github.com/paiml/aprender/issues/4354" + +equations: + candidate: + formula: "cand(i, j) ⇔ sketch(i) ∩ sketch(j) ≠ ∅" + near_dup: + formula: "near(i, j) ⇔ cand(i, j) ∧ max(|sketch(i) ∩ S(j)| / |sketch(i)|, |sketch(j) ∩ S(i)| / |sketch(j)|) ≥ 0.7" + tiny: + formula: "|S(i)| < 32 ⇒ (i ~ j ⇔ tokens(i) = tokens(j))" + closure: + formula: "cluster = transitive closure of near ∪ tiny" + cluster_id: + formula: "id(C) = argmin_{m ∈ C} (at(m), id(m)); invariant under permutation of the input" + +falsification_tests: + - id: FALSIFY-TDD-001 + name: a_perturbed_copy_joins_its_original_and_an_unrelated_diff_does_not + prediction: "a re-pathed, re-indented, renamed copy and a diff embedded in a larger one share the original's cluster, in either arrival order; an unrelated diff stays alone" + test_harness: "cargo test -p aprender-review-experiment --lib falsify_tdd_001" + expected_output: "exit 0" + if_fails: "A copy of a test item could sit in train under a different sha." + - id: FALSIFY-TDD-002 + name: the_cluster_id_is_the_earliest_member_whatever_the_order + prediction: "the id is the member with the least (at, id), for any input order" + test_harness: "cargo test -p aprender-review-experiment --lib falsify_tdd_002" + expected_output: "exit 0" + if_fails: "A cluster's split would depend on read order, not on time." + - id: FALSIFY-TDD-003 + name: near_dup_chains_close_transitively + prediction: "X~Y and Y~Z put X, Y, Z in one cluster although X and Z alone do not match" + test_harness: "cargo test -p aprender-review-experiment --lib falsify_tdd_003" + expected_output: "exit 0" + if_fails: "A chain of edits could straddle splits one link at a time." + - id: FALSIFY-TDD-004 + name: tiny_diffs_merge_only_when_identical + prediction: "diffs below 32 shingles merge only on identical normalised tokens" + test_harness: "cargo test -p aprender-review-experiment --lib falsify_tdd_004" + expected_output: "exit 0" + if_fails: "Every one-line diff would collapse into one cluster, or none would dedup." diff --git a/contracts/trace-outcome-join-v1.yaml b/contracts/trace-outcome-join-v1.yaml new file mode 100644 index 0000000000..0b9ccc0c05 --- /dev/null +++ b/contracts/trace-outcome-join-v1.yaml @@ -0,0 +1,62 @@ +metadata: + version: 1.0.0 + kind: schema + description: | + PRM-001 v3 PRM-C10 (was PRA T10), gate G-MAT: "gold only after the 14 d outcome window; + pending is never a negative; 0 immature gold". Done when: "3-day-old gold refused; weekly + matured count". `outcome_join::resolve` (crates/aprender-review-experiment/src/outcome_join.rs) + reads a PR's merge / close / revert / escape events into the v3 lifecycle `outcome` + {pending, merged, reverted_le14d, regression_escape, closed_unmerged}. A revert at most 14 UTC + dates after the merge is reverted_le14d; a later revert or any escape after the merge is + regression_escape. The outcome stays `pending`, with no `outcome_matured_at`, until 14 UTC dates + after the merge (or close). A revert or escape of a PR never merged, or an unreadable date, is + refused. `outcome_join::gold` emits `label_source: outcome` rows for matured outcomes only and + refuses a non-pending row with no maturity date; it emits `label_source: hrq_ruling` rows for + HRQ rulings as soon as they are made (latest per round wins; no ruler or no date is refused). + `outcome_join::matured_between` is the weekly `gold_matured_7d` count. + references: + - "docs/specifications/research-design-pr-agent-qwen-3.5.md" + - "contracts/trace-split-guard-v1.yaml" + - "https://github.com/paiml/aprender/issues/4354" + +equations: + outcome: + formula: "m = first merge; outcome = reverted_le14d if ∃ revert r ≥ m with r − m ≤ 14, regression_escape if ∃ revert r > m + 14 or escape ≥ m, merged otherwise; closed_unmerged if closed and never merged" + maturity: + formula: "matured ⇔ day(now) − day(m or close) ≥ 14; ¬matured ⇒ outcome = pending ∧ outcome_matured_at = None" + gold: + formula: "gold = { outcome rows : matured } ∪ { latest HRQ ruling per (repo#PR, head) }; pending ∉ gold" + weekly: + formula: "gold_matured_7d(from, to) = |{ r : from ≤ outcome_matured_at(r) < to }|" + +falsification_tests: + - id: FALSIFY-TOJ-001 + name: a_3_day_old_outcome_is_refused_as_gold + prediction: "a merge or revert 3 d ago reads pending and yields no gold; 13 UTC dates is still pending; 14 matures with outcome_matured_at = merge + 14" + test_harness: "cargo test -p aprender-review-experiment --lib falsify_toj_001" + expected_output: "exit 0" + if_fails: "An outcome could be trained on before a revert had time to happen." + - id: FALSIFY-TOJ-002 + name: the_outcome_follows_the_revert_window + prediction: "revert on day 14 = reverted_le14d, day 15 = regression_escape, escape = regression_escape, closed = closed_unmerged; the window runs from the merge" + test_harness: "cargo test -p aprender-review-experiment --lib falsify_toj_002" + expected_output: "exit 0" + if_fails: "A late regression could be labelled a clean merge, or a fast revert an escape." + - id: FALSIFY-TOJ-003 + name: pending_is_never_a_label_and_bad_joins_refuse + prediction: "a pending PR yields no gold; a revert or escape with no merge, an unreadable date, and a forged immature row are all errors" + test_harness: "cargo test -p aprender-review-experiment --lib falsify_toj_003" + expected_output: "exit 0" + if_fails: "A pending PR could be read as a negative label." + - id: FALSIFY-TOJ-004 + name: hrq_rulings_are_gold + prediction: "a ruling is gold at once, label_source hrq_ruling (a pool GOLD_SOURCES value); the latest per round wins; no ruler or no date is refused" + test_harness: "cargo test -p aprender-review-experiment --lib falsify_toj_004" + expected_output: "exit 0" + if_fails: "A superseded or anonymous ruling could become a training label." + - id: FALSIFY-TOJ-005 + name: the_weekly_matured_count + prediction: "matured_between counts outcomes whose window closed in [from, to) and no pending PR" + test_harness: "cargo test -p aprender-review-experiment --lib falsify_toj_005" + expected_output: "exit 0" + if_fails: "The weekly receipt could count immature or out-of-week gold." diff --git a/contracts/trace-split-guard-v1.yaml b/contracts/trace-split-guard-v1.yaml new file mode 100644 index 0000000000..8be449aab2 --- /dev/null +++ b/contracts/trace-split-guard-v1.yaml @@ -0,0 +1,55 @@ +metadata: + version: 1.0.0 + kind: schema + description: | + PRM-001 v3 PRM-C9 (was PRA T9), gate G-DUP: "split by repo#PR, time-ordered, >= 14 d + embargo, clusters closed to their earliest member; 0 cross-split clusters". + `split_guard::assign` (crates/aprender-review-experiment/src/split_guard.rs) writes the + `split_guard{group_key, split, dedup_cluster}` a trace row carries. Units sharing a PR or + a trace-dedup-v1 cluster form one component; the component's earliest first-seen day + picks its split against the plan's val_from / test_from; a component first seen within + 14 d before a boundary is embargoed (split None, in no pool); a component holding a + near-dup of a sealed item is `sealed` (in no pool). A plan whose val window is not longer + than the embargo, or an unreadable date, is refused. `split_guard::audit` checks ANY + assignment, not only its own, for PRs and clusters that straddle splits; G-DUP holds + when both lists are empty. Embargoed members are in no pool and never make a straddle. + references: + - "docs/specifications/research-design-pr-agent-qwen-3.5.md" + - "contracts/trace-dedup-v1.yaml" + - "https://github.com/paiml/aprender/issues/4354" + +equations: + component: + formula: "u ~ w ⇔ repo#PR(u) = repo#PR(w) ∨ cluster(u) = cluster(w); components = transitive closure of ~" + split: + formula: "d = min_{u ∈ C} day(at(u)); split(C) = sealed if C ∋ near-dup of a sealed item, else train if d < val−14, None if d < val, val if d < test−14, None if d < test, else test" + plan: + formula: "assign refused ⇔ test − val ≤ 14 ∨ some date unreadable" + audit: + formula: "straddle(K) = { k : |{ split(u) : key(u) = k, split(u) ≠ None }| > 1 }; G-DUP ⇔ straddle(repo#PR) = ∅ ∧ straddle(cluster) = ∅" + +falsification_tests: + - id: FALSIFY-TSG-001 + name: a_pr_never_straddles_splits + prediction: "a PR's later round keeps its first round's split; a straddling assignment is named by audit" + test_harness: "cargo test -p aprender-review-experiment --lib falsify_tsg_001" + expected_output: "exit 0" + if_fails: "Two rounds of one PR could sit in train and test." + - id: FALSIFY-TSG-002 + name: a_near_dup_cluster_closes_to_its_earliest_members_split + prediction: "a PR/cluster chain takes its earliest member's split; a per-row split of the same units shows a cross-split cluster" + test_harness: "cargo test -p aprender-review-experiment --lib falsify_tsg_002" + expected_output: "exit 0" + if_fails: "A re-indented copy of a train diff could be scored as test." + - id: FALSIFY-TSG-003 + name: the_embargo_is_at_least_14_days + prediction: "first seen 14 d or less before a boundary is embargoed; 15 d is not; a short val window and a bad date are refused" + test_harness: "cargo test -p aprender-review-experiment --lib falsify_tsg_003" + expected_output: "exit 0" + if_fails: "Work from the same fortnight could span train and test." + - id: FALSIFY-TSG-004 + name: a_perturbed_sealed_item_is_refused_with_its_whole_pr + prediction: "a renamed, re-indented sealed diff puts its whole component in `sealed`, named by the sealed id" + test_harness: "cargo test -p aprender-review-experiment --lib falsify_tsg_004" + expected_output: "exit 0" + if_fails: "A sealed test item could reach a training pool under a new sha." diff --git a/contracts/workload-profile-v1.yaml b/contracts/workload-profile-v1.yaml new file mode 100644 index 0000000000..2cb912daac --- /dev/null +++ b/contracts/workload-profile-v1.yaml @@ -0,0 +1,58 @@ +metadata: + version: 1.0.0 + kind: schema + description: | + PRM-S2 (PRM-001 v3 "PROMETHEUS" §0.2, §2.4): the review workload profile. It reads + `agent-trace-v1` rows (`timing.tokens{input,output}`) and reports the measured distribution of + input and output tokens per lane. `rex workload ROWS --min-n N` (crates/aprender-review-experiment: + workload.rs, examples/rex.rs) computes it. Rows with `trace_status = capture_failed`, rows that + do not parse, and rows without a positive `tokens.input` or with no `tokens.output` are excluded. + They are counted as excluded and never read as zero tokens. Quantiles are nearest-rank + (stats::percentile, the same rule as every other rex percentile). + §0.2 says a diff of 2k–32k tokens goes in and a verdict plus at most 5 bullets comes out, so review + is prefill-heavy. It is decided per lane, with both bounds taken from §0.2's own text: + in_band ⇔ input p50 ≥ 2000 ∧ input p95 ≤ 32000; prefill_heavy ⇔ output p95 < input p50. + Only lanes with at least min_n rows decide. The claim is confirmed when every deciding lane holds + both, falsified when a deciding lane fails either (the lane is named), and undecided when no lane + reaches min_n. An undecided claim is refused and never read as confirmed. + Strata weights (§2.4: 2k / 8k / 16k / 32k) are taken over distinct work units (`diff_sha256`). + A unit's size is the largest `tokens.input` any lane measured for it, and its stratum is the + smallest bound ≥ that size. A unit over 32k is counted as `over_32k` and left out of the + weights, which sum to 1 over the in-range units. + references: + - "docs/specifications/PRM-001-prometheus.md" + - "contracts/trace-datacard-v1.yaml" + +equations: + quantile: + formula: "p_q(lane, k) = nearest_rank_q({tokens.k(r) : r ∈ rows(lane), measured(r)}), k ∈ {input, output}, q ∈ {0.5, 0.95}" + claim: + formula: "D = {lane : n(lane) ≥ min_n}; claim = None if D = ∅ else Some(∀ lane ∈ D: p50_in ≥ 2000 ∧ p95_in ≤ 32000 ∧ p95_out < p50_in)" + strata: + formula: "size(u) = max_{r ∈ u} tokens.input(r); stratum(u) = min{b ∈ {2000, 8000, 16000, 32000} : size(u) ≤ b}; w_b = |{u : stratum(u) = b}| / |{u : size(u) ≤ 32000}|" + +falsification_tests: + - id: FALSIFY-WLP-001 + name: quantiles_are_nearest_rank_on_measured_rows_only + prediction: "capture_failed, unparseable, input-0 and output-missing rows are excluded and counted, and they do not move any lane's p50/p95; p50/p95 equal nearest rank on the planted sample" + test_harness: "cargo test -p aprender-review-experiment --lib falsify_wlp_001" + expected_output: "exit 0" + if_fails: "A failed capture would read as a 0-token review and drag the profile toward decode-bound." + - id: FALSIFY-WLP-002 + name: claim_is_decided_per_lane_and_refused_when_thin + prediction: "in-band prefill-heavy lanes confirm; a lane whose output p95 ≥ input p50, or whose input p50 < 2000 or p95 > 32000, falsifies and is named; lanes below min_n do not decide; with no lane at min_n the claim is undecided and refused" + test_harness: "cargo test -p aprender-review-experiment --lib falsify_wlp_002" + expected_output: "exit 0" + if_fails: "§0.2 would stay an assertion: speed work would chase prefill on a workload nobody measured." + - id: FALSIFY-WLP-003 + name: strata_weights_are_over_distinct_diffs + prediction: "one diff reviewed by 3 lanes counts once, at its largest lane measurement; bounds are inclusive (2000 → 2k, 2001 → 8k); a unit over 32k is over_32k and unweighted; weights sum to 1" + test_harness: "cargo test -p aprender-review-experiment --lib falsify_wlp_003" + expected_output: "exit 0" + if_fails: "Replay strata would be weighted by lane count rather than by the diffs the lane actually sees." + - id: FALSIFY-WLP-004 + name: receipt_block_matches_the_v3_template + prediction: "the rendered block is `workload: {input_tokens_p50_p95: {lane: [p50, p95]}, output_tokens_p50_p95: {…}}` plus claim, strata and excluded; an undecided claim renders null, not true" + test_harness: "cargo test -p aprender-review-experiment --lib falsify_wlp_004" + expected_output: "exit 0" + if_fails: "The §7 receipt reader would miss the workload block or read an undecided claim as confirmed." diff --git a/crates/apr-cli/src/commands/chat_generate_safetensors.rs b/crates/apr-cli/src/commands/chat_generate_safetensors.rs index 78d20d11b1..b327de1d4e 100644 --- a/crates/apr-cli/src/commands/chat_generate_safetensors.rs +++ b/crates/apr-cli/src/commands/chat_generate_safetensors.rs @@ -72,6 +72,7 @@ impl ChatSession { seed: realizar::apr_transformer::DEFAULT_SEED, repeat_penalty: 1.0, repeat_last_n: 0, + logprobs_top_k: 0, // apr_transformer::generation::is_eos_token (GH-330) treated // token 0 as EOS unconditionally; Session has no such builtin, // so it is carried here as an explicit stop token to keep this diff --git a/crates/apr-cli/src/commands/inference_output.rs b/crates/apr-cli/src/commands/inference_output.rs index bec1955d6d..9dba0ff345 100644 --- a/crates/apr-cli/src/commands/inference_output.rs +++ b/crates/apr-cli/src/commands/inference_output.rs @@ -188,6 +188,8 @@ struct InferenceOutput { token_texts: Option>, /// Prompt and completion counts plus the finish reason (#3718). usage: RunUsage, + /// #4026: see `RunResult::logprobs`. + logprobs: Option, } /// Execute inference on model @@ -222,6 +224,7 @@ fn execute_inference( input_path.map_or_else(|| "stdin".to_string(), |p| p.display().to_string()); Ok(InferenceOutput { + logprobs: None, text: format!( "[Inference requires --features inference]\nModel: {}\nInput: {}\nFormat: {}\nGPU: {}", model_path.display(), @@ -335,7 +338,8 @@ pub(crate) fn realizar_config( .with_repeat_penalty(options.repeat_penalty) .with_repeat_last_n(options.repeat_last_n) .with_force_chat_template(options.chat_template) - .with_thinking(options.thinking); + .with_thinking(options.thinking) + .with_logprobs_top_k(options.logprobs_top_k); if options.no_gpu { config = config.without_gpu(); @@ -377,7 +381,14 @@ fn execute_with_realizar( // #3718: the report carries what only the decode path knows (finish reason, // context window); the prompt count is `input_token_count`, taken after the // chat template, so it is the number the model actually read. + let _ = realizar::session::take_last_turn_steps(); // #4026: never an earlier run's steps let (result, report) = run_inference_report(&config).map_err(inference_error)?; + let logprobs = run_logprobs( + options.logprobs_top_k, + &result.format, + &result.tokens[..result.input_token_count.min(result.tokens.len())], + realizar::session::take_last_turn_steps(), + )?; // Report performance if benchmarking if options.benchmark { @@ -427,9 +438,37 @@ fn execute_with_realizar( .generation_ms .map(|g| (result.inference_ms - g).max(0.0).round() as u64), }, + logprobs, }) } +/// #4026: the `logprobs` object of `apr run --json`, or a refusal by name when +/// `--logprobs K` was asked and the path that served the run recorded nothing. +/// An empty list is never reported for a path that could not see the logits. +#[cfg(feature = "inference")] +fn run_logprobs( + top_k: usize, + format: &str, + prompt_token_ids: &[u32], + steps: Option>, +) -> Result> { + if top_k == 0 { + return Ok(None); + } + let steps = steps.ok_or_else(|| { + CliError::InvalidInput(format!( + "--logprobs {top_k}: the {format} path that served this run does not record \ + per-step logprobs (only the GGUF session engine does, on CPU and CUDA) — \ + refused rather than reported empty (#4026)" + )) + })?; + Ok(Some(serde_json::json!({ + "top_k": top_k, + "prompt_token_ids": prompt_token_ids, + "steps": steps, + }))) +} + #[cfg(all(test, feature = "inference"))] mod tests_2403 { use super::inference_error; @@ -456,3 +495,45 @@ mod tests_2403 { ); } } + +#[cfg(all(test, feature = "inference"))] +mod tests_4026 { + use super::run_logprobs; + use realizar::gguf::{StepLogprobs, TopLogprob}; + + #[test] + fn logprobs_off_reports_nothing_even_without_steps() { + assert_eq!(run_logprobs(0, "GGUF", &[1], None).expect("ok"), None); + } + + #[test] + fn logprobs_on_a_path_that_records_nothing_is_refused_by_name() { + let err = run_logprobs(3, "SafeTensors", &[1], None).expect_err("refused"); + let msg = err.to_string(); + assert!(msg.contains("SafeTensors"), "{msg}"); + assert!(msg.contains("--logprobs 3"), "{msg}"); + } + + #[test] + fn logprobs_report_the_prompt_ids_and_every_step() { + let steps = vec![StepLogprobs { + step: 0, + chosen: 17, + top: vec![TopLogprob { + token_id: 17, + logit: 4.0, + logprob: -0.25, + }], + chosen_logprob: -0.25, + logsumexp_full: 4.25, + }]; + let v = run_logprobs(1, "GGUF", &[5, 6], Some(steps)) + .expect("ok") + .expect("some"); + assert_eq!(v["top_k"], 1); + assert_eq!(v["prompt_token_ids"], serde_json::json!([5, 6])); + assert_eq!(v["steps"][0]["chosen"], 17); + assert_eq!(v["steps"][0]["top"][0]["token_id"], 17); + assert_eq!(v["steps"][0]["top"][0]["logprob"], -0.25); + } +} diff --git a/crates/apr-cli/src/commands/run.rs b/crates/apr-cli/src/commands/run.rs index da2899b05f..b31f299547 100644 --- a/crates/apr-cli/src/commands/run.rs +++ b/crates/apr-cli/src/commands/run.rs @@ -175,6 +175,8 @@ pub(crate) struct RunOptions { /// #3723: `--thinking on|off`. `None` renders the production default; realizar applies it /// to the rendered prompt and refuses `on` for a template with no thinking mode. pub thinking: Option, + /// #4026: `--logprobs K`. 0 records nothing. + pub logprobs_top_k: usize, /// `--stream`: emit one NDJSON event per generated token. /// /// Known here (not only at the print site) because streaming is the one @@ -211,6 +213,7 @@ impl Default for RunOptions { split_prompt: false, chat_template: false, thinking: None, + logprobs_top_k: 0, stream: false, } } @@ -273,6 +276,8 @@ pub(crate) struct RunResult { pub token_texts: Option>, /// Prompt and completion counts plus the finish reason (#3718). pub usage: RunUsage, + /// #4026: `{top_k, prompt_token_ids, steps}` when `--logprobs K` > 0, else `None`. + pub logprobs: Option, } /// Resolve a user-supplied model argument into a [`ModelSource`]. @@ -361,6 +366,7 @@ pub(crate) fn run_model(source: &str, options: &RunOptions) -> Result generated_tokens: output.generated_tokens, token_texts: output.token_texts, usage: output.usage, + logprobs: output.logprobs, }) } diff --git a/crates/apr-cli/src/commands/run_entry.rs b/crates/apr-cli/src/commands/run_entry.rs index 9b5ca2cb41..004f5dcaf2 100644 --- a/crates/apr-cli/src/commands/run_entry.rs +++ b/crates/apr-cli/src/commands/run_entry.rs @@ -46,6 +46,8 @@ pub(crate) fn run( chat_template: bool, // #3723: `--thinking on|off` (None: the production default), applied in realizar. thinking: Option, + // #4026: `--logprobs K`: the K most likely tokens at every generated step. + logprobs_top_k: usize, ) -> Result<()> { // GH-516: Warn on --language/--task since whisper integration is not yet wired up if language.is_some() { @@ -118,6 +120,7 @@ pub(crate) fn run( split_prompt, chat_template, thinking, + logprobs_top_k, stream, }; @@ -634,6 +637,10 @@ fn build_final_json( // is the whole window (load, upload, F2, generation), kept for compatibility. "generation_ms": result.usage.generation_ms, "setup_ms": result.usage.setup_ms, + // #4026: present only when `--logprobs K` asked for it: the prompt ids the + // model read and, per generated step, the K most likely tokens before any + // penalty or sampling. A path that cannot record them refused the run. + "logprobs": result.logprobs, // #3602: `used_gpu: false` alone collapses two different outcomes — "no // accelerator was asked for" and "one was asked for, attempted, and // REFUSED at runtime". A consumer cannot tell a CPU run from a rejected diff --git a/crates/apr-cli/src/commands/run_tests_accel_reconcile.rs b/crates/apr-cli/src/commands/run_tests_accel_reconcile.rs index b496ab747a..dd99b3f85f 100644 --- a/crates/apr-cli/src/commands/run_tests_accel_reconcile.rs +++ b/crates/apr-cli/src/commands/run_tests_accel_reconcile.rs @@ -54,6 +54,7 @@ #[test] fn a_forced_accelerator_that_ran_on_cpu_is_refused() { let result = RunResult { + logprobs: None, text: "Hello".to_string(), duration_secs: 33.646, cached: true, @@ -142,6 +143,7 @@ fn a_successful_gpu_run_is_not_labelled_a_fallback() { fn gpu_result(used_gpu: Option) -> RunResult { RunResult { + logprobs: None, text: "Hello".to_string(), duration_secs: 1.0, cached: true, diff --git a/crates/apr-cli/src/commands/run_tests_chrome_trace.rs b/crates/apr-cli/src/commands/run_tests_chrome_trace.rs index 1c03f69512..d6cd2d92de 100644 --- a/crates/apr-cli/src/commands/run_tests_chrome_trace.rs +++ b/crates/apr-cli/src/commands/run_tests_chrome_trace.rs @@ -16,6 +16,7 @@ #[test] fn test_chrome_trace_event_categories() { let result = RunResult { + logprobs: None, text: "test".to_string(), duration_secs: 2.0, cached: false, @@ -48,6 +49,7 @@ #[test] fn test_chrome_trace_zero_tokens() { let result = RunResult { + logprobs: None, text: String::new(), duration_secs: 0.5, cached: true, @@ -69,6 +71,7 @@ #[test] fn test_chrome_trace_no_tokens_generated_field() { let result = RunResult { + logprobs: None, text: "output".to_string(), duration_secs: 1.0, cached: false, @@ -90,6 +93,7 @@ #[test] fn test_chrome_trace_metadata_source() { let result = RunResult { + logprobs: None, text: "hi".to_string(), duration_secs: 0.1, cached: false, @@ -111,6 +115,7 @@ #[test] fn test_chrome_trace_event_format() { let result = RunResult { + logprobs: None, text: "test".to_string(), duration_secs: 1.0, cached: false, @@ -142,6 +147,7 @@ #[test] fn test_chrome_trace_token_count() { let result = RunResult { + logprobs: None, text: "test".to_string(), duration_secs: 5.0, cached: false, @@ -163,6 +169,7 @@ #[test] fn test_chrome_trace_display_time_unit() { let result = RunResult { + logprobs: None, text: "t".to_string(), duration_secs: 1.0, cached: false, @@ -184,6 +191,7 @@ // This test writes to CWD. Use a unique tempdir and set_current_dir. // May conflict with parallel tests, but the function itself should not panic. let result = RunResult { + logprobs: None, text: "Hello world".to_string(), duration_secs: 1.0, cached: false, @@ -218,6 +226,7 @@ let target = dir.join("requested.json"); let result = RunResult { + logprobs: None, text: "Hello world".to_string(), duration_secs: 1.0, cached: false, @@ -249,6 +258,7 @@ #[test] fn test_print_benchmark_results_text() { let result = RunResult { + logprobs: None, text: "test output".to_string(), duration_secs: 2.0, cached: false, @@ -266,6 +276,7 @@ #[test] fn test_print_benchmark_results_json() { let result = RunResult { + logprobs: None, text: "test".to_string(), duration_secs: 1.0, cached: false, @@ -283,6 +294,7 @@ #[test] fn test_print_benchmark_zero_duration() { let result = RunResult { + logprobs: None, text: "".to_string(), duration_secs: 0.0, cached: false, diff --git a/crates/apr-cli/src/commands/run_tests_inference_output.rs b/crates/apr-cli/src/commands/run_tests_inference_output.rs index 824885b810..0f2f8ade08 100644 --- a/crates/apr-cli/src/commands/run_tests_inference_output.rs +++ b/crates/apr-cli/src/commands/run_tests_inference_output.rs @@ -8,6 +8,7 @@ #[test] fn inference_output_fields() { let output = InferenceOutput { + logprobs: None, text: "hello".to_string(), tokens_generated: Some(5), inference_ms: Some(10.0), @@ -27,6 +28,7 @@ #[test] fn inference_output_no_metrics() { let output = InferenceOutput { + logprobs: None, text: "result".to_string(), tokens_generated: None, inference_ms: None, diff --git a/crates/apr-cli/src/commands/run_tests_layer_trace.rs b/crates/apr-cli/src/commands/run_tests_layer_trace.rs index adc85c3ee9..48391aaee4 100644 --- a/crates/apr-cli/src/commands/run_tests_layer_trace.rs +++ b/crates/apr-cli/src/commands/run_tests_layer_trace.rs @@ -12,6 +12,7 @@ fn layer_trace_result(duration_secs: f64, tokens: usize) -> RunResult { RunResult { + logprobs: None, text: "hi".to_string(), duration_secs, cached: true, diff --git a/crates/apr-cli/src/commands/run_tests_model_source.rs b/crates/apr-cli/src/commands/run_tests_model_source.rs index 3e5bd41760..d35e3a5230 100644 --- a/crates/apr-cli/src/commands/run_tests_model_source.rs +++ b/crates/apr-cli/src/commands/run_tests_model_source.rs @@ -31,6 +31,7 @@ #[test] fn test_run_result_debug() { let result = RunResult { + logprobs: None, text: "Hello".to_string(), duration_secs: 0.1, cached: true, @@ -141,6 +142,7 @@ #[test] fn test_run_result_clone() { let result = RunResult { + logprobs: None, text: "Test".to_string(), duration_secs: 1.0, cached: false, diff --git a/crates/apr-cli/src/commands/run_tests_parse_token_ids.rs b/crates/apr-cli/src/commands/run_tests_parse_token_ids.rs index 778dd02cbb..6210a31830 100644 --- a/crates/apr-cli/src/commands/run_tests_parse_token_ids.rs +++ b/crates/apr-cli/src/commands/run_tests_parse_token_ids.rs @@ -60,6 +60,7 @@ #[test] fn run_result_tokens_generated_none_vs_zero() { let result_none = RunResult { + logprobs: None, text: String::new(), duration_secs: 0.0, cached: false, @@ -72,6 +73,7 @@ usage: Default::default(), }; let result_zero = RunResult { + logprobs: None, text: String::new(), duration_secs: 0.0, cached: false, @@ -94,6 +96,7 @@ #[test] fn run_result_field_independence() { let result = RunResult { + logprobs: None, text: "output".to_string(), duration_secs: 1.234, cached: true, diff --git a/crates/apr-cli/src/commands/run_tests_stream_output.rs b/crates/apr-cli/src/commands/run_tests_stream_output.rs index 0f61f43a66..f53320b7ba 100644 --- a/crates/apr-cli/src/commands/run_tests_stream_output.rs +++ b/crates/apr-cli/src/commands/run_tests_stream_output.rs @@ -4,6 +4,7 @@ #[test] fn stream_output_emits_n_plus_one_json_lines() { let result = RunResult { + logprobs: None, text: "Hello world".to_string(), duration_secs: 0.25, cached: false, @@ -74,6 +75,7 @@ fn stream_output_emits_n_plus_one_json_lines() { #[test] fn stream_token_events_carry_their_own_decoded_text() { let result = RunResult { + logprobs: None, text: "I'm here to help".to_string(), duration_secs: 1.0, cached: true, @@ -121,6 +123,7 @@ fn stream_token_events_carry_their_own_decoded_text() { #[test] fn stream_token_events_degrade_to_empty_text_without_a_tokenizer() { let result = RunResult { + logprobs: None, text: "abc".to_string(), duration_secs: 1.0, cached: true, @@ -148,6 +151,7 @@ fn stream_token_events_degrade_to_empty_text_without_a_tokenizer() { #[test] fn stream_output_no_tokens_emits_only_final() { let result = RunResult { + logprobs: None, text: String::new(), duration_secs: 0.0, cached: true, @@ -175,6 +179,7 @@ fn stream_output_no_tokens_emits_only_final() { #[test] fn stream_output_none_tokens_emits_only_final() { let result = RunResult { + logprobs: None, text: String::new(), duration_secs: 0.0, cached: false, @@ -201,6 +206,7 @@ fn stream_output_none_tokens_emits_only_final() { #[test] fn build_final_json_matches_legacy_json_shape() { let result = RunResult { + logprobs: None, text: "abc".to_string(), duration_secs: 1.0, cached: true, diff --git a/crates/apr-cli/src/commands/run_tests_usage_3718.rs b/crates/apr-cli/src/commands/run_tests_usage_3718.rs index 7a18aa1366..cf374137f9 100644 --- a/crates/apr-cli/src/commands/run_tests_usage_3718.rs +++ b/crates/apr-cli/src/commands/run_tests_usage_3718.rs @@ -2,6 +2,7 @@ fn result_with(usage: RunUsage) -> RunResult { RunResult { + logprobs: None, text: "4".to_string(), duration_secs: 1.0, cached: true, diff --git a/crates/apr-cli/src/commands/serve/safetensors.rs b/crates/apr-cli/src/commands/serve/safetensors.rs index e161493507..4070f05662 100644 --- a/crates/apr-cli/src/commands/serve/safetensors.rs +++ b/crates/apr-cli/src/commands/serve/safetensors.rs @@ -403,6 +403,7 @@ fn st_cpu_generate( seed: realizar::apr_transformer::DEFAULT_SEED, repeat_penalty: 1.0, repeat_last_n: 0, + logprobs_top_k: 0, // apr_transformer::generation::is_eos_token (GH-330) stopped on token 0 // unconditionally; Session has no such builtin, so it is an explicit // stop token here to keep the handlers' stopping behavior identical. diff --git a/crates/apr-cli/src/commands_enum.rs b/crates/apr-cli/src/commands_enum.rs index 2eadea8471..c772f043f3 100644 --- a/crates/apr-cli/src/commands_enum.rs +++ b/crates/apr-cli/src/commands_enum.rs @@ -224,6 +224,12 @@ pub enum Commands { /// F-CLIPARITY-01 / PMAT-384 / paiml/aprender#571 #[arg(long, default_value = "64")] repeat_last_n: usize, + /// Record the K most likely next tokens (token id, logit, logprob) at every + /// generated step, plus the prompt ids actually encoded, in `--json` output + /// (#4026). 0 (the default) records nothing. Refused by name on a path that + /// cannot record them (`--trace`, APR/SafeTensors, wgpu, qwen3_moe). + #[arg(long, value_name = "K", default_value_t = 0)] + logprobs: usize, /// Process prompt tokens one-by-one instead of batched prefill. /// Useful for debugging prefill correctness (comparing per-token attention). /// F-CLIPARITY-01 / PMAT-385 / paiml/aprender#572 diff --git a/crates/apr-cli/src/dispatch.rs b/crates/apr-cli/src/dispatch.rs index 22edb34697..3290fb83f9 100644 --- a/crates/apr-cli/src/dispatch.rs +++ b/crates/apr-cli/src/dispatch.rs @@ -253,6 +253,7 @@ fn dispatch_runtime_commands(cli: &Cli) -> Option> { seed, repeat_penalty, repeat_last_n, + logprobs, chat, split_prompt, batch_jsonl, @@ -339,6 +340,7 @@ fn dispatch_runtime_commands(cli: &Cli) -> Option> { *repeat_last_n, *split_prompt, thinking.mode(), + *logprobs, ) } diff --git a/crates/apr-cli/src/dispatch_run.rs b/crates/apr-cli/src/dispatch_run.rs index 92fc146005..5163147d1e 100644 --- a/crates/apr-cli/src/dispatch_run.rs +++ b/crates/apr-cli/src/dispatch_run.rs @@ -36,6 +36,8 @@ fn dispatch_run( split_prompt: bool, // #3723: `--thinking on|off`, None when absent. thinking: Option, + // #4026: `--logprobs K`, 0 when absent. + logprobs_top_k: usize, ) -> Result<(), CliError> { let effective_trace = trace || trace_payload; let effective_trace_level = if trace_payload { @@ -74,6 +76,7 @@ fn dispatch_run( split_prompt, chat_template, thinking, + logprobs_top_k, ) } diff --git a/crates/apr-cli/src/lib_parse_rosetta.rs b/crates/apr-cli/src/lib_parse_rosetta.rs index d96b2f2fbb..e88208042b 100644 --- a/crates/apr-cli/src/lib_parse_rosetta.rs +++ b/crates/apr-cli/src/lib_parse_rosetta.rs @@ -320,6 +320,7 @@ seed: 299_792_458, repeat_penalty: 1.0, repeat_last_n: 64, + logprobs: 0, split_prompt: false, batch_jsonl: None, verbose: false, diff --git a/crates/aprender-orchestrate/src/agent/driver/realizar.rs b/crates/aprender-orchestrate/src/agent/driver/realizar.rs index 0400bc8045..ce98b993e8 100644 --- a/crates/aprender-orchestrate/src/agent/driver/realizar.rs +++ b/crates/aprender-orchestrate/src/agent/driver/realizar.rs @@ -97,6 +97,7 @@ impl LlmDriver for RealizarDriver { use_mock_backend: false, force_chat_template: false, thinking: None, + logprobs_top_k: 0, stop_tokens: vec![], }; diff --git a/crates/aprender-review-experiment/Cargo.toml b/crates/aprender-review-experiment/Cargo.toml new file mode 100644 index 0000000000..be2ca3645a --- /dev/null +++ b/crates/aprender-review-experiment/Cargo.toml @@ -0,0 +1,25 @@ +[package] +name = "aprender-review-experiment" +description = "REX-001 review-lane experiment: pre-registration lock, corpus, receipts and the frozen analysis code (Wilson, McNemar exact, seeded bootstrap, Holm)" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license.workspace = true +authors.workspace = true +repository.workspace = true +# The pre-registration check hashes repo files (the spec, the prompt, the +# corpus manifest) via include_str!, so the crate is meaningful only in-tree. +publish = false + +[dependencies] +half = { workspace = true } +serde = { workspace = true } +serde_json = { workspace = true } +regex = { workspace = true } +toml = { workspace = true } +sha2 = { workspace = true } +ureq = { workspace = true } +zstd = { workspace = true } + +[lints] +workspace = true diff --git a/crates/aprender-review-experiment/README.md b/crates/aprender-review-experiment/README.md new file mode 100644 index 0000000000..7157071cb5 --- /dev/null +++ b/crates/aprender-review-experiment/README.md @@ -0,0 +1,17 @@ +# aprender-review-experiment + +PROMETHEUS (PRM-001, was REX-001) — the review-lane experiment for the Qwen 3.5 4B +review lane on a released `apr`, across fleet device cells. Part of the +[aprender monorepo](https://github.com/paiml/aprender); epic paiml/aprender#4354. + +- `prereg` — the REX-00 pre-registration lock (`contracts/rex-prereg-v1.yaml`). + `docs/audits/rex-001/prereg.lock` freezes spec §2–§5, the analysis code, the + analysis plan and prompt v1; every receipt carries the prereg sha. +- `stats` — the frozen analysis code: Wilson, McNemar exact, seeded bootstrap + (SplitMix64), Holm, H6 interference. + +```bash +cargo run -p aprender-review-experiment --example rex -- prereg-check +``` + +Not published (`publish = false`): it hashes repo files via `include_str!`. diff --git a/crates/aprender-review-experiment/data/gitleaks/LICENSE b/crates/aprender-review-experiment/data/gitleaks/LICENSE new file mode 100644 index 0000000000..3c270b36c5 --- /dev/null +++ b/crates/aprender-review-experiment/data/gitleaks/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2019 Zachary Rice + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/crates/aprender-review-experiment/data/gitleaks/gitleaks-v8.30.1.toml b/crates/aprender-review-experiment/data/gitleaks/gitleaks-v8.30.1.toml new file mode 100644 index 0000000000..256f64790e --- /dev/null +++ b/crates/aprender-review-experiment/data/gitleaks/gitleaks-v8.30.1.toml @@ -0,0 +1,3209 @@ +# This file has been auto-generated. Do not edit manually. +# If you would like to contribute new rules, please use +# cmd/generate/config/main.go and follow the contributing guidelines +# at https://github.com/gitleaks/gitleaks/blob/master/CONTRIBUTING.md +# +# How the hell does secret scanning work? Read this: +# https://lookingatcomputer.substack.com/p/regex-is-almost-all-you-need +# +# This is the default gitleaks configuration file. +# Rules and allowlists are defined within this file. +# Rules instruct gitleaks on what should be considered a secret. +# Allowlists instruct gitleaks on what is allowed, i.e. not a secret. + +title = "gitleaks config" + +# minVersion indicates the minimum Gitleaks version required to use this config. +# If the running version is older, a warning will be logged and not all +# config-enabled features are guaranteed to work. +minVersion = "v8.25.0" + +# TODO: change to [[allowlists]] +[allowlist] +description = "global allow lists" +paths = [ + '''gitleaks\.toml''', + '''(?i)\.(?:bmp|gif|jpe?g|png|svg|tiff?)$''', + '''(?i)\.(?:eot|[ot]tf|woff2?)$''', + '''(?i)\.(?:docx?|xlsx?|pdf|bin|socket|vsidx|v2|suo|wsuo|.dll|pdb|exe|gltf)$''', + '''go\.(?:mod|sum|work(?:\.sum)?)$''', + '''(?:^|/)vendor/modules\.txt$''', + '''(?:^|/)vendor/(?:github\.com|golang\.org/x|google\.golang\.org|gopkg\.in|istio\.io|k8s\.io|sigs\.k8s\.io)(?:/.*)?$''', + '''(?:^|/)gradlew(?:\.bat)?$''', + '''(?:^|/)gradle\.lockfile$''', + '''(?:^|/)mvnw(?:\.cmd)?$''', + '''(?:^|/)\.mvn/wrapper/MavenWrapperDownloader\.java$''', + '''(?:^|/)node_modules(?:/.*)?$''', + '''(?:^|/)(?:deno\.lock|npm-shrinkwrap\.json|package-lock\.json|pnpm-lock\.yaml|yarn\.lock)$''', + '''(?:^|/)bower_components(?:/.*)?$''', + '''(?:^|/)(?:angular|bootstrap|jquery(?:-?ui)?|plotly|swagger-?ui)[a-zA-Z0-9.-]*(?:\.min)?\.js(?:\.map)?$''', + '''(?:^|/)javascript\.json$''', + '''(?:^|/)(?:Pipfile|poetry)\.lock$''', + '''(?i)(?:^|/)(?:v?env|virtualenv)/lib(?:64)?(?:/.*)?$''', + '''(?i)(?:^|/)(?:lib(?:64)?/python[23](?:\.\d{1,2})+|python/[23](?:\.\d{1,2})+/lib(?:64)?)(?:/.*)?$''', + '''(?i)(?:^|/)[a-z0-9_.]+-[0-9.]+\.dist-info(?:/.+)?$''', + '''(?:^|/)vendor/(?:bundle|ruby)(?:/.*?)?$''', + '''\.gem$''', + '''verification-metadata\.xml''', + '''Database.refactorlog''', + '''(?:^|/)\.git$''', +] +regexes = [ + '''(?i)^true|false|null$''', + '''^(?i:a+|b+|c+|d+|e+|f+|g+|h+|i+|j+|k+|l+|m+|n+|o+|p+|q+|r+|s+|t+|u+|v+|w+|x+|y+|z+|\*+|\.+)$''', + '''^\$(?:\d+|{\d+})$''', + '''^\$(?:[A-Z_]+|[a-z_]+)$''', + '''^\${(?:[A-Z_]+|[a-z_]+)}$''', + '''^\{\{[ \t]*[\w ().|]+[ \t]*}}$''', + '''^\$\{\{[ \t]*(?:(?:env|github|secrets|vars)(?:\.[A-Za-z]\w+)+[\w "'&./=|]*)[ \t]*}}$''', + '''^%(?:[A-Z_]+|[a-z_]+)%$''', + '''^%[+\-# 0]?[bcdeEfFgGoOpqstTUvxX]$''', + '''^\{\d{0,2}}$''', + '''^@(?:[A-Z_]+|[a-z_]+)@$''', + '''^/Users/(?i)[a-z0-9]+/[\w .-/]+$''', + '''^/(?:bin|etc|home|opt|tmp|usr|var)/[\w ./-]+$''', +] +stopwords = [ + "014df517-39d1-4453-b7b3-9930c563627c", + "abcdefghijklmnopqrstuvwxyz", +] + +[[rules]] +id = "1password-secret-key" +description = "Uncovered a possible 1Password secret key, potentially compromising access to secrets in vaults." +regex = '''\bA3-[A-Z0-9]{6}-(?:(?:[A-Z0-9]{11})|(?:[A-Z0-9]{6}-[A-Z0-9]{5}))-[A-Z0-9]{5}-[A-Z0-9]{5}-[A-Z0-9]{5}\b''' +entropy = 3.8 +keywords = ["a3-"] + +[[rules]] +id = "1password-service-account-token" +description = "Uncovered a possible 1Password service account token, potentially compromising access to secrets in vaults." +regex = '''ops_eyJ[a-zA-Z0-9+/]{250,}={0,3}''' +entropy = 4 +keywords = ["ops_"] + +[[rules]] +id = "adafruit-api-key" +description = "Identified a potential Adafruit API Key, which could lead to unauthorized access to Adafruit services and sensitive data exposure." +regex = '''(?i)[\w.-]{0,50}?(?:adafruit)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9_-]{32})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["adafruit"] + +[[rules]] +id = "adobe-client-id" +description = "Detected a pattern that resembles an Adobe OAuth Web Client ID, posing a risk of compromised Adobe integrations and data breaches." +regex = '''(?i)[\w.-]{0,50}?(?:adobe)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-f0-9]{32})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 2 +keywords = ["adobe"] + +[[rules]] +id = "adobe-client-secret" +description = "Discovered a potential Adobe Client Secret, which, if exposed, could allow unauthorized Adobe service access and data manipulation." +regex = '''\b(p8e-(?i)[a-z0-9]{32})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 2 +keywords = ["p8e-"] + +[[rules]] +id = "age-secret-key" +description = "Discovered a potential Age encryption tool secret key, risking data decryption and unauthorized access to sensitive information." +regex = '''AGE-SECRET-KEY-1[QPZRY9X8GF2TVDW0S3JN54KHCE6MUA7L]{58}''' +keywords = ["age-secret-key-1"] + +[[rules]] +id = "airtable-api-key" +description = "Uncovered a possible Airtable API Key, potentially compromising database access and leading to data leakage or alteration." +regex = '''(?i)[\w.-]{0,50}?(?:airtable)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9]{17})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["airtable"] + +[[rules]] +id = "airtable-personnal-access-token" +description = "Uncovered a possible Airtable Personal AccessToken, potentially compromising database access and leading to data leakage or alteration." +regex = '''\b(pat[[:alnum:]]{14}\.[a-f0-9]{64})\b''' +keywords = ["airtable"] + +[[rules]] +id = "algolia-api-key" +description = "Identified an Algolia API Key, which could result in unauthorized search operations and data exposure on Algolia-managed platforms." +regex = '''(?i)[\w.-]{0,50}?(?:algolia)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9]{32})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["algolia"] + +[[rules]] +id = "alibaba-access-key-id" +description = "Detected an Alibaba Cloud AccessKey ID, posing a risk of unauthorized cloud resource access and potential data compromise." +regex = '''\b(LTAI(?i)[a-z0-9]{20})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 2 +keywords = ["ltai"] + +[[rules]] +id = "alibaba-secret-key" +description = "Discovered a potential Alibaba Cloud Secret Key, potentially allowing unauthorized operations and data access within Alibaba Cloud." +regex = '''(?i)[\w.-]{0,50}?(?:alibaba)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9]{30})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 2 +keywords = ["alibaba"] + +[[rules]] +id = "anthropic-admin-api-key" +description = "Detected an Anthropic Admin API Key, risking unauthorized access to administrative functions and sensitive AI model configurations." +regex = '''\b(sk-ant-admin01-[a-zA-Z0-9_\-]{93}AA)(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["sk-ant-admin01"] + +[[rules]] +id = "anthropic-api-key" +description = "Identified an Anthropic API Key, which may compromise AI assistant integrations and expose sensitive data to unauthorized access." +regex = '''\b(sk-ant-api03-[a-zA-Z0-9_\-]{93}AA)(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["sk-ant-api03"] + +[[rules]] +id = "artifactory-api-key" +description = "Detected an Artifactory api key, posing a risk unauthorized access to the central repository." +regex = '''\bAKCp[A-Za-z0-9]{69}\b''' +entropy = 4.5 +keywords = ["akcp"] + +[[rules]] +id = "artifactory-reference-token" +description = "Detected an Artifactory reference token, posing a risk of impersonation and unauthorized access to the central repository." +regex = '''\bcmVmd[A-Za-z0-9]{59}\b''' +entropy = 4.5 +keywords = ["cmvmd"] + +[[rules]] +id = "asana-client-id" +description = "Discovered a potential Asana Client ID, risking unauthorized access to Asana projects and sensitive task information." +regex = '''(?i)[\w.-]{0,50}?(?:asana)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([0-9]{16})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["asana"] + +[[rules]] +id = "asana-client-secret" +description = "Identified an Asana Client Secret, which could lead to compromised project management integrity and unauthorized access." +regex = '''(?i)[\w.-]{0,50}?(?:asana)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9]{32})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["asana"] + +[[rules]] +id = "atlassian-api-token" +description = "Detected an Atlassian API token, posing a threat to project management and collaboration tool security and data confidentiality." +regex = '''(?i)[\w.-]{0,50}?(?:(?-i:ATLASSIAN|[Aa]tlassian)|(?-i:CONFLUENCE|[Cc]onfluence)|(?-i:JIRA|[Jj]ira))(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9]{20}[a-f0-9]{4})(?:[\x60'"\s;]|\\[nr]|$)|\b(ATATT3[A-Za-z0-9_\-=]{186})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 3.5 +keywords = [ + "atlassian", + "confluence", + "jira", + "atatt3", +] + +[[rules]] +id = "authress-service-client-access-key" +description = "Uncovered a possible Authress Service Client Access Key, which may compromise access control services and sensitive data." +regex = '''\b((?:sc|ext|scauth|authress)_(?i)[a-z0-9]{5,30}\.[a-z0-9]{4,6}\.(?-i:acc)[_-][a-z0-9-]{10,32}\.[a-z0-9+/_=-]{30,120})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 2 +keywords = [ + "sc_", + "ext_", + "scauth_", + "authress_", +] + +[[rules]] +id = "aws-access-token" +description = "Identified a pattern that may indicate AWS credentials, risking unauthorized cloud resource access and data breaches on AWS platforms." +regex = '''\b((?:A3T[A-Z0-9]|AKIA|ASIA|ABIA|ACCA)[A-Z2-7]{16})\b''' +entropy = 3 +keywords = [ + "a3t", + "akia", + "asia", + "abia", + "acca", +] +[[rules.allowlists]] +regexes = [ + '''.+EXAMPLE$''', +] + +[[rules]] +id = "aws-amazon-bedrock-api-key-long-lived" +description = "Identified a pattern that may indicate long-lived Amazon Bedrock API keys, risking unauthorized Amazon Bedrock usage" +regex = '''\b(ABSK[A-Za-z0-9+/]{109,269}={0,2})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 3 +keywords = ["absk"] + +[[rules]] +id = "aws-amazon-bedrock-api-key-short-lived" +description = "Identified a pattern that may indicate short-lived Amazon Bedrock API keys, risking unauthorized Amazon Bedrock usage" +regex = '''bedrock-api-key-YmVkcm9jay5hbWF6b25hd3MuY29t''' +entropy = 3 +keywords = ["bedrock-api-key-"] + +[[rules]] +id = "azure-ad-client-secret" +description = "Azure AD Client Secret" +regex = '''(?:^|[\\'"\x60\s>=:(,)])([a-zA-Z0-9_~.]{3}\dQ~[a-zA-Z0-9_~.-]{31,34})(?:$|[\\'"\x60\s<),])''' +entropy = 3 +keywords = ["q~"] + +[[rules]] +id = "beamer-api-token" +description = "Detected a Beamer API token, potentially compromising content management and exposing sensitive notifications and updates." +regex = '''(?i)[\w.-]{0,50}?(?:beamer)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}(b_[a-z0-9=_\-]{44})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["beamer"] + +[[rules]] +id = "bitbucket-client-id" +description = "Discovered a potential Bitbucket Client ID, risking unauthorized repository access and potential codebase exposure." +regex = '''(?i)[\w.-]{0,50}?(?:bitbucket)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9]{32})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["bitbucket"] + +[[rules]] +id = "bitbucket-client-secret" +description = "Discovered a potential Bitbucket Client Secret, posing a risk of compromised code repositories and unauthorized access." +regex = '''(?i)[\w.-]{0,50}?(?:bitbucket)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9=_\-]{64})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["bitbucket"] + +[[rules]] +id = "bittrex-access-key" +description = "Identified a Bittrex Access Key, which could lead to unauthorized access to cryptocurrency trading accounts and financial loss." +regex = '''(?i)[\w.-]{0,50}?(?:bittrex)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9]{32})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["bittrex"] + +[[rules]] +id = "bittrex-secret-key" +description = "Detected a Bittrex Secret Key, potentially compromising cryptocurrency transactions and financial security." +regex = '''(?i)[\w.-]{0,50}?(?:bittrex)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9]{32})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["bittrex"] + +[[rules]] +id = "cisco-meraki-api-key" +description = "Cisco Meraki is a cloud-managed IT solution that provides networking, security, and device management through an easy-to-use interface." +regex = '''[\w.-]{0,50}?(?i:[\w.-]{0,50}?(?:(?-i:[Mm]eraki|MERAKI))(?:[ \t\w.-]{0,20})[\s'"]{0,3})(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([0-9a-f]{40})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 3 +keywords = ["meraki"] + +[[rules]] +id = "clickhouse-cloud-api-secret-key" +description = "Identified a pattern that may indicate clickhouse cloud API secret key, risking unauthorized clickhouse cloud api access and data breaches on ClickHouse Cloud platforms." +regex = '''\b(4b1d[A-Za-z0-9]{38})\b''' +entropy = 3 +keywords = ["4b1d"] + +[[rules]] +id = "clojars-api-token" +description = "Uncovered a possible Clojars API token, risking unauthorized access to Clojure libraries and potential code manipulation." +regex = '''(?i)CLOJARS_[a-z0-9]{60}''' +entropy = 2 +keywords = ["clojars_"] + +[[rules]] +id = "cloudflare-api-key" +description = "Detected a Cloudflare API Key, potentially compromising cloud application deployments and operational security." +regex = '''(?i)[\w.-]{0,50}?(?:cloudflare)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9_-]{40})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 2 +keywords = ["cloudflare"] + +[[rules]] +id = "cloudflare-global-api-key" +description = "Detected a Cloudflare Global API Key, potentially compromising cloud application deployments and operational security." +regex = '''(?i)[\w.-]{0,50}?(?:cloudflare)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-f0-9]{37})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 2 +keywords = ["cloudflare"] + +[[rules]] +id = "cloudflare-origin-ca-key" +description = "Detected a Cloudflare Origin CA Key, potentially compromising cloud application deployments and operational security." +regex = '''\b(v1\.0-[a-f0-9]{24}-[a-f0-9]{146})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 2 +keywords = [ + "cloudflare", + "v1.0-", +] + +[[rules]] +id = "codecov-access-token" +description = "Found a pattern resembling a Codecov Access Token, posing a risk of unauthorized access to code coverage reports and sensitive data." +regex = '''(?i)[\w.-]{0,50}?(?:codecov)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9]{32})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["codecov"] + +[[rules]] +id = "cohere-api-token" +description = "Identified a Cohere Token, posing a risk of unauthorized access to AI services and data manipulation." +regex = '''[\w.-]{0,50}?(?i:[\w.-]{0,50}?(?:cohere|CO_API_KEY)(?:[ \t\w.-]{0,20})[\s'"]{0,3})(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-zA-Z0-9]{40})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 4 +keywords = [ + "cohere", + "co_api_key", +] + +[[rules]] +id = "coinbase-access-token" +description = "Detected a Coinbase Access Token, posing a risk of unauthorized access to cryptocurrency accounts and financial transactions." +regex = '''(?i)[\w.-]{0,50}?(?:coinbase)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9_-]{64})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["coinbase"] + +[[rules]] +id = "confluent-access-token" +description = "Identified a Confluent Access Token, which could compromise access to streaming data platforms and sensitive data flow." +regex = '''(?i)[\w.-]{0,50}?(?:confluent)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9]{16})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["confluent"] + +[[rules]] +id = "confluent-secret-key" +description = "Found a Confluent Secret Key, potentially risking unauthorized operations and data access within Confluent services." +regex = '''(?i)[\w.-]{0,50}?(?:confluent)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9]{64})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["confluent"] + +[[rules]] +id = "contentful-delivery-api-token" +description = "Discovered a Contentful delivery API token, posing a risk to content management systems and data integrity." +regex = '''(?i)[\w.-]{0,50}?(?:contentful)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9=_\-]{43})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["contentful"] + +[[rules]] +id = "curl-auth-header" +description = "Discovered a potential authorization token provided in a curl command header, which could compromise the curl accessed resource." +regex = '''\bcurl\b(?:.*?|.*?(?:[\r\n]{1,2}.*?){1,5})[ \t\n\r](?:-H|--header)(?:=|[ \t]{0,5})(?:"(?i)(?:Authorization:[ \t]{0,5}(?:Basic[ \t]([a-z0-9+/]{8,}={0,3})|(?:Bearer|(?:Api-)?Token)[ \t]([\w=~@.+/-]{8,})|([\w=~@.+/-]{8,}))|(?:(?:X-(?:[a-z]+-)?)?(?:Api-?)?(?:Key|Token)):[ \t]{0,5}([\w=~@.+/-]{8,}))"|'(?i)(?:Authorization:[ \t]{0,5}(?:Basic[ \t]([a-z0-9+/]{8,}={0,3})|(?:Bearer|(?:Api-)?Token)[ \t]([\w=~@.+/-]{8,})|([\w=~@.+/-]{8,}))|(?:(?:X-(?:[a-z]+-)?)?(?:Api-?)?(?:Key|Token)):[ \t]{0,5}([\w=~@.+/-]{8,}))')(?:\B|\s|\z)''' +entropy = 2.75 +keywords = ["curl"] + +[[rules]] +id = "curl-auth-user" +description = "Discovered a potential basic authorization token provided in a curl command, which could compromise the curl accessed resource." +regex = '''\bcurl\b(?:.*|.*(?:[\r\n]{1,2}.*){1,5})[ \t\n\r](?:-u|--user)(?:=|[ \t]{0,5})("(:[^"]{3,}|[^:"]{3,}:|[^:"]{3,}:[^"]{3,})"|'([^:']{3,}:[^']{3,})'|((?:"[^"]{3,}"|'[^']{3,}'|[\w$@.-]+):(?:"[^"]{3,}"|'[^']{3,}'|[\w${}@.-]+)))(?:\s|\z)''' +entropy = 2 +keywords = ["curl"] +[[rules.allowlists]] +regexes = [ + '''[^:]+:(?:change(?:it|me)|pass(?:word)?|pwd|test|token|\*+|x+)''', + '''['"]?<[^>]+>['"]?:['"]?<[^>]+>|<[^:]+:[^>]+>['"]?''', + '''[^:]+:\[[^]]+]''', + '''['"]?[^:]+['"]?:['"]?\$(?:\d|\w+|\{(?:\d|\w+)})['"]?''', + '''\$\([^)]+\):\$\([^)]+\)''', + '''['"]?\$?{{[^}]+}}['"]?:['"]?\$?{{[^}]+}}['"]?''', +] + +[[rules]] +id = "databricks-api-token" +description = "Uncovered a Databricks API token, which may compromise big data analytics platforms and sensitive data processing." +regex = '''\b(dapi[a-f0-9]{32}(?:-\d)?)(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 3 +keywords = ["dapi"] + +[[rules]] +id = "datadog-access-token" +description = "Detected a Datadog Access Token, potentially risking monitoring and analytics data exposure and manipulation." +regex = '''(?i)[\w.-]{0,50}?(?:datadog)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9]{40})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["datadog"] + +[[rules]] +id = "defined-networking-api-token" +description = "Identified a Defined Networking API token, which could lead to unauthorized network operations and data breaches." +regex = '''(?i)[\w.-]{0,50}?(?:dnkey)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}(dnkey-[a-z0-9=_\-]{26}-[a-z0-9=_\-]{52})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["dnkey"] + +[[rules]] +id = "digitalocean-access-token" +description = "Found a DigitalOcean OAuth Access Token, risking unauthorized cloud resource access and data compromise." +regex = '''\b(doo_v1_[a-f0-9]{64})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 3 +keywords = ["doo_v1_"] + +[[rules]] +id = "digitalocean-pat" +description = "Discovered a DigitalOcean Personal Access Token, posing a threat to cloud infrastructure security and data privacy." +regex = '''\b(dop_v1_[a-f0-9]{64})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 3 +keywords = ["dop_v1_"] + +[[rules]] +id = "digitalocean-refresh-token" +description = "Uncovered a DigitalOcean OAuth Refresh Token, which could allow prolonged unauthorized access and resource manipulation." +regex = '''(?i)\b(dor_v1_[a-f0-9]{64})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["dor_v1_"] + +[[rules]] +id = "discord-api-token" +description = "Detected a Discord API key, potentially compromising communication channels and user data privacy on Discord." +regex = '''(?i)[\w.-]{0,50}?(?:discord)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-f0-9]{64})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["discord"] + +[[rules]] +id = "discord-client-id" +description = "Identified a Discord client ID, which may lead to unauthorized integrations and data exposure in Discord applications." +regex = '''(?i)[\w.-]{0,50}?(?:discord)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([0-9]{18})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 2 +keywords = ["discord"] + +[[rules]] +id = "discord-client-secret" +description = "Discovered a potential Discord client secret, risking compromised Discord bot integrations and data leaks." +regex = '''(?i)[\w.-]{0,50}?(?:discord)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9=_\-]{32})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 2 +keywords = ["discord"] + +[[rules]] +id = "doppler-api-token" +description = "Discovered a Doppler API token, posing a risk to environment and secrets management security." +regex = '''dp\.pt\.(?i)[a-z0-9]{43}''' +entropy = 2 +keywords = ["dp.pt."] + +[[rules]] +id = "droneci-access-token" +description = "Detected a Droneci Access Token, potentially compromising continuous integration and deployment workflows." +regex = '''(?i)[\w.-]{0,50}?(?:droneci)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9]{32})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["droneci"] + +[[rules]] +id = "dropbox-api-token" +description = "Identified a Dropbox API secret, which could lead to unauthorized file access and data breaches in Dropbox storage." +regex = '''(?i)[\w.-]{0,50}?(?:dropbox)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9]{15})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["dropbox"] + +[[rules]] +id = "dropbox-long-lived-api-token" +description = "Found a Dropbox long-lived API token, risking prolonged unauthorized access to cloud storage and sensitive data." +regex = '''(?i)[\w.-]{0,50}?(?:dropbox)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9]{11}(AAAAAAAAAA)[a-z0-9\-_=]{43})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["dropbox"] + +[[rules]] +id = "dropbox-short-lived-api-token" +description = "Discovered a Dropbox short-lived API token, posing a risk of temporary but potentially harmful data access and manipulation." +regex = '''(?i)[\w.-]{0,50}?(?:dropbox)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}(sl\.[a-z0-9\-=_]{135})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["dropbox"] + +[[rules]] +id = "duffel-api-token" +description = "Uncovered a Duffel API token, which may compromise travel platform integrations and sensitive customer data." +regex = '''duffel_(?:test|live)_(?i)[a-z0-9_\-=]{43}''' +entropy = 2 +keywords = ["duffel_"] + +[[rules]] +id = "dynatrace-api-token" +description = "Detected a Dynatrace API token, potentially risking application performance monitoring and data exposure." +regex = '''dt0c01\.(?i)[a-z0-9]{24}\.[a-z0-9]{64}''' +entropy = 4 +keywords = ["dt0c01."] + +[[rules]] +id = "easypost-api-token" +description = "Identified an EasyPost API token, which could lead to unauthorized postal and shipment service access and data exposure." +regex = '''\bEZAK(?i)[a-z0-9]{54}\b''' +entropy = 2 +keywords = ["ezak"] + +[[rules]] +id = "easypost-test-api-token" +description = "Detected an EasyPost test API token, risking exposure of test environments and potentially sensitive shipment data." +regex = '''\bEZTK(?i)[a-z0-9]{54}\b''' +entropy = 2 +keywords = ["eztk"] + +[[rules]] +id = "etsy-access-token" +description = "Found an Etsy Access Token, potentially compromising Etsy shop management and customer data." +regex = '''(?i)[\w.-]{0,50}?(?:(?-i:ETSY|[Ee]tsy))(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9]{24})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 3 +keywords = ["etsy"] + +[[rules]] +id = "facebook-access-token" +description = "Discovered a Facebook Access Token, posing a risk of unauthorized access to Facebook accounts and personal data exposure." +regex = '''(?i)\b(\d{15,16}(\||%)[0-9a-z\-_]{27,40})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 3 +keywords = ["facebook"] + +[[rules]] +id = "facebook-page-access-token" +description = "Discovered a Facebook Page Access Token, posing a risk of unauthorized access to Facebook accounts and personal data exposure." +regex = '''\b(EAA[MC](?i)[a-z0-9]{100,})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 4 +keywords = [ + "eaam", + "eaac", +] + +[[rules]] +id = "facebook-secret" +description = "Discovered a Facebook Application secret, posing a risk of unauthorized access to Facebook accounts and personal data exposure." +regex = '''(?i)[\w.-]{0,50}?(?:facebook)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-f0-9]{32})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 3 +keywords = ["facebook"] + +[[rules]] +id = "fastly-api-token" +description = "Uncovered a Fastly API key, which may compromise CDN and edge cloud services, leading to content delivery and security issues." +regex = '''(?i)[\w.-]{0,50}?(?:fastly)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9=_\-]{32})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["fastly"] + +[[rules]] +id = "finicity-api-token" +description = "Detected a Finicity API token, potentially risking financial data access and unauthorized financial operations." +regex = '''(?i)[\w.-]{0,50}?(?:finicity)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-f0-9]{32})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["finicity"] + +[[rules]] +id = "finicity-client-secret" +description = "Identified a Finicity Client Secret, which could lead to compromised financial service integrations and data breaches." +regex = '''(?i)[\w.-]{0,50}?(?:finicity)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9]{20})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["finicity"] + +[[rules]] +id = "finnhub-access-token" +description = "Found a Finnhub Access Token, risking unauthorized access to financial market data and analytics." +regex = '''(?i)[\w.-]{0,50}?(?:finnhub)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9]{20})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["finnhub"] + +[[rules]] +id = "flickr-access-token" +description = "Discovered a Flickr Access Token, posing a risk of unauthorized photo management and potential data leakage." +regex = '''(?i)[\w.-]{0,50}?(?:flickr)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9]{32})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["flickr"] + +[[rules]] +id = "flutterwave-encryption-key" +description = "Uncovered a Flutterwave Encryption Key, which may compromise payment processing and sensitive financial information." +regex = '''FLWSECK_TEST-(?i)[a-h0-9]{12}''' +entropy = 2 +keywords = ["flwseck_test"] + +[[rules]] +id = "flutterwave-public-key" +description = "Detected a Finicity Public Key, potentially exposing public cryptographic operations and integrations." +regex = '''FLWPUBK_TEST-(?i)[a-h0-9]{32}-X''' +entropy = 2 +keywords = ["flwpubk_test"] + +[[rules]] +id = "flutterwave-secret-key" +description = "Identified a Flutterwave Secret Key, risking unauthorized financial transactions and data breaches." +regex = '''FLWSECK_TEST-(?i)[a-h0-9]{32}-X''' +entropy = 2 +keywords = ["flwseck_test"] + +[[rules]] +id = "flyio-access-token" +description = "Uncovered a Fly.io API key" +regex = '''\b((?:fo1_[\w-]{43}|fm1[ar]_[a-zA-Z0-9+\/]{100,}={0,3}|fm2_[a-zA-Z0-9+\/]{100,}={0,3}))(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 4 +keywords = [ + "fo1_", + "fm1", + "fm2_", +] + +[[rules]] +id = "frameio-api-token" +description = "Found a Frame.io API token, potentially compromising video collaboration and project management." +regex = '''fio-u-(?i)[a-z0-9\-_=]{64}''' +keywords = ["fio-u-"] + +[[rules]] +id = "freemius-secret-key" +description = "Detected a Freemius secret key, potentially exposing sensitive information." +regex = '''(?i)["']secret_key["']\s*=>\s*["'](sk_[\S]{29})["']''' +path = '''(?i)\.php$''' +keywords = ["secret_key"] + +[[rules]] +id = "freshbooks-access-token" +description = "Discovered a Freshbooks Access Token, posing a risk to accounting software access and sensitive financial data exposure." +regex = '''(?i)[\w.-]{0,50}?(?:freshbooks)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9]{64})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["freshbooks"] + +[[rules]] +id = "gcp-api-key" +description = "Uncovered a GCP API key, which could lead to unauthorized access to Google Cloud services and data breaches." +regex = '''\b(AIza[\w-]{35})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 4 +keywords = ["aiza"] +[[rules.allowlists]] +regexes = [ + '''AIzaSyabcdefghijklmnopqrstuvwxyz1234567''', + '''AIzaSyAnLA7NfeLquW1tJFpx_eQCxoX-oo6YyIs''', + '''AIzaSyCkEhVjf3pduRDt6d1yKOMitrUEke8agEM''', + '''AIzaSyDMAScliyLx7F0NPDEJi1QmyCgHIAODrlU''', + '''AIzaSyD3asb-2pEZVqMkmL6M9N6nHZRR_znhrh0''', + '''AIzayDNSXIbFmlXbIE6mCzDLQAqITYefhixbX4A''', + '''AIzaSyAdOS2zB6NCsk1pCdZ4-P6GBdi_UUPwX7c''', + '''AIzaSyASWm6HmTMdYWpgMnjRBjxcQ9CKctWmLd4''', + '''AIzaSyANUvH9H9BsUccjsu2pCmEkOPjjaXeDQgY''', + '''AIzaSyA5_iVawFQ8ABuTZNUdcwERLJv_a_p4wtM''', + '''AIzaSyA4UrcGxgwQFTfaI3no3t7Lt1sjmdnP5sQ''', + '''AIzaSyDSb51JiIcB6OJpwwMicseKRhhrOq1cS7g''', + '''AIzaSyBF2RrAIm4a0mO64EShQfqfd2AFnzAvvuU''', + '''AIzaSyBcE-OOIbhjyR83gm4r2MFCu4MJmprNXsw''', + '''AIzaSyB8qGxt4ec15vitgn44duC5ucxaOi4FmqE''', + '''AIzaSyA8vmApnrHNFE0bApF4hoZ11srVL_n0nvY''', +] + +[[rules]] +id = "generic-api-key" +description = "Detected a Generic API Key, potentially exposing access to various services and sensitive operations." +regex = '''(?i)[\w.-]{0,50}?(?:access|auth|(?-i:[Aa]pi|API)|credential|creds|key|passw(?:or)?d|secret|token)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([\w.=-]{10,150}|[a-z0-9][a-z0-9+/]{11,}={0,3})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 3.5 +keywords = [ + "access", + "api", + "auth", + "key", + "credential", + "creds", + "passwd", + "password", + "secret", + "token", +] +[[rules.allowlists]] +regexes = [ + '''^[a-zA-Z_.-]+$''', +] +[[rules.allowlists]] +description = "Allowlist for Generic API Keys" +regexTarget = "match" +regexes = [ + '''(?i)(?:access(?:ibility|or)|access[_.-]?id|random[_.-]?access|api[_.-]?(?:id|name|version)|rapid|capital|[a-z0-9-]*?api[a-z0-9-]*?:jar:|author|X-MS-Exchange-Organization-Auth|Authentication-Results|(?:credentials?[_.-]?id|withCredentials)|(?:bucket|foreign|hot|idx|natural|primary|pub(?:lic)?|schema|sequence)[_.-]?key|(?:turkey)|key[_.-]?(?:alias|board|code|frame|id|length|mesh|name|pair|press(?:ed)?|ring|selector|signature|size|stone|storetype|word|up|down|left|right)|key[_.-]?vault[_.-]?(?:id|name)|keyVaultToStoreSecrets|key(?:store|tab)[_.-]?(?:file|path)|issuerkeyhash|(?-i:[DdMm]onkey|[DM]ONKEY)|keying|(?:secret)[_.-]?(?:length|name|size)|UserSecretsId|(?:csrf)[_.-]?token|(?:io\.jsonwebtoken[ \t]?:[ \t]?[\w-]+)|(?:api|credentials|token)[_.-]?(?:endpoint|ur[il])|public[_.-]?token|(?:key|token)[_.-]?file|(?-i:(?:[A-Z_]+=\n[A-Z_]+=|[a-z_]+=\n[a-z_]+=)(?:\n|\z))|(?-i:(?:[A-Z.]+=\n[A-Z.]+=|[a-z.]+=\n[a-z.]+=)(?:\n|\z)))''', +] +stopwords = [ + "000000", + "6fe4476ee5a1832882e326b506d14126", + "_ec2_", + "aaaaaa", + "about", + "abstract", + "academy", + "acces", + "account", + "act-", + "act.", + "act_", + "action", + "active", + "actively", + "activity", + "adapter", + "add-", + "add-on", + "add.", + "add_", + "addon", + "addres", + "admin", + "adobe", + "advanced", + "adventure", + "agent", + "agile", + "air-", + "air.", + "air_", + "ajax", + "akka", + "alert", + "alfred", + "algorithm", + "all-", + "all.", + "all_", + "alloy", + "alpha", + "amazon", + "amqp", + "analysi", + "analytic", + "analyzer", + "android", + "angular", + "angularj", + "animate", + "animation", + "another", + "ansible", + "answer", + "ant-", + "ant.", + "ant_", + "any-", + "any.", + "any_", + "apache", + "app-", + "app.", + "app_", + "apple", + "arch", + "archive", + "archived", + "arduino", + "array", + "art-", + "art.", + "art_", + "article", + "asp-", + "asp.", + "asp_", + "asset", + "async", + "atom", + "attention", + "audio", + "audit", + "aura", + "auth", + "author", + "authorize", + "auto", + "automated", + "automatic", + "awesome", + "aws_", + "azure", + "back", + "backbone", + "backend", + "backup", + "bar-", + "bar.", + "bar_", + "base", + "based", + "bash", + "basic", + "batch", + "been", + "beer", + "behavior", + "being", + "benchmark", + "best", + "beta", + "better", + "big-", + "big.", + "big_", + "binary", + "binding", + "bit-", + "bit.", + "bit_", + "bitcoin", + "block", + "blog", + "board", + "book", + "bookmark", + "boost", + "boot", + "bootstrap", + "bosh", + "bot-", + "bot.", + "bot_", + "bower", + "box-", + "box.", + "box_", + "boxen", + "bracket", + "branch", + "bridge", + "browser", + "brunch", + "buffer", + "bug-", + "bug.", + "bug_", + "build", + "builder", + "building", + "buildout", + "buildpack", + "built", + "bundle", + "busines", + "but-", + "but.", + "but_", + "button", + "cache", + "caching", + "cakephp", + "calendar", + "call", + "camera", + "campfire", + "can-", + "can.", + "can_", + "canva", + "captcha", + "capture", + "card", + "carousel", + "case", + "cassandra", + "cat-", + "cat.", + "cat_", + "category", + "center", + "cento", + "challenge", + "change", + "changelog", + "channel", + "chart", + "chat", + "cheat", + "check", + "checker", + "chef", + "ches", + "chinese", + "chosen", + "chrome", + "ckeditor", + "clas", + "classe", + "classic", + "clean", + "cli-", + "cli.", + "cli_", + "client", + "clojure", + "clone", + "closure", + "cloud", + "club", + "cluster", + "cms-", + "cms_", + "coco", + "code", + "coding", + "coffee", + "color", + "combination", + "combo", + "command", + "commander", + "comment", + "commit", + "common", + "community", + "compas", + "compiler", + "complete", + "component", + "composer", + "computer", + "computing", + "con-", + "con.", + "con_", + "concept", + "conf", + "config", + "connect", + "connector", + "console", + "contact", + "container", + "contao", + "content", + "contest", + "context", + "control", + "convert", + "converter", + "conway'", + "cookbook", + "cookie", + "cool", + "copy", + "cordova", + "core", + "couchbase", + "couchdb", + "countdown", + "counter", + "course", + "craft", + "crawler", + "create", + "creating", + "creator", + "credential", + "crm-", + "crm.", + "crm_", + "cros", + "crud", + "csv-", + "csv.", + "csv_", + "cube", + "cucumber", + "cuda", + "current", + "currently", + "custom", + "daemon", + "dark", + "dart", + "dash", + "dashboard", + "data", + "database", + "date", + "day-", + "day.", + "day_", + "dead", + "debian", + "debug", + "debugger", + "deck", + "define", + "del-", + "del.", + "del_", + "delete", + "demo", + "deploy", + "design", + "designer", + "desktop", + "detection", + "detector", + "dev-", + "dev.", + "dev_", + "develop", + "developer", + "device", + "devise", + "diff", + "digital", + "directive", + "directory", + "discovery", + "display", + "django", + "dns-", + "dns_", + "doc-", + "doc.", + "doc_", + "docker", + "docpad", + "doctrine", + "document", + "doe-", + "doe.", + "doe_", + "dojo", + "dom-", + "dom.", + "dom_", + "domain", + "don't", + "done", + "dot-", + "dot.", + "dot_", + "dotfile", + "download", + "draft", + "drag", + "drill", + "drive", + "driven", + "driver", + "drop", + "dropbox", + "drupal", + "dsl-", + "dsl.", + "dsl_", + "dynamic", + "easy", + "ecdsa", + "eclipse", + "edit", + "editing", + "edition", + "editor", + "element", + "emac", + "email", + "embed", + "embedded", + "ember", + "emitter", + "emulator", + "encoding", + "endpoint", + "engine", + "english", + "enhanced", + "entity", + "entry", + "env_", + "episode", + "erlang", + "error", + "espresso", + "event", + "evented", + "example", + "exchange", + "exercise", + "experiment", + "expire", + "exploit", + "explorer", + "export", + "exporter", + "expres", + "ext-", + "ext.", + "ext_", + "extended", + "extension", + "external", + "extra", + "extractor", + "fabric", + "facebook", + "factory", + "fake", + "fast", + "feature", + "feed", + "fewfwef", + "ffmpeg", + "field", + "file", + "filter", + "find", + "finder", + "firefox", + "firmware", + "first", + "fish", + "fix-", + "fix_", + "flash", + "flask", + "flat", + "flex", + "flexible", + "flickr", + "flow", + "fluent", + "fluentd", + "fluid", + "folder", + "font", + "force", + "foreman", + "fork", + "form", + "format", + "formatter", + "forum", + "foundry", + "framework", + "free", + "friend", + "friendly", + "front-end", + "frontend", + "ftp-", + "ftp.", + "ftp_", + "fuel", + "full", + "fun-", + "fun.", + "fun_", + "func", + "future", + "gaia", + "gallery", + "game", + "gateway", + "gem-", + "gem.", + "gem_", + "gen-", + "gen.", + "gen_", + "general", + "generator", + "generic", + "genetic", + "get-", + "get.", + "get_", + "getenv", + "getting", + "ghost", + "gist", + "git-", + "git.", + "git_", + "github", + "gitignore", + "gitlab", + "glas", + "gmail", + "gnome", + "gnu-", + "gnu.", + "gnu_", + "goal", + "golang", + "gollum", + "good", + "google", + "gpu-", + "gpu.", + "gpu_", + "gradle", + "grail", + "graph", + "graphic", + "great", + "grid", + "groovy", + "group", + "grunt", + "guard", + "gui-", + "gui.", + "gui_", + "guide", + "guideline", + "gulp", + "gwt-", + "gwt.", + "gwt_", + "hack", + "hackathon", + "hacker", + "hacking", + "hadoop", + "haml", + "handler", + "hardware", + "has-", + "has_", + "hash", + "haskell", + "have", + "haxe", + "hello", + "help", + "helper", + "here", + "hero", + "heroku", + "high", + "hipchat", + "history", + "home", + "homebrew", + "homepage", + "hook", + "host", + "hosting", + "hot-", + "hot.", + "hot_", + "house", + "how-", + "how.", + "how_", + "html", + "http", + "hub-", + "hub.", + "hub_", + "hubot", + "human", + "icon", + "ide-", + "ide.", + "ide_", + "idea", + "identity", + "idiomatic", + "image", + "impact", + "import", + "important", + "importer", + "impres", + "index", + "infinite", + "info", + "injection", + "inline", + "input", + "inside", + "inspector", + "instagram", + "install", + "installer", + "instant", + "intellij", + "interface", + "internet", + "interview", + "into", + "intro", + "ionic", + "iphone", + "ipython", + "irc-", + "irc_", + "iso-", + "iso.", + "iso_", + "issue", + "jade", + "jasmine", + "java", + "jbos", + "jekyll", + "jenkin", + "jetbrains", + "job-", + "job.", + "job_", + "joomla", + "jpa-", + "jpa.", + "jpa_", + "jquery", + "json", + "just", + "kafka", + "karma", + "kata", + "kernel", + "keyboard", + "kindle", + "kit-", + "kit.", + "kit_", + "kitchen", + "knife", + "koan", + "kohana", + "lab-", + "lab.", + "lab_", + "lambda", + "lamp", + "language", + "laravel", + "last", + "latest", + "latex", + "launcher", + "layer", + "layout", + "lazy", + "ldap", + "leaflet", + "league", + "learn", + "learning", + "led-", + "led.", + "led_", + "leetcode", + "les-", + "les.", + "les_", + "level", + "leveldb", + "lib-", + "lib.", + "lib_", + "librarie", + "library", + "license", + "life", + "liferay", + "light", + "lightbox", + "like", + "line", + "link", + "linked", + "linkedin", + "linux", + "lisp", + "list", + "lite", + "little", + "load", + "loader", + "local", + "location", + "lock", + "log-", + "log.", + "log_", + "logger", + "logging", + "logic", + "login", + "logstash", + "longer", + "look", + "love", + "lua-", + "lua.", + "lua_", + "mac-", + "mac.", + "mac_", + "machine", + "made", + "magento", + "magic", + "mail", + "make", + "maker", + "making", + "man-", + "man.", + "man_", + "manage", + "manager", + "manifest", + "manual", + "map-", + "map.", + "map_", + "mapper", + "mapping", + "markdown", + "markup", + "master", + "math", + "matrix", + "maven", + "md5", + "mean", + "media", + "mediawiki", + "meetup", + "memcached", + "memory", + "menu", + "merchant", + "message", + "messaging", + "meta", + "metadata", + "meteor", + "method", + "metric", + "micro", + "middleman", + "migration", + "minecraft", + "miner", + "mini", + "minimal", + "mirror", + "mit-", + "mit.", + "mit_", + "mobile", + "mocha", + "mock", + "mod-", + "mod.", + "mod_", + "mode", + "model", + "modern", + "modular", + "module", + "modx", + "money", + "mongo", + "mongodb", + "mongoid", + "mongoose", + "monitor", + "monkey", + "more", + "motion", + "moved", + "movie", + "mozilla", + "mqtt", + "mule", + "multi", + "multiple", + "music", + "mustache", + "mvc-", + "mvc.", + "mvc_", + "mysql", + "nagio", + "name", + "native", + "need", + "neo-", + "neo.", + "neo_", + "nest", + "nested", + "net-", + "net.", + "net_", + "nette", + "network", + "new-", + "new.", + "new_", + "next", + "nginx", + "ninja", + "nlp-", + "nlp.", + "nlp_", + "node", + "nodej", + "nosql", + "not-", + "not.", + "not_", + "note", + "notebook", + "notepad", + "notice", + "notifier", + "now-", + "now.", + "now_", + "number", + "oauth", + "object", + "objective", + "obsolete", + "ocaml", + "octopres", + "official", + "old-", + "old.", + "old_", + "onboard", + "online", + "only", + "open", + "opencv", + "opengl", + "openshift", + "openwrt", + "option", + "oracle", + "org-", + "org.", + "org_", + "origin", + "original", + "orm-", + "orm.", + "orm_", + "osx-", + "osx_", + "our-", + "our.", + "our_", + "out-", + "out.", + "out_", + "output", + "over", + "overview", + "own-", + "own.", + "own_", + "pack", + "package", + "packet", + "page", + "panel", + "paper", + "paperclip", + "para", + "parallax", + "parallel", + "parse", + "parser", + "parsing", + "particle", + "party", + "password", + "patch", + "path", + "pattern", + "payment", + "paypal", + "pdf-", + "pdf.", + "pdf_", + "pebble", + "people", + "perl", + "personal", + "phalcon", + "phoenix", + "phone", + "phonegap", + "photo", + "php-", + "php.", + "php_", + "physic", + "picker", + "pipeline", + "platform", + "play", + "player", + "please", + "plu-", + "plu.", + "plu_", + "plug-in", + "plugin", + "plupload", + "png-", + "png.", + "png_", + "poker", + "polyfill", + "polymer", + "pool", + "pop-", + "pop.", + "pop_", + "popcorn", + "popup", + "port", + "portable", + "portal", + "portfolio", + "post", + "power", + "powered", + "powerful", + "prelude", + "pretty", + "preview", + "principle", + "print", + "pro-", + "pro.", + "pro_", + "problem", + "proc", + "product", + "profile", + "profiler", + "program", + "progres", + "project", + "protocol", + "prototype", + "provider", + "proxy", + "public", + "pull", + "puppet", + "pure", + "purpose", + "push", + "pusher", + "pyramid", + "python", + "quality", + "query", + "queue", + "quick", + "rabbitmq", + "rack", + "radio", + "rail", + "railscast", + "random", + "range", + "raspberry", + "rdf-", + "rdf.", + "rdf_", + "react", + "reactive", + "read", + "reader", + "readme", + "ready", + "real", + "real-time", + "reality", + "realtime", + "recipe", + "recorder", + "red-", + "red.", + "red_", + "reddit", + "redi", + "redmine", + "reference", + "refinery", + "refresh", + "registry", + "related", + "release", + "remote", + "rendering", + "repo", + "report", + "request", + "require", + "required", + "requirej", + "research", + "resource", + "response", + "resque", + "rest", + "restful", + "resume", + "reveal", + "reverse", + "review", + "riak", + "rich", + "right", + "ring", + "robot", + "role", + "room", + "router", + "routing", + "rpc-", + "rpc.", + "rpc_", + "rpg-", + "rpg.", + "rpg_", + "rspec", + "ruby-", + "ruby.", + "ruby_", + "rule", + "run-", + "run.", + "run_", + "runner", + "running", + "runtime", + "rust", + "rvm-", + "rvm.", + "rvm_", + "salt", + "sample", + "sandbox", + "sas-", + "sas.", + "sas_", + "sbt-", + "sbt.", + "sbt_", + "scala", + "scalable", + "scanner", + "schema", + "scheme", + "school", + "science", + "scraper", + "scratch", + "screen", + "script", + "scroll", + "scs-", + "scs.", + "scs_", + "sdk-", + "sdk.", + "sdk_", + "sdl-", + "sdl.", + "sdl_", + "search", + "secure", + "security", + "see-", + "see.", + "see_", + "seed", + "select", + "selector", + "selenium", + "semantic", + "sencha", + "send", + "sentiment", + "serie", + "server", + "service", + "session", + "set-", + "set.", + "set_", + "setting", + "setup", + "sha1", + "sha2", + "sha256", + "share", + "shared", + "sharing", + "sheet", + "shell", + "shield", + "shipping", + "shop", + "shopify", + "shortener", + "should", + "show", + "showcase", + "side", + "silex", + "simple", + "simulator", + "single", + "site", + "skeleton", + "sketch", + "skin", + "slack", + "slide", + "slider", + "slim", + "small", + "smart", + "smtp", + "snake", + "snapshot", + "snippet", + "soap", + "social", + "socket", + "software", + "solarized", + "solr", + "solution", + "solver", + "some", + "soon", + "source", + "space", + "spark", + "spatial", + "spec", + "sphinx", + "spine", + "spotify", + "spree", + "spring", + "sprite", + "sql-", + "sql.", + "sql_", + "sqlite", + "ssh-", + "ssh.", + "ssh_", + "stack", + "staging", + "standard", + "stanford", + "start", + "started", + "starter", + "startup", + "stat", + "statamic", + "state", + "static", + "statistic", + "statsd", + "statu", + "steam", + "step", + "still", + "stm-", + "stm.", + "stm_", + "storage", + "store", + "storm", + "story", + "strategy", + "stream", + "streaming", + "string", + "stripe", + "structure", + "studio", + "study", + "stuff", + "style", + "sublime", + "sugar", + "suite", + "summary", + "super", + "support", + "supported", + "svg-", + "svg.", + "svg_", + "svn-", + "svn.", + "svn_", + "swagger", + "swift", + "switch", + "switcher", + "symfony", + "symphony", + "sync", + "synopsi", + "syntax", + "system", + "tab-", + "tab.", + "tab_", + "table", + "tag-", + "tag.", + "tag_", + "talk", + "target", + "task", + "tcp-", + "tcp.", + "tcp_", + "tdd-", + "tdd.", + "tdd_", + "team", + "tech", + "template", + "term", + "terminal", + "testing", + "tetri", + "text", + "textmate", + "theme", + "theory", + "three", + "thrift", + "time", + "timeline", + "timer", + "tiny", + "tinymce", + "tip-", + "tip.", + "tip_", + "title", + "todo", + "todomvc", + "token", + "tool", + "toolbox", + "toolkit", + "top-", + "top.", + "top_", + "tornado", + "touch", + "tower", + "tracker", + "tracking", + "traffic", + "training", + "transfer", + "translate", + "transport", + "tree", + "trello", + "try-", + "try.", + "try_", + "tumblr", + "tut-", + "tut.", + "tut_", + "tutorial", + "tweet", + "twig", + "twitter", + "type", + "typo", + "ubuntu", + "uiview", + "ultimate", + "under", + "unit", + "unity", + "universal", + "unix", + "update", + "updated", + "upgrade", + "upload", + "uploader", + "uri-", + "uri.", + "uri_", + "url-", + "url.", + "url_", + "usage", + "usb-", + "usb.", + "usb_", + "use-", + "use.", + "use_", + "used", + "useful", + "user", + "using", + "util", + "utilitie", + "utility", + "vagrant", + "validator", + "value", + "variou", + "varnish", + "version", + "via-", + "via.", + "via_", + "video", + "view", + "viewer", + "vim-", + "vim.", + "vim_", + "vimrc", + "virtual", + "vision", + "visual", + "vpn", + "want", + "warning", + "watch", + "watcher", + "wave", + "way-", + "way.", + "way_", + "weather", + "web-", + "web_", + "webapp", + "webgl", + "webhook", + "webkit", + "webrtc", + "website", + "websocket", + "welcome", + "what", + "what'", + "when", + "where", + "which", + "why-", + "why.", + "why_", + "widget", + "wifi", + "wiki", + "win-", + "win.", + "win_", + "window", + "wip-", + "wip.", + "wip_", + "within", + "without", + "wizard", + "word", + "wordpres", + "work", + "worker", + "workflow", + "working", + "workshop", + "world", + "wrapper", + "write", + "writer", + "writing", + "written", + "www-", + "www.", + "www_", + "xamarin", + "xcode", + "xml-", + "xml.", + "xml_", + "xmpp", + "xxxxxx", + "yahoo", + "yaml", + "yandex", + "yeoman", + "yet-", + "yet.", + "yet_", + "yii-", + "yii.", + "yii_", + "youtube", + "yui-", + "yui.", + "yui_", + "zend", + "zero", + "zip-", + "zip.", + "zip_", + "zsh-", + "zsh.", + "zsh_", +] +[[rules.allowlists]] +regexTarget = "line" +regexes = [ + '''--mount=type=secret,''', + '''import[ \t]+{[ \t\w,]+}[ \t]+from[ \t]+['"][^'"]+['"]''', +] +[[rules.allowlists]] +condition = "AND" +paths = [ + '''\.bb$''','''\.bbappend$''','''\.bbclass$''','''\.inc$''', +] +regexTarget = "line" +regexes = [ + '''LICENSE[^=]*=\s*"[^"]+''', + '''LIC_FILES_CHKSUM[^=]*=\s*"[^"]+''', + '''SRC[^=]*=\s*"[a-zA-Z0-9]+''', +] + +[[rules]] +id = "github-app-token" +description = "Identified a GitHub App Token, which may compromise GitHub application integrations and source code security." +regex = '''(?:ghu|ghs)_[0-9a-zA-Z]{36}''' +entropy = 3 +keywords = [ + "ghu_", + "ghs_", +] +[[rules.allowlists]] +paths = [ + '''(?:^|/)@octokit/auth-token/README\.md$''', +] + +[[rules]] +id = "github-fine-grained-pat" +description = "Found a GitHub Fine-Grained Personal Access Token, risking unauthorized repository access and code manipulation." +regex = '''github_pat_\w{82}''' +entropy = 3 +keywords = ["github_pat_"] + +[[rules]] +id = "github-oauth" +description = "Discovered a GitHub OAuth Access Token, posing a risk of compromised GitHub account integrations and data leaks." +regex = '''gho_[0-9a-zA-Z]{36}''' +entropy = 3 +keywords = ["gho_"] + +[[rules]] +id = "github-pat" +description = "Uncovered a GitHub Personal Access Token, potentially leading to unauthorized repository access and sensitive content exposure." +regex = '''ghp_[0-9a-zA-Z]{36}''' +entropy = 3 +keywords = ["ghp_"] +[[rules.allowlists]] +paths = [ + '''(?:^|/)@octokit/auth-token/README\.md$''', +] + +[[rules]] +id = "github-refresh-token" +description = "Detected a GitHub Refresh Token, which could allow prolonged unauthorized access to GitHub services." +regex = '''ghr_[0-9a-zA-Z]{36}''' +entropy = 3 +keywords = ["ghr_"] + +[[rules]] +id = "gitlab-cicd-job-token" +description = "Identified a GitLab CI/CD Job Token, potential access to projects and some APIs on behalf of a user while the CI job is running." +regex = '''glcbt-[0-9a-zA-Z]{1,5}_[0-9a-zA-Z_-]{20}''' +entropy = 3 +keywords = ["glcbt-"] + +[[rules]] +id = "gitlab-deploy-token" +description = "Identified a GitLab Deploy Token, risking access to repositories, packages and containers with write access." +regex = '''gldt-[0-9a-zA-Z_\-]{20}''' +entropy = 3 +keywords = ["gldt-"] + +[[rules]] +id = "gitlab-feature-flag-client-token" +description = "Identified a GitLab feature flag client token, risks exposing user lists and features flags used by an application." +regex = '''glffct-[0-9a-zA-Z_\-]{20}''' +entropy = 3 +keywords = ["glffct-"] + +[[rules]] +id = "gitlab-feed-token" +description = "Identified a GitLab feed token, risking exposure of user data." +regex = '''glft-[0-9a-zA-Z_\-]{20}''' +entropy = 3 +keywords = ["glft-"] + +[[rules]] +id = "gitlab-incoming-mail-token" +description = "Identified a GitLab incoming mail token, risking manipulation of data sent by mail." +regex = '''glimt-[0-9a-zA-Z_\-]{25}''' +entropy = 3 +keywords = ["glimt-"] + +[[rules]] +id = "gitlab-kubernetes-agent-token" +description = "Identified a GitLab Kubernetes Agent token, risking access to repos and registry of projects connected via agent." +regex = '''glagent-[0-9a-zA-Z_\-]{50}''' +entropy = 3 +keywords = ["glagent-"] + +[[rules]] +id = "gitlab-oauth-app-secret" +description = "Identified a GitLab OIDC Application Secret, risking access to apps using GitLab as authentication provider." +regex = '''gloas-[0-9a-zA-Z_\-]{64}''' +entropy = 3 +keywords = ["gloas-"] + +[[rules]] +id = "gitlab-pat" +description = "Identified a GitLab Personal Access Token, risking unauthorized access to GitLab repositories and codebase exposure." +regex = '''glpat-[\w-]{20}''' +entropy = 3 +keywords = ["glpat-"] + +[[rules]] +id = "gitlab-pat-routable" +description = "Identified a GitLab Personal Access Token (routable), risking unauthorized access to GitLab repositories and codebase exposure." +regex = '''\bglpat-[0-9a-zA-Z_-]{27,300}\.[0-9a-z]{2}[0-9a-z]{7}\b''' +entropy = 4 +keywords = ["glpat-"] + +[[rules]] +id = "gitlab-ptt" +description = "Found a GitLab Pipeline Trigger Token, potentially compromising continuous integration workflows and project security." +regex = '''glptt-[0-9a-f]{40}''' +entropy = 3 +keywords = ["glptt-"] + +[[rules]] +id = "gitlab-rrt" +description = "Discovered a GitLab Runner Registration Token, posing a risk to CI/CD pipeline integrity and unauthorized access." +regex = '''GR1348941[\w-]{20}''' +entropy = 3 +keywords = ["gr1348941"] + +[[rules]] +id = "gitlab-runner-authentication-token" +description = "Discovered a GitLab Runner Authentication Token, posing a risk to CI/CD pipeline integrity and unauthorized access." +regex = '''glrt-[0-9a-zA-Z_\-]{20}''' +entropy = 3 +keywords = ["glrt-"] + +[[rules]] +id = "gitlab-runner-authentication-token-routable" +description = "Discovered a GitLab Runner Authentication Token (Routable), posing a risk to CI/CD pipeline integrity and unauthorized access." +regex = '''\bglrt-t\d_[0-9a-zA-Z_\-]{27,300}\.[0-9a-z]{2}[0-9a-z]{7}\b''' +entropy = 4 +keywords = ["glrt-"] + +[[rules]] +id = "gitlab-scim-token" +description = "Discovered a GitLab SCIM Token, posing a risk to unauthorized access for a organization or instance." +regex = '''glsoat-[0-9a-zA-Z_\-]{20}''' +entropy = 3 +keywords = ["glsoat-"] + +[[rules]] +id = "gitlab-session-cookie" +description = "Discovered a GitLab Session Cookie, posing a risk to unauthorized access to a user account." +regex = '''_gitlab_session=[0-9a-z]{32}''' +entropy = 3 +keywords = ["_gitlab_session="] + +[[rules]] +id = "gitter-access-token" +description = "Uncovered a Gitter Access Token, which may lead to unauthorized access to chat and communication services." +regex = '''(?i)[\w.-]{0,50}?(?:gitter)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9_-]{40})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["gitter"] + +[[rules]] +id = "gocardless-api-token" +description = "Detected a GoCardless API token, potentially risking unauthorized direct debit payment operations and financial data exposure." +regex = '''(?i)[\w.-]{0,50}?(?:gocardless)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}(live_(?i)[a-z0-9\-_=]{40})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = [ + "live_", + "gocardless", +] + +[[rules]] +id = "grafana-api-key" +description = "Identified a Grafana API key, which could compromise monitoring dashboards and sensitive data analytics." +regex = '''(?i)\b(eyJrIjoi[A-Za-z0-9]{70,400}={0,3})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 3 +keywords = ["eyjrijoi"] + +[[rules]] +id = "grafana-cloud-api-token" +description = "Found a Grafana cloud API token, risking unauthorized access to cloud-based monitoring services and data exposure." +regex = '''(?i)\b(glc_[A-Za-z0-9+/]{32,400}={0,3})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 3 +keywords = ["glc_"] + +[[rules]] +id = "grafana-service-account-token" +description = "Discovered a Grafana service account token, posing a risk of compromised monitoring services and data integrity." +regex = '''(?i)\b(glsa_[A-Za-z0-9]{32}_[A-Fa-f0-9]{8})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 3 +keywords = ["glsa_"] + +[[rules]] +id = "harness-api-key" +description = "Identified a Harness Access Token (PAT or SAT), risking unauthorized access to a Harness account." +regex = '''(?:pat|sat)\.[a-zA-Z0-9_-]{22}\.[a-zA-Z0-9]{24}\.[a-zA-Z0-9]{20}''' +keywords = [ + "pat.", + "sat.", +] + +[[rules]] +id = "hashicorp-tf-api-token" +description = "Uncovered a HashiCorp Terraform user/org API token, which may lead to unauthorized infrastructure management and security breaches." +regex = '''(?i)[a-z0-9]{14}\.(?-i:atlasv1)\.[a-z0-9\-_=]{60,70}''' +entropy = 3.5 +keywords = ["atlasv1"] + +[[rules]] +id = "hashicorp-tf-password" +description = "Identified a HashiCorp Terraform password field, risking unauthorized infrastructure configuration and security breaches." +regex = '''(?i)[\w.-]{0,50}?(?:administrator_login_password|password)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}("[a-z0-9=_\-]{8,20}")(?:[\x60'"\s;]|\\[nr]|$)''' +path = '''(?i)\.(?:tf|hcl)$''' +entropy = 2 +keywords = [ + "administrator_login_password", + "password", +] + +[[rules]] +id = "heroku-api-key" +description = "Detected a Heroku API Key, potentially compromising cloud application deployments and operational security." +regex = '''(?i)[\w.-]{0,50}?(?:heroku)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["heroku"] + +[[rules]] +id = "heroku-api-key-v2" +description = "Detected a Heroku API Key, potentially compromising cloud application deployments and operational security." +regex = '''\b((HRKU-AA[0-9a-zA-Z_-]{58}))(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 4 +keywords = ["hrku-aa"] + +[[rules]] +id = "hubspot-api-key" +description = "Found a HubSpot API Token, posing a risk to CRM data integrity and unauthorized marketing operations." +regex = '''(?i)[\w.-]{0,50}?(?:hubspot)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([0-9A-F]{8}-[0-9A-F]{4}-[0-9A-F]{4}-[0-9A-F]{4}-[0-9A-F]{12})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["hubspot"] + +[[rules]] +id = "huggingface-access-token" +description = "Discovered a Hugging Face Access token, which could lead to unauthorized access to AI models and sensitive data." +regex = '''\b(hf_(?i:[a-z]{34}))(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 2 +keywords = ["hf_"] + +[[rules]] +id = "huggingface-organization-api-token" +description = "Uncovered a Hugging Face Organization API token, potentially compromising AI organization accounts and associated data." +regex = '''\b(api_org_(?i:[a-z]{34}))(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 2 +keywords = ["api_org_"] + +[[rules]] +id = "infracost-api-token" +description = "Detected an Infracost API Token, risking unauthorized access to cloud cost estimation tools and financial data." +regex = '''\b(ico-[a-zA-Z0-9]{32})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 3 +keywords = ["ico-"] + +[[rules]] +id = "intercom-api-key" +description = "Identified an Intercom API Token, which could compromise customer communication channels and data privacy." +regex = '''(?i)[\w.-]{0,50}?(?:intercom)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9=_\-]{60})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["intercom"] + +[[rules]] +id = "intra42-client-secret" +description = "Found a Intra42 client secret, which could lead to unauthorized access to the 42School API and sensitive data." +regex = '''\b(s-s4t2(?:ud|af)-(?i)[abcdef0123456789]{64})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 3 +keywords = [ + "intra", + "s-s4t2ud-", + "s-s4t2af-", +] + +[[rules]] +id = "jfrog-api-key" +description = "Found a JFrog API Key, posing a risk of unauthorized access to software artifact repositories and build pipelines." +regex = '''(?i)[\w.-]{0,50}?(?:jfrog|artifactory|bintray|xray)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9]{73})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = [ + "jfrog", + "artifactory", + "bintray", + "xray", +] + +[[rules]] +id = "jfrog-identity-token" +description = "Discovered a JFrog Identity Token, potentially compromising access to JFrog services and sensitive software artifacts." +regex = '''(?i)[\w.-]{0,50}?(?:jfrog|artifactory|bintray|xray)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9]{64})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = [ + "jfrog", + "artifactory", + "bintray", + "xray", +] + +[[rules]] +id = "jwt" +description = "Uncovered a JSON Web Token, which may lead to unauthorized access to web applications and sensitive user data." +regex = '''\b(ey[a-zA-Z0-9]{17,}\.ey[a-zA-Z0-9\/\\_-]{17,}\.(?:[a-zA-Z0-9\/\\_-]{10,}={0,2})?)(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 3 +keywords = ["ey"] + +[[rules]] +id = "jwt-base64" +description = "Detected a Base64-encoded JSON Web Token, posing a risk of exposing encoded authentication and data exchange information." +regex = '''\bZXlK(?:(?PaGJHY2lPaU)|(?PaGNIVWlPaU)|(?PaGNIWWlPaU)|(?PaGRXUWlPaU)|(?PaU5qUWlP)|(?PamNtbDBJanBi)|(?PamRIa2lPaU)|(?PbGNHc2lPbn)|(?PbGJtTWlPaU)|(?PcWEzVWlPaU)|(?PcWQyc2lPb)|(?PcGMzTWlPaU)|(?PcGRpSTZJ)|(?PcmFXUWlP)|(?PclpYbGZiM0J6SWpwY)|(?PcmRIa2lPaUp)|(?PdWIyNWpaU0k2)|(?Pd01tTWlP)|(?Pd01uTWlPaU)|(?Pd2NIUWlPaU)|(?PemRXSWlPaU)|(?PemRuUWlP)|(?PMFlXY2lPaU)|(?PMGVYQWlPaUp)|(?PMWNtd2l)|(?PMWMyVWlPaUp)|(?PMlpYSWlPaU)|(?PMlpYSnphVzl1SWpv)|(?PNElqb2)|(?PNE5XTWlP)|(?PNE5YUWlPaU)|(?PNE5YUWpVekkxTmlJNkl)|(?PNE5YVWlPaU)|(?PNmFYQWlPaU))[a-zA-Z0-9\/\\_+\-\r\n]{40,}={0,2}''' +entropy = 2 +keywords = ["zxlk"] + +[[rules]] +id = "kraken-access-token" +description = "Identified a Kraken Access Token, potentially compromising cryptocurrency trading accounts and financial security." +regex = '''(?i)[\w.-]{0,50}?(?:kraken)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9\/=_\+\-]{80,90})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["kraken"] + +[[rules]] +id = "kubernetes-secret-yaml" +description = "Possible Kubernetes Secret detected, posing a risk of leaking credentials/tokens from your deployments" +regex = '''(?i)(?:\bkind:[ \t]*["']?\bsecret\b["']?(?s:.){0,200}?\bdata:(?s:.){0,100}?\s+([\w.-]+:(?:[ \t]*(?:\||>[-+]?)\s+)?[ \t]*(?:["']?[a-z0-9+/]{10,}={0,3}["']?|\{\{[ \t\w"|$:=,.-]+}}|""|''))|\bdata:(?s:.){0,100}?\s+([\w.-]+:(?:[ \t]*(?:\||>[-+]?)\s+)?[ \t]*(?:["']?[a-z0-9+/]{10,}={0,3}["']?|\{\{[ \t\w"|$:=,.-]+}}|""|''))(?s:.){0,200}?\bkind:[ \t]*["']?\bsecret\b["']?)''' +path = '''(?i)\.ya?ml$''' +keywords = ["secret"] +[[rules.allowlists]] +regexes = [ + '''[\w.-]+:(?:[ \t]*(?:\||>[-+]?)\s+)?[ \t]*(?:\{\{[ \t\w"|$:=,.-]+}}|""|'')''', +] +[[rules.allowlists]] +regexTarget = "match" +regexes = [ + '''(kind:(?s:.)+\n---\n(?s:.)+\bdata:|data:(?s:.)+\n---\n(?s:.)+\bkind:)''', +] + +[[rules]] +id = "kucoin-access-token" +description = "Found a Kucoin Access Token, risking unauthorized access to cryptocurrency exchange services and transactions." +regex = '''(?i)[\w.-]{0,50}?(?:kucoin)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-f0-9]{24})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["kucoin"] + +[[rules]] +id = "kucoin-secret-key" +description = "Discovered a Kucoin Secret Key, which could lead to compromised cryptocurrency operations and financial data breaches." +regex = '''(?i)[\w.-]{0,50}?(?:kucoin)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["kucoin"] + +[[rules]] +id = "launchdarkly-access-token" +description = "Uncovered a Launchdarkly Access Token, potentially compromising feature flag management and application functionality." +regex = '''(?i)[\w.-]{0,50}?(?:launchdarkly)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9=_\-]{40})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["launchdarkly"] + +[[rules]] +id = "linear-api-key" +description = "Detected a Linear API Token, posing a risk to project management tools and sensitive task data." +regex = '''lin_api_(?i)[a-z0-9]{40}''' +entropy = 2 +keywords = ["lin_api_"] + +[[rules]] +id = "linear-client-secret" +description = "Identified a Linear Client Secret, which may compromise secure integrations and sensitive project management data." +regex = '''(?i)[\w.-]{0,50}?(?:linear)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-f0-9]{32})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 2 +keywords = ["linear"] + +[[rules]] +id = "linkedin-client-id" +description = "Found a LinkedIn Client ID, risking unauthorized access to LinkedIn integrations and professional data exposure." +regex = '''(?i)[\w.-]{0,50}?(?:linked[_-]?in)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9]{14})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 2 +keywords = [ + "linkedin", + "linked_in", + "linked-in", +] + +[[rules]] +id = "linkedin-client-secret" +description = "Discovered a LinkedIn Client secret, potentially compromising LinkedIn application integrations and user data." +regex = '''(?i)[\w.-]{0,50}?(?:linked[_-]?in)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9]{16})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 2 +keywords = [ + "linkedin", + "linked_in", + "linked-in", +] + +[[rules]] +id = "lob-api-key" +description = "Uncovered a Lob API Key, which could lead to unauthorized access to mailing and address verification services." +regex = '''(?i)[\w.-]{0,50}?(?:lob)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}((live|test)_[a-f0-9]{35})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = [ + "test_", + "live_", +] + +[[rules]] +id = "lob-pub-api-key" +description = "Detected a Lob Publishable API Key, posing a risk of exposing mail and print service integrations." +regex = '''(?i)[\w.-]{0,50}?(?:lob)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}((test|live)_pub_[a-f0-9]{31})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = [ + "test_pub", + "live_pub", + "_pub", +] + +[[rules]] +id = "looker-client-id" +description = "Found a Looker Client ID, risking unauthorized access to a Looker account and exposing sensitive data." +regex = '''(?i)[\w.-]{0,50}?(?:looker)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9]{20})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["looker"] + +[[rules]] +id = "looker-client-secret" +description = "Found a Looker Client Secret, risking unauthorized access to a Looker account and exposing sensitive data." +regex = '''(?i)[\w.-]{0,50}?(?:looker)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9]{24})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["looker"] + +[[rules]] +id = "mailchimp-api-key" +description = "Identified a Mailchimp API key, potentially compromising email marketing campaigns and subscriber data." +regex = '''(?i)[\w.-]{0,50}?(?:MailchimpSDK.initialize|mailchimp)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-f0-9]{32}-us\d\d)(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["mailchimp"] + +[[rules]] +id = "mailgun-private-api-token" +description = "Found a Mailgun private API token, risking unauthorized email service operations and data breaches." +regex = '''(?i)[\w.-]{0,50}?(?:mailgun)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}(key-[a-f0-9]{32})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["mailgun"] + +[[rules]] +id = "mailgun-pub-key" +description = "Discovered a Mailgun public validation key, which could expose email verification processes and associated data." +regex = '''(?i)[\w.-]{0,50}?(?:mailgun)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}(pubkey-[a-f0-9]{32})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["mailgun"] + +[[rules]] +id = "mailgun-signing-key" +description = "Uncovered a Mailgun webhook signing key, potentially compromising email automation and data integrity." +regex = '''(?i)[\w.-]{0,50}?(?:mailgun)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-h0-9]{32}-[a-h0-9]{8}-[a-h0-9]{8})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["mailgun"] + +[[rules]] +id = "mapbox-api-token" +description = "Detected a MapBox API token, posing a risk to geospatial services and sensitive location data exposure." +regex = '''(?i)[\w.-]{0,50}?(?:mapbox)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}(pk\.[a-z0-9]{60}\.[a-z0-9]{22})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["mapbox"] + +[[rules]] +id = "mattermost-access-token" +description = "Identified a Mattermost Access Token, which may compromise team communication channels and data privacy." +regex = '''(?i)[\w.-]{0,50}?(?:mattermost)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9]{26})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["mattermost"] + +[[rules]] +id = "maxmind-license-key" +description = "Discovered a potential MaxMind license key." +regex = '''\b([A-Za-z0-9]{6}_[A-Za-z0-9]{29}_mmk)(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 4 +keywords = ["_mmk"] + +[[rules]] +id = "messagebird-api-token" +description = "Found a MessageBird API token, risking unauthorized access to communication platforms and message data." +regex = '''(?i)[\w.-]{0,50}?(?:message[_-]?bird)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9]{25})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = [ + "messagebird", + "message-bird", + "message_bird", +] + +[[rules]] +id = "messagebird-client-id" +description = "Discovered a MessageBird client ID, potentially compromising API integrations and sensitive communication data." +regex = '''(?i)[\w.-]{0,50}?(?:message[_-]?bird)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = [ + "messagebird", + "message-bird", + "message_bird", +] + +[[rules]] +id = "microsoft-teams-webhook" +description = "Uncovered a Microsoft Teams Webhook, which could lead to unauthorized access to team collaboration tools and data leaks." +regex = '''https://[a-z0-9]+\.webhook\.office\.com/webhookb2/[a-z0-9]{8}-([a-z0-9]{4}-){3}[a-z0-9]{12}@[a-z0-9]{8}-([a-z0-9]{4}-){3}[a-z0-9]{12}/IncomingWebhook/[a-z0-9]{32}/[a-z0-9]{8}-([a-z0-9]{4}-){3}[a-z0-9]{12}''' +keywords = [ + "webhook.office.com", + "webhookb2", + "incomingwebhook", +] + +[[rules]] +id = "netlify-access-token" +description = "Detected a Netlify Access Token, potentially compromising web hosting services and site management." +regex = '''(?i)[\w.-]{0,50}?(?:netlify)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9=_\-]{40,46})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["netlify"] + +[[rules]] +id = "new-relic-browser-api-token" +description = "Identified a New Relic ingest browser API token, risking unauthorized access to application performance data and analytics." +regex = '''(?i)[\w.-]{0,50}?(?:new-relic|newrelic|new_relic)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}(NRJS-[a-f0-9]{19})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["nrjs-"] + +[[rules]] +id = "new-relic-insert-key" +description = "Discovered a New Relic insight insert key, compromising data injection into the platform." +regex = '''(?i)[\w.-]{0,50}?(?:new-relic|newrelic|new_relic)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}(NRII-[a-z0-9-]{32})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["nrii-"] + +[[rules]] +id = "new-relic-user-api-id" +description = "Found a New Relic user API ID, posing a risk to application monitoring services and data integrity." +regex = '''(?i)[\w.-]{0,50}?(?:new-relic|newrelic|new_relic)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9]{64})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = [ + "new-relic", + "newrelic", + "new_relic", +] + +[[rules]] +id = "new-relic-user-api-key" +description = "Discovered a New Relic user API Key, which could lead to compromised application insights and performance monitoring." +regex = '''(?i)[\w.-]{0,50}?(?:new-relic|newrelic|new_relic)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}(NRAK-[a-z0-9]{27})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["nrak"] + +[[rules]] +id = "notion-api-token" +description = "Notion API token" +regex = '''\b(ntn_[0-9]{11}[A-Za-z0-9]{32}[A-Za-z0-9]{3})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 4 +keywords = ["ntn_"] + +[[rules]] +id = "npm-access-token" +description = "Uncovered an npm access token, potentially compromising package management and code repository access." +regex = '''(?i)\b(npm_[a-z0-9]{36})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 2 +keywords = ["npm_"] + +[[rules]] +id = "nuget-config-password" +description = "Identified a password within a Nuget config file, potentially compromising package management access." +regex = '''(?i)''' +path = '''(?i)nuget\.config$''' +entropy = 1 +keywords = ["|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9=_\-]{32})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = [ + "nytimes", + "new-york-times", + "newyorktimes", +] + +[[rules]] +id = "octopus-deploy-api-key" +description = "Discovered a potential Octopus Deploy API key, risking application deployments and operational security." +regex = '''\b(API-[A-Z0-9]{26})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 3 +keywords = ["api-"] + +[[rules]] +id = "okta-access-token" +description = "Identified an Okta Access Token, which may compromise identity management services and user authentication data." +regex = '''[\w.-]{0,50}?(?i:[\w.-]{0,50}?(?:(?-i:[Oo]kta|OKTA))(?:[ \t\w.-]{0,20})[\s'"]{0,3})(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}(00[\w=\-]{40})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 4 +keywords = ["okta"] + +[[rules]] +id = "openai-api-key" +description = "Found an OpenAI API Key, posing a risk of unauthorized access to AI services and data manipulation." +regex = '''\b(sk-(?:proj|svcacct|admin)-(?:[A-Za-z0-9_-]{74}|[A-Za-z0-9_-]{58})T3BlbkFJ(?:[A-Za-z0-9_-]{74}|[A-Za-z0-9_-]{58})\b|sk-[a-zA-Z0-9]{20}T3BlbkFJ[a-zA-Z0-9]{20})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 3 +keywords = ["t3blbkfj"] + +[[rules]] +id = "openshift-user-token" +description = "Found an OpenShift user token, potentially compromising an OpenShift/Kubernetes cluster." +regex = '''\b(sha256~[\w-]{43})(?:[^\w-]|\z)''' +entropy = 3.5 +keywords = ["sha256~"] + +[[rules]] +id = "perplexity-api-key" +description = "Detected a Perplexity API key, which could lead to unauthorized access to Perplexity AI services and data exposure." +regex = '''\b(pplx-[a-zA-Z0-9]{48})(?:[\x60'"\s;]|\\[nr]|$|\b)''' +entropy = 4 +keywords = ["pplx-"] + +[[rules]] +id = "pkcs12-file" +description = "Found a PKCS #12 file, which commonly contain bundled private keys." +path = '''(?i)(?:^|\/)[^\/]+\.p(?:12|fx)$''' + +[[rules]] +id = "plaid-api-token" +description = "Discovered a Plaid API Token, potentially compromising financial data aggregation and banking services." +regex = '''(?i)[\w.-]{0,50}?(?:plaid)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}(access-(?:sandbox|development|production)-[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["plaid"] + +[[rules]] +id = "plaid-client-id" +description = "Uncovered a Plaid Client ID, which could lead to unauthorized financial service integrations and data breaches." +regex = '''(?i)[\w.-]{0,50}?(?:plaid)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9]{24})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 3.5 +keywords = ["plaid"] + +[[rules]] +id = "plaid-secret-key" +description = "Detected a Plaid Secret key, risking unauthorized access to financial accounts and sensitive transaction data." +regex = '''(?i)[\w.-]{0,50}?(?:plaid)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9]{30})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 3.5 +keywords = ["plaid"] + +[[rules]] +id = "planetscale-api-token" +description = "Identified a PlanetScale API token, potentially compromising database management and operations." +regex = '''\b(pscale_tkn_(?i)[\w=\.-]{32,64})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 3 +keywords = ["pscale_tkn_"] + +[[rules]] +id = "planetscale-oauth-token" +description = "Found a PlanetScale OAuth token, posing a risk to database access control and sensitive data integrity." +regex = '''\b(pscale_oauth_[\w=\.-]{32,64})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 3 +keywords = ["pscale_oauth_"] + +[[rules]] +id = "planetscale-password" +description = "Discovered a PlanetScale password, which could lead to unauthorized database operations and data breaches." +regex = '''(?i)\b(pscale_pw_(?i)[\w=\.-]{32,64})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 3 +keywords = ["pscale_pw_"] + +[[rules]] +id = "postman-api-token" +description = "Uncovered a Postman API token, potentially compromising API testing and development workflows." +regex = '''\b(PMAK-(?i)[a-f0-9]{24}\-[a-f0-9]{34})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 3 +keywords = ["pmak-"] + +[[rules]] +id = "prefect-api-token" +description = "Detected a Prefect API token, risking unauthorized access to workflow management and automation services." +regex = '''\b(pnu_[a-zA-Z0-9]{36})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 2 +keywords = ["pnu_"] + +[[rules]] +id = "private-key" +description = "Identified a Private Key, which may compromise cryptographic security and sensitive data encryption." +regex = '''(?i)-----BEGIN[ A-Z0-9_-]{0,100}PRIVATE KEY(?: BLOCK)?-----[\s\S-]{64,}?KEY(?: BLOCK)?-----''' +keywords = ["-----begin"] + +[[rules]] +id = "privateai-api-token" +description = "Identified a PrivateAI Token, posing a risk of unauthorized access to AI services and data manipulation." +regex = '''[\w.-]{0,50}?(?i:[\w.-]{0,50}?(?:private[_-]?ai)(?:[ \t\w.-]{0,20})[\s'"]{0,3})(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9]{32})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 3 +keywords = [ + "privateai", + "private_ai", + "private-ai", +] + +[[rules]] +id = "pulumi-api-token" +description = "Found a Pulumi API token, posing a risk to infrastructure as code services and cloud resource management." +regex = '''\b(pul-[a-f0-9]{40})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 2 +keywords = ["pul-"] + +[[rules]] +id = "pypi-upload-token" +description = "Discovered a PyPI upload token, potentially compromising Python package distribution and repository integrity." +regex = '''pypi-AgEIcHlwaS5vcmc[\w-]{50,1000}''' +entropy = 3 +keywords = ["pypi-ageichlwas5vcmc"] + +[[rules]] +id = "rapidapi-access-token" +description = "Uncovered a RapidAPI Access Token, which could lead to unauthorized access to various APIs and data services." +regex = '''(?i)[\w.-]{0,50}?(?:rapidapi)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9_-]{50})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["rapidapi"] + +[[rules]] +id = "readme-api-token" +description = "Detected a Readme API token, risking unauthorized documentation management and content exposure." +regex = '''\b(rdme_[a-z0-9]{70})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 2 +keywords = ["rdme_"] + +[[rules]] +id = "rubygems-api-token" +description = "Identified a Rubygem API token, potentially compromising Ruby library distribution and package management." +regex = '''\b(rubygems_[a-f0-9]{48})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 2 +keywords = ["rubygems_"] + +[[rules]] +id = "scalingo-api-token" +description = "Found a Scalingo API token, posing a risk to cloud platform services and application deployment security." +regex = '''\b(tk-us-[\w-]{48})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 2 +keywords = ["tk-us-"] + +[[rules]] +id = "sendbird-access-id" +description = "Discovered a Sendbird Access ID, which could compromise chat and messaging platform integrations." +regex = '''(?i)[\w.-]{0,50}?(?:sendbird)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["sendbird"] + +[[rules]] +id = "sendbird-access-token" +description = "Uncovered a Sendbird Access Token, potentially risking unauthorized access to communication services and user data." +regex = '''(?i)[\w.-]{0,50}?(?:sendbird)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-f0-9]{40})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["sendbird"] + +[[rules]] +id = "sendgrid-api-token" +description = "Detected a SendGrid API token, posing a risk of unauthorized email service operations and data exposure." +regex = '''\b(SG\.(?i)[a-z0-9=_\-\.]{66})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 2 +keywords = ["sg."] + +[[rules]] +id = "sendinblue-api-token" +description = "Identified a Sendinblue API token, which may compromise email marketing services and subscriber data privacy." +regex = '''\b(xkeysib-[a-f0-9]{64}\-(?i)[a-z0-9]{16})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 2 +keywords = ["xkeysib-"] + +[[rules]] +id = "sentry-access-token" +description = "Found a Sentry.io Access Token (old format), risking unauthorized access to error tracking services and sensitive application data." +regex = '''(?i)[\w.-]{0,50}?(?:sentry)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-f0-9]{64})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 3 +keywords = ["sentry"] + +[[rules]] +id = "sentry-org-token" +description = "Found a Sentry.io Organization Token, risking unauthorized access to error tracking services and sensitive application data." +regex = '''\bsntrys_eyJpYXQiO[a-zA-Z0-9+/]{10,200}(?:LCJyZWdpb25fdXJs|InJlZ2lvbl91cmwi|cmVnaW9uX3VybCI6)[a-zA-Z0-9+/]{10,200}={0,2}_[a-zA-Z0-9+/]{43}(?:[^a-zA-Z0-9+/]|\z)''' +entropy = 4.5 +keywords = ["sntrys_eyjpyxqio"] + +[[rules]] +id = "sentry-user-token" +description = "Found a Sentry.io User Token, risking unauthorized access to error tracking services and sensitive application data." +regex = '''\b(sntryu_[a-f0-9]{64})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 3.5 +keywords = ["sntryu_"] + +[[rules]] +id = "settlemint-application-access-token" +description = "Found a Settlemint Application Access Token." +regex = '''\b(sm_aat_[a-zA-Z0-9]{16})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 3 +keywords = ["sm_aat"] + +[[rules]] +id = "settlemint-personal-access-token" +description = "Found a Settlemint Personal Access Token." +regex = '''\b(sm_pat_[a-zA-Z0-9]{16})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 3 +keywords = ["sm_pat"] + +[[rules]] +id = "settlemint-service-access-token" +description = "Found a Settlemint Service Access Token." +regex = '''\b(sm_sat_[a-zA-Z0-9]{16})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 3 +keywords = ["sm_sat"] + +[[rules]] +id = "shippo-api-token" +description = "Discovered a Shippo API token, potentially compromising shipping services and customer order data." +regex = '''\b(shippo_(?:live|test)_[a-fA-F0-9]{40})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 2 +keywords = ["shippo_"] + +[[rules]] +id = "shopify-access-token" +description = "Uncovered a Shopify access token, which could lead to unauthorized e-commerce platform access and data breaches." +regex = '''shpat_[a-fA-F0-9]{32}''' +entropy = 2 +keywords = ["shpat_"] + +[[rules]] +id = "shopify-custom-access-token" +description = "Detected a Shopify custom access token, potentially compromising custom app integrations and e-commerce data security." +regex = '''shpca_[a-fA-F0-9]{32}''' +entropy = 2 +keywords = ["shpca_"] + +[[rules]] +id = "shopify-private-app-access-token" +description = "Identified a Shopify private app access token, risking unauthorized access to private app data and store operations." +regex = '''shppa_[a-fA-F0-9]{32}''' +entropy = 2 +keywords = ["shppa_"] + +[[rules]] +id = "shopify-shared-secret" +description = "Found a Shopify shared secret, posing a risk to application authentication and e-commerce platform security." +regex = '''shpss_[a-fA-F0-9]{32}''' +entropy = 2 +keywords = ["shpss_"] + +[[rules]] +id = "sidekiq-secret" +description = "Discovered a Sidekiq Secret, which could lead to compromised background job processing and application data breaches." +regex = '''(?i)[\w.-]{0,50}?(?:BUNDLE_ENTERPRISE__CONTRIBSYS__COM|BUNDLE_GEMS__CONTRIBSYS__COM)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-f0-9]{8}:[a-f0-9]{8})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = [ + "bundle_enterprise__contribsys__com", + "bundle_gems__contribsys__com", +] + +[[rules]] +id = "sidekiq-sensitive-url" +description = "Uncovered a Sidekiq Sensitive URL, potentially exposing internal job queues and sensitive operation details." +regex = '''(?i)\bhttps?://([a-f0-9]{8}:[a-f0-9]{8})@(?:gems.contribsys.com|enterprise.contribsys.com)(?:[\/|\#|\?|:]|$)''' +keywords = [ + "gems.contribsys.com", + "enterprise.contribsys.com", +] + +[[rules]] +id = "slack-app-token" +description = "Detected a Slack App-level token, risking unauthorized access to Slack applications and workspace data." +regex = '''(?i)xapp-\d-[A-Z0-9]+-\d+-[a-z0-9]+''' +entropy = 2 +keywords = ["xapp"] + +[[rules]] +id = "slack-bot-token" +description = "Identified a Slack Bot token, which may compromise bot integrations and communication channel security." +regex = '''xoxb-[0-9]{10,13}-[0-9]{10,13}[a-zA-Z0-9-]*''' +entropy = 3 +keywords = ["xoxb"] + +[[rules]] +id = "slack-config-access-token" +description = "Found a Slack Configuration access token, posing a risk to workspace configuration and sensitive data access." +regex = '''(?i)xoxe.xox[bp]-\d-[A-Z0-9]{163,166}''' +entropy = 2 +keywords = [ + "xoxe.xoxb-", + "xoxe.xoxp-", +] + +[[rules]] +id = "slack-config-refresh-token" +description = "Discovered a Slack Configuration refresh token, potentially allowing prolonged unauthorized access to configuration settings." +regex = '''(?i)xoxe-\d-[A-Z0-9]{146}''' +entropy = 2 +keywords = ["xoxe-"] + +[[rules]] +id = "slack-legacy-bot-token" +description = "Uncovered a Slack Legacy bot token, which could lead to compromised legacy bot operations and data exposure." +regex = '''xoxb-[0-9]{8,14}-[a-zA-Z0-9]{18,26}''' +entropy = 2 +keywords = ["xoxb"] + +[[rules]] +id = "slack-legacy-token" +description = "Detected a Slack Legacy token, risking unauthorized access to older Slack integrations and user data." +regex = '''xox[os]-\d+-\d+-\d+-[a-fA-F\d]+''' +entropy = 2 +keywords = [ + "xoxo", + "xoxs", +] + +[[rules]] +id = "slack-legacy-workspace-token" +description = "Identified a Slack Legacy Workspace token, potentially compromising access to workspace data and legacy features." +regex = '''xox[ar]-(?:\d-)?[0-9a-zA-Z]{8,48}''' +entropy = 2 +keywords = [ + "xoxa", + "xoxr", +] + +[[rules]] +id = "slack-user-token" +description = "Found a Slack User token, posing a risk of unauthorized user impersonation and data access within Slack workspaces." +regex = '''xox[pe](?:-[0-9]{10,13}){3}-[a-zA-Z0-9-]{28,34}''' +entropy = 2 +keywords = [ + "xoxp-", + "xoxe-", +] + +[[rules]] +id = "slack-webhook-url" +description = "Discovered a Slack Webhook, which could lead to unauthorized message posting and data leakage in Slack channels." +regex = '''(?:https?://)?hooks.slack.com/(?:services|workflows|triggers)/[A-Za-z0-9+/]{43,56}''' +keywords = ["hooks.slack.com"] + +[[rules]] +id = "snyk-api-token" +description = "Uncovered a Snyk API token, potentially compromising software vulnerability scanning and code security." +regex = '''(?i)[\w.-]{0,50}?(?:snyk[_.-]?(?:(?:api|oauth)[_.-]?)?(?:key|token))(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["snyk"] + +[[rules]] +id = "sonar-api-token" +description = "Uncovered a Sonar API token, potentially compromising software vulnerability scanning and code security." +regex = '''(?i)[\w.-]{0,50}?(?:sonar[_.-]?(login|token))(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}((?:squ_|sqp_|sqa_)?[a-z0-9=_\-]{40})(?:[\x60'"\s;]|\\[nr]|$)''' +secretGroup = 2 +keywords = ["sonar"] + +[[rules]] +id = "sourcegraph-access-token" +description = "Sourcegraph is a code search and navigation engine." +regex = '''(?i)\b(\b(sgp_(?:[a-fA-F0-9]{16}|local)_[a-fA-F0-9]{40}|sgp_[a-fA-F0-9]{40}|[a-fA-F0-9]{40})\b)(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 3 +keywords = [ + "sgp_", + "sourcegraph", +] + +[[rules]] +id = "square-access-token" +description = "Detected a Square Access Token, risking unauthorized payment processing and financial transaction exposure." +regex = '''\b((?:EAAA|sq0atp-)[\w-]{22,60})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 2 +keywords = [ + "sq0atp-", + "eaaa", +] + +[[rules]] +id = "squarespace-access-token" +description = "Identified a Squarespace Access Token, which may compromise website management and content control on Squarespace." +regex = '''(?i)[\w.-]{0,50}?(?:squarespace)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["squarespace"] + +[[rules]] +id = "stripe-access-token" +description = "Found a Stripe Access Token, posing a risk to payment processing services and sensitive financial data." +regex = '''\b((?:sk|rk)_(?:test|live|prod)_[a-zA-Z0-9]{10,99})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 2 +keywords = [ + "sk_test", + "sk_live", + "sk_prod", + "rk_test", + "rk_live", + "rk_prod", +] + +[[rules]] +id = "sumologic-access-id" +description = "Discovered a SumoLogic Access ID, potentially compromising log management services and data analytics integrity." +regex = '''[\w.-]{0,50}?(?i:[\w.-]{0,50}?(?:(?-i:[Ss]umo|SUMO))(?:[ \t\w.-]{0,20})[\s'"]{0,3})(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}(su[a-zA-Z0-9]{12})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 3 +keywords = ["sumo"] + +[[rules]] +id = "sumologic-access-token" +description = "Uncovered a SumoLogic Access Token, which could lead to unauthorized access to log data and analytics insights." +regex = '''(?i)[\w.-]{0,50}?(?:(?-i:[Ss]umo|SUMO))(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9]{64})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 3 +keywords = ["sumo"] + +[[rules]] +id = "telegram-bot-api-token" +description = "Detected a Telegram Bot API Token, risking unauthorized bot operations and message interception on Telegram." +regex = '''(?i)[\w.-]{0,50}?(?:telegr)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([0-9]{5,16}:(?-i:A)[a-z0-9_\-]{34})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["telegr"] + +[[rules]] +id = "travisci-access-token" +description = "Identified a Travis CI Access Token, potentially compromising continuous integration services and codebase security." +regex = '''(?i)[\w.-]{0,50}?(?:travis)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9]{22})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["travis"] + +[[rules]] +id = "twilio-api-key" +description = "Found a Twilio API Key, posing a risk to communication services and sensitive customer interaction data." +regex = '''SK[0-9a-fA-F]{32}''' +entropy = 3 +keywords = ["sk"] + +[[rules]] +id = "twitch-api-token" +description = "Discovered a Twitch API token, which could compromise streaming services and account integrations." +regex = '''(?i)[\w.-]{0,50}?(?:twitch)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9]{30})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["twitch"] + +[[rules]] +id = "twitter-access-secret" +description = "Uncovered a Twitter Access Secret, potentially risking unauthorized Twitter integrations and data breaches." +regex = '''(?i)[\w.-]{0,50}?(?:twitter)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9]{45})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["twitter"] + +[[rules]] +id = "twitter-access-token" +description = "Detected a Twitter Access Token, posing a risk of unauthorized account operations and social media data exposure." +regex = '''(?i)[\w.-]{0,50}?(?:twitter)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([0-9]{15,25}-[a-zA-Z0-9]{20,40})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["twitter"] + +[[rules]] +id = "twitter-api-key" +description = "Identified a Twitter API Key, which may compromise Twitter application integrations and user data security." +regex = '''(?i)[\w.-]{0,50}?(?:twitter)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9]{25})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["twitter"] + +[[rules]] +id = "twitter-api-secret" +description = "Found a Twitter API Secret, risking the security of Twitter app integrations and sensitive data access." +regex = '''(?i)[\w.-]{0,50}?(?:twitter)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9]{50})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["twitter"] + +[[rules]] +id = "twitter-bearer-token" +description = "Discovered a Twitter Bearer Token, potentially compromising API access and data retrieval from Twitter." +regex = '''(?i)[\w.-]{0,50}?(?:twitter)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}(A{22}[a-zA-Z0-9%]{80,100})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["twitter"] + +[[rules]] +id = "typeform-api-token" +description = "Uncovered a Typeform API token, which could lead to unauthorized survey management and data collection." +regex = '''(?i)[\w.-]{0,50}?(?:typeform)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}(tfp_[a-z0-9\-_\.=]{59})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["tfp_"] + +[[rules]] +id = "vault-batch-token" +description = "Detected a Vault Batch Token, risking unauthorized access to secret management services and sensitive data." +regex = '''\b(hvb\.[\w-]{138,300})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 4 +keywords = ["hvb."] + +[[rules]] +id = "vault-service-token" +description = "Identified a Vault Service Token, potentially compromising infrastructure security and access to sensitive credentials." +regex = '''\b((?:hvs\.[\w-]{90,120}|s\.(?i:[a-z0-9]{24})))(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 3.5 +keywords = [ + "hvs.", + "s.", +] +[[rules.allowlists]] +regexes = [ + '''s\.[A-Za-z]{24}''', +] + +[[rules]] +id = "yandex-access-token" +description = "Found a Yandex Access Token, posing a risk to Yandex service integrations and user data privacy." +regex = '''(?i)[\w.-]{0,50}?(?:yandex)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}(t1\.[A-Z0-9a-z_-]+[=]{0,2}\.[A-Z0-9a-z_-]{86}[=]{0,2})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["yandex"] + +[[rules]] +id = "yandex-api-key" +description = "Discovered a Yandex API Key, which could lead to unauthorized access to Yandex services and data manipulation." +regex = '''(?i)[\w.-]{0,50}?(?:yandex)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}(AQVN[A-Za-z0-9_\-]{35,38})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["yandex"] + +[[rules]] +id = "yandex-aws-access-token" +description = "Uncovered a Yandex AWS Access Token, potentially compromising cloud resource access and data security on Yandex Cloud." +regex = '''(?i)[\w.-]{0,50}?(?:yandex)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}(YC[a-zA-Z0-9_\-]{38})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["yandex"] + +[[rules]] +id = "zendesk-secret-key" +description = "Detected a Zendesk Secret Key, risking unauthorized access to customer support services and sensitive ticketing data." +regex = '''(?i)[\w.-]{0,50}?(?:zendesk)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9]{40})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["zendesk"] + diff --git a/crates/aprender-review-experiment/examples/replay.rs b/crates/aprender-review-experiment/examples/replay.rs new file mode 100644 index 0000000000..4fd5c5503e --- /dev/null +++ b/crates/aprender-review-experiment/examples/replay.rs @@ -0,0 +1,358 @@ +//! `cargo run -p aprender-review-experiment --example replay -- ` +//! +//! PRM-S1: the `review-replay-v1` speed benchmark (contract `review-replay-v1`, +//! library `replay.rs`). An example, not a `[[bin]]` (see `rex.rs`). +//! +//! Commands: +//! - `build --version V --candidates F --tokenize-url U --diffs-out DIR --out SET +//! [--per-stratum N]` F is JSONL `{"group":"owner/repo#N","diff":"path"}`, one +//! line per quorum diff (inputs only). Each diff is composed with the review +//! prompt, templated by `U/apply-template` and counted by `U/tokenize` (a +//! `llama-server` on the replay GGUF). The sealed test manifest and its sketch +//! file are required. Writes the set and `DIR/.diff` for the +//! chosen items, and prints counts and the set sha only. +//! - `run --set SET --diffs DIR --url U --engine apr|llama_cpp --engine-version X +//! --apr-tag T --cell C --gguf-sha G --model M --out ROWS [--server-pid P]` +//! replays every item once through the chat-completions endpoint `U` and +//! appends one `review-replay-receipt-v1` row per item. `--server-pid` adds +//! the serve's VmHWM (peak RSS so far) to each row. +//! - `sketch --items DIR` writes `test-sketch-v1.txt` (the near-dup sketches +//! `build` needs) from the corpus items in DIR, e.g. the unpacked +//! `items-v1.tar`. Every test-split diff must hash to its corpus +//! `diff_sha256`, or nothing is written. Prints counts only. +//! - `voters DIR` prints `lane seconds` for every +//! `predicate.consultations..duration_seconds` in `DIR/*/*/receipt.intoto.jsonl` +//! (quorum receipts under `evidence/pr-review/`). +//! - `summary --rows F --voter LANE=FILE... [--prev F]` the §9 speed block as +//! JSON (valid YAML). A voter FILE holds one latency in seconds per line. +//! Exit 1 when the run is refused, and the error names the reason. + +use std::collections::BTreeMap; +use std::io::Write as _; +use std::process::ExitCode; + +use aprender_review_experiment::cluster::{parse_sketches, render_sketches, sketch}; +use aprender_review_experiment::contamination::Index; +use aprender_review_experiment::corpus::{parse_manifest, sha256_hex, Item, Split}; +use aprender_review_experiment::harness::{classify, compose, post, request_body}; +use aprender_review_experiment::prereg::PROMPT_V1; +use aprender_review_experiment::replay::{ + build, summarize, BuildError, Candidate, Engine, Row, Set, PER_STRATUM, ROW_SCHEMA, +}; +use serde_json::Value; + +const SEED: u64 = 4354; +const CORPUS_DIR: &str = "docs/audits/review-corpus"; + +type Flags = BTreeMap>; + +fn main() -> ExitCode { + let a: Vec = std::env::args().skip(1).collect(); + let r = match a.first().map(String::as_str) { + Some("build") => flags(&a[1..]).and_then(|f| cmd_build(&f)), + Some("run") => flags(&a[1..]).and_then(|f| cmd_run(&f)), + Some("sketch") => flags(&a[1..]).and_then(|f| cmd_sketch(&f)), + Some("voters") => a + .get(1) + .ok_or_else(|| "voters DIR".to_string()) + .and_then(|d| cmd_voters(d)), + Some("summary") => flags(&a[1..]).and_then(|f| cmd_summary(&f)), + _ => Err("usage: replay build|run|sketch|voters|summary (see the example's docs)".into()), + }; + match r { + Ok(()) => ExitCode::SUCCESS, + Err(e) => { + eprintln!("replay: {e}"); + ExitCode::FAILURE + } + } +} + +/// `--key value` pairs; a key may repeat. +fn flags(a: &[String]) -> Result { + let mut f = Flags::new(); + let mut it = a.iter(); + while let Some(k) = it.next() { + let k = k + .strip_prefix("--") + .ok_or_else(|| format!("expected --flag, got {k:?}"))?; + let v = it.next().ok_or_else(|| format!("--{k} needs a value"))?; + f.entry(k.to_string()).or_default().push(v.clone()); + } + Ok(f) +} + +fn need<'a>(f: &'a Flags, k: &str) -> Result<&'a str, String> { + f.get(k) + .and_then(|v| v.last()) + .map(String::as_str) + .ok_or_else(|| format!("--{k} is required")) +} + +fn read(path: &str) -> Result { + std::fs::read_to_string(path).map_err(|e| format!("{path}: {e}")) +} + +fn sealed_index() -> Result { + let manifest = parse_manifest(&read(&format!("{CORPUS_DIR}/test-manifest-v1.txt"))?) + .ok_or("test manifest does not parse")?; + let sketches = parse_sketches(&read(&format!("{CORPUS_DIR}/test-sketch-v1.txt"))?) + .ok_or("test sketch file does not parse")?; + Ok(Index::new(&manifest).with_sketches(sketches)) +} + +fn post_json(url: &str, body: &Value) -> Result { + let reply = post(url, body)?; + if reply.status != 200 { + return Err(format!("{url}: HTTP {}", reply.status)); + } + serde_json::from_str(&reply.body).map_err(|e| format!("{url}: {e}")) +} + +/// Composed-prompt tokens under the serve's chat template. +fn count_tokens(base: &str, diff: &str) -> Result { + let msgs = Value::Array(vec![Value::Object( + [ + ("role".to_string(), Value::from("user")), + ("content".to_string(), Value::from(compose(PROMPT_V1, diff))), + ] + .into_iter() + .collect(), + )]); + let body = Value::Object([("messages".to_string(), msgs)].into_iter().collect()); + let templated = post_json(&format!("{base}/apply-template"), &body)?; + let prompt = templated["prompt"] + .as_str() + .ok_or("apply-template: no prompt")?; + let body = Value::Object( + [ + ("content".to_string(), Value::from(prompt)), + ("add_special".to_string(), Value::from(true)), + ] + .into_iter() + .collect(), + ); + let toks = post_json(&format!("{base}/tokenize"), &body)?; + toks["tokens"] + .as_array() + .map(|t| t.len() as u64) + .ok_or_else(|| "tokenize: no tokens".to_string()) +} + +fn cmd_build(f: &Flags) -> Result<(), String> { + let index = sealed_index()?; + let base = need(f, "tokenize-url")?.trim_end_matches('/'); + let per = match f.get("per-stratum") { + Some(_) => need(f, "per-stratum")? + .parse() + .map_err(|e| format!("--per-stratum: {e}"))?, + None => PER_STRATUM, + }; + let mut cands = Vec::new(); + for (n, line) in read(need(f, "candidates")?)?.lines().enumerate() { + let v: Value = + serde_json::from_str(line).map_err(|e| format!("candidates:{}: {e}", n + 1))?; + let (Some(group), Some(path)) = (v["group"].as_str(), v["diff"].as_str()) else { + return Err(format!("candidates:{}: needs group and diff", n + 1)); + }; + let diff = read(path)?; + cands.push(Candidate { + group: group.to_string(), + diff_sha256: sha256_hex(diff.as_bytes()), + input_tokens: count_tokens(base, &diff)?, + diff, + }); + } + let set = match build(need(f, "version")?, SEED, per, &cands, &index) { + Ok(s) => s, + Err(BuildError::EmptySealedIndex) => return Err("sealed test manifest is empty".into()), + Err(BuildError::Short(s)) => { + let s: Vec = s + .iter() + .map(|x| format!("{} {}/{}", x.stratum.as_str(), x.have, x.need)) + .collect(); + return Err(format!("short strata (never padded): {}", s.join(", "))); + } + }; + let dir = need(f, "diffs-out")?; + std::fs::create_dir_all(dir).map_err(|e| format!("{dir}: {e}"))?; + for it in &set.items { + let c = cands + .iter() + .find(|c| c.diff_sha256 == it.diff_sha256) + .ok_or("lost a diff")?; + let p = format!("{dir}/{}.diff", it.diff_sha256); + std::fs::write(&p, &c.diff).map_err(|e| format!("{p}: {e}"))?; + } + let out = need(f, "out")?; + std::fs::write(out, set.render()).map_err(|e| format!("{out}: {e}"))?; + println!( + "{{\"candidates\":{},\"items\":{},\"set_sha\":\"{}\"}}", + cands.len(), + set.items.len(), + set.sha() + ); + Ok(()) +} + +fn vm_hwm_mb(pid: &str) -> Option { + let s = std::fs::read_to_string(format!("/proc/{pid}/status")).ok()?; + let kb: f64 = s + .lines() + .find_map(|l| l.strip_prefix("VmHWM:"))? + .trim() + .trim_end_matches("kB") + .trim() + .parse() + .ok()?; + Some(kb / 1024.0) +} + +fn cmd_run(f: &Flags) -> Result<(), String> { + let text = read(need(f, "set")?)?; + let set = Set::parse(&text)?; + let set_sha = sha256_hex(text.as_bytes()); + let engine = match need(f, "engine")? { + "apr" => Engine::Apr, + "llama_cpp" => Engine::LlamaCpp, + e => return Err(format!("--engine {e}: want apr|llama_cpp")), + }; + let (url, model, dir) = (need(f, "url")?, need(f, "model")?, need(f, "diffs")?); + let pid = f.get("server-pid").and_then(|v| v.last()); + let out_path = need(f, "out")?; + let mut out = std::fs::OpenOptions::new() + .create(true) + .append(true) + .open(out_path) + .map_err(|e| format!("{out_path}: {e}"))?; + for it in &set.items { + let diff = read(&format!("{dir}/{}.diff", it.diff_sha256))?; + if sha256_hex(diff.as_bytes()) != it.diff_sha256 { + return Err(format!( + "{dir}/{}.diff does not hash to its name", + it.diff_sha256 + )); + } + let reply = post(url, &request_body(model, PROMPT_V1, &diff))?; + let p = classify(&reply); + let row = Row { + schema: ROW_SCHEMA.into(), + replay_version: set.version.clone(), + set_sha: set_sha.clone(), + diff_sha256: it.diff_sha256.clone(), + stratum: it.stratum, + engine, + engine_version: need(f, "engine-version")?.into(), + apr_tag: need(f, "apr-tag")?.into(), + cell: need(f, "cell")?.into(), + gguf_sha256: need(f, "gguf-sha")?.into(), + wall_ms: reply.wall_ms, + prompt_ms: p.server.map(|s| s.prompt_ms), + prompt_tps: p.server.and_then(|s| s.prompt_per_second), + decode_tps: p.server.and_then(|s| s.predicted_per_second), + input_tokens: p.tokens.map(|t| t.prompt), + output_tokens: p.tokens.map(|t| t.completion), + peak_rss_mb: pid.and_then(|p| vm_hwm_mb(p)), + verdict: p.verdict, + }; + let line = serde_json::to_string(&row).map_err(|e| e.to_string())?; + writeln!(out, "{line}").map_err(|e| format!("{out_path}: {e}"))?; + } + println!("{{\"rows\":{},\"set_sha\":\"{set_sha}\"}}", set.items.len()); + Ok(()) +} + +fn cmd_voters(dir: &str) -> Result<(), String> { + let mut n = 0; + for pr in std::fs::read_dir(dir) + .map_err(|e| format!("{dir}: {e}"))? + .flatten() + { + for head in std::fs::read_dir(pr.path()).into_iter().flatten().flatten() { + let Ok(text) = std::fs::read_to_string(head.path().join("receipt.intoto.jsonl")) else { + continue; + }; + for line in text.lines() { + let Ok(v) = serde_json::from_str::(line) else { + continue; + }; + let Some(c) = v["predicate"]["consultations"].as_object() else { + continue; + }; + for (lane, body) in c { + if let Some(s) = body["duration_seconds"].as_f64() { + println!("{lane} {s}"); + n += 1; + } + } + } + } + } + if n == 0 { + return Err(format!("{dir}: no consultation latencies")); + } + Ok(()) +} + +fn rows_of(path: &str) -> Result, String> { + read(path)? + .lines() + .enumerate() + .map(|(i, l)| serde_json::from_str(l).map_err(|e| format!("{path}:{}: {e}", i + 1))) + .collect() +} + +fn cmd_summary(f: &Flags) -> Result<(), String> { + let rows = rows_of(need(f, "rows")?)?; + let prev = match f.get("prev") { + Some(_) => Some(rows_of(need(f, "prev")?)?), + None => None, + }; + let mut voters = Vec::new(); + for v in f.get("voter").into_iter().flatten() { + let (lane, path) = v + .split_once('=') + .ok_or_else(|| format!("--voter {v}: want LANE=FILE"))?; + let xs = read(path)? + .split_whitespace() + .map(|x| x.parse::().map_err(|e| format!("{path}: {x}: {e}"))) + .collect::, _>>()?; + voters.push((lane.to_string(), xs)); + } + let speed = + summarize(&rows, &voters, prev.as_deref(), SEED).map_err(|e| format!("refused: {e:?}"))?; + println!( + "{}", + serde_json::to_string_pretty(&speed).map_err(|e| e.to_string())? + ); + Ok(()) +} + +/// `sketch`: the test items' near-dup sketches, written only when every +/// test-split diff in DIR is byte-identical to the one the corpus sealed. +fn cmd_sketch(f: &Flags) -> Result<(), String> { + let dir = need(f, "items")?; + let mut out: Vec<(String, Vec)> = Vec::new(); + let mut tests = 0usize; + for line in read(&format!("{CORPUS_DIR}/corpus-v1.jsonl"))?.lines() { + let item: Item = serde_json::from_str(line).map_err(|e| e.to_string())?; + if item.split != Split::Test { + continue; + } + tests += 1; + let diff = read(&format!("{dir}/{}.diff", item.id))?; + if sha256_hex(diff.as_bytes()) != item.diff_sha256 { + return Err(format!("{}: diff sha differs from the corpus", item.id)); + } + if let Some(s) = sketch(&diff) { + out.push((item.id, s)); + } + } + if tests == 0 { + return Err("the corpus has no test items".into()); + } + let path = format!("{CORPUS_DIR}/test-sketch-v1.txt"); + std::fs::write(&path, render_sketches(&out)).map_err(|e| format!("{path}: {e}"))?; + println!("test_items {tests} sketched {}", out.len()); + Ok(()) +} diff --git a/crates/aprender-review-experiment/examples/rex.rs b/crates/aprender-review-experiment/examples/rex.rs new file mode 100644 index 0000000000..a95d800964 --- /dev/null +++ b/crates/aprender-review-experiment/examples/rex.rs @@ -0,0 +1,1114 @@ +//! `cargo run -p aprender-review-experiment --example rex -- ` +//! +//! An example, not a `[[bin]]`: the workspace binary register only shrinks +//! (monorepo_invariants::test_no_unauthorized_binaries). +//! +//! Commands: +//! - `prereg` print the prereg lock the tree implies +//! - `prereg-check` exit 1 unless the committed lock matches the tree +//! - `corpus-build PRS_JSON CUTOFF ITEMS_DIR MUTANTS_JSON...` +//! build corpus v1 (REX-02): writes `ITEMS_DIR/.diff`, and in the repo +//! `docs/audits/review-corpus/corpus-v1.jsonl` + `test-manifest-v1.txt` + +//! `test-sketch-v1.txt` (PRA-001 T7 near-dup sketches of the test items). +//! Prints counts only; it never prints a test item. +//! - `review --url U --cell C --host H --backend B --arm A --model-id M +//! --weights-sha W --apr-tag T --apr-sha S --items DIR --out DIR --split dev|test +//! [--limit N] [--rerun] [--cold-first]` +//! REX-03: drive a resident `apr serve`; append receipts to +//! `OUT/receipts.jsonl` and raw outputs under `OUT/raw/`. Prints a count line. +//! - `not-run --why NoDeclaredExecutor|Refused (identity flags as review)` +//! write explicit `NotRun` receipts for a cell that cannot run. +//! - `score --receipts F --cell C --arm A --split dev|test [--rerun] +//! [--pubkey P]` the §2.3 metrics as JSON. Without `--pubkey` the result +//! is labelled `unsigned` (exploratory); with it, `minisign -V` must pass. +//! `--pilot` (with `--split dev`) adds the REX-05 projection: warm/cold +//! wall-clock spread, projected test runtime, and the §2.2 sample-size rule. +//! - `admit --cell C|all --why NoDeclaredExecutor --model-id M --weights-sha W +//! --apr-tag T --apr-sha S --out FILE` append `NotRun` admission rows (REX-04); +//! `admit --cell C --removed-by R ...` appends a `Refused` row; +//! `admit --cell C --parity-receipt R --threshold T --threshold-basis B +//! --min-positions N ...` appends an `Admitted` row derived from the receipt +//! (FALSIFY-RCA-004..006), or writes nothing, exits 1 and says why. +//! - `admission-check --file F` every §2.1 cell resolved exactly once; prints the +//! summary JSON. Exit 1 if inadmissible, 10 if admissible but S-7 (no cell admitted). +//! - `ledger REPO OUT_JSONL QUORUM_RECEIPT...` REX-07: write `review-ledger-v2` +//! rows from quorum receipts and print shadow coverage. Exit 10 unless every +//! receipt carries an uncounted shadow row that leaves the width alone. +//! - `ladder REPORT_JSON [--current RUNG]` PRM-09: the lane's next rung +//! (shadow/tripwire/tie-breaker/vote) from a `prm-001-report-v1` report, one +//! rung from `--current` (default shadow), with every reason it stopped below +//! vote. Exit 0 whatever the rung, 2 on usage; the gates read `.mode`. +//! - `ratchet record --file F --tag T --cell C --receipts R [--llama-receipts L +//! --llama-cell LC]` REX-10: append one `review-lane-perf-ratchet-v1` entry +//! from the tag's warm, non-rerun receipts on the cell (and llama.cpp's p95 on +//! the same items). A tag with no timings is refused: it did not run. +//! - `ratchet check --file F` the andon JSON (arming/green/red). Exit 10 on RED: +//! a paired Harrell–Davis p95 rise over the best earlier tag, or a violation. +//! - `challenge --ledger F --test-version V --id ID --tier B1a --weights-sha W +//! --prompt-sha P [--adapter-sha A] --initial-weights-sha W0 --initial-prompt-sha P0 +//! --cell C --pubkey K --challenger-receipts R --challenger-arm A +//! --champion-receipts R --champion-arm A [--h1 holds|fails] [--h2 holds|fails] +//! [--contamination-hits N] [--p95-s X] [--budget-p95-s Y]` REX-11: the §5.4 +//! decision on the sealed test split, appended to the +//! `review-champion-challenger-v1` ledger. Both receipt files must verify under +//! `--pubkey` (unsigned data cannot promote, R-1); an omitted gate fails. +//! Exit 0 promoted, 10 rejected, 11 refused (no evaluation spent). +//! - `b2 status [--refusals F] [--accepted verb=receipt,...]` REX-12: one +//! `review-b2-loop-v1` row per B2 tier (ready, verb_refused, unledgered). +//! Exit 11 while any row is NotRun. +//! - `b2 teacher-receipt --logits F --teacher-sha W --k K` the teacher-logit +//! dataset receipt (sha, count, 0 test hashes) or every reason it is refused. +//! - `lane-kappa ROWS --split val [--shadow qwen-shadow] (--min-n N | --manifest M +//! --candidate ROWS2)` PRM-C13 (was PRA-001 T13) (contract lane-independence-v1): κ_err of the +//! shadow lane against every counted lane on matured gold rows of the split, as +//! JSON; a thin pair prints `insufficient`, never a number. With `--candidate` +//! the gate runs H7 (δ-free, `stats::h7_holds`) on the candidate rows: the +//! voters and min_n come only from the manifest's `lane_independence` block. Exit 12 (andon, S-14) on a refused candidate, an unusable manifest, +//! or `--candidate` without `--manifest`. +//! - `workload ROWS --min-n N` PRM-S2 (contract workload-profile-v1): per-lane +//! input/output token p50/p95 over agent-trace-v1 rows, §0.2 decided per lane +//! and the replay strata weights over distinct diffs, as the receipt's +//! `workload` block. A falsified §0.2 is a measured result (exit 0); an +//! undecided one (no lane at min_n) is exit 13 (andon, S-14). + +use aprender_review_experiment::b2; +use aprender_review_experiment::build_corpus::{ + choose, g_candidates, is_green, p_candidates, r_candidates, seal, Mutant, Pr, PER_CLASS, +}; +use aprender_review_experiment::champion; +use aprender_review_experiment::cluster::{parse_sketches, render_sketches, sketch}; +use aprender_review_experiment::contamination::Index; +use aprender_review_experiment::corpus::{ + assign_splits, corpus_version, parse_manifest, render_manifest, sha256_hex, Item, Split, +}; +use aprender_review_experiment::harness::{ + classify, post, request_body, rerun_subset, run_order, utc_now, Run, +}; +use aprender_review_experiment::pilot::{project, Projection}; +use aprender_review_experiment::prereg; +use aprender_review_experiment::ratchet; +use aprender_review_experiment::receipt::{admissible, Arm, Expect, NotRun, Receipt}; +use aprender_review_experiment::score::{collect, score, Scored}; +use std::collections::BTreeMap; +use std::process::ExitCode; + +/// Seed for every draw (the epic number, analysis plan §Seeds). +const SEED: u64 = 4354; +const CORPUS_DIR: &str = "docs/audits/review-corpus"; + +fn main() -> ExitCode { + let args: Vec = std::env::args().skip(1).collect(); + match args.first().map(String::as_str) { + Some("prereg") => prereg_cmd(false), + Some("prereg-check") => prereg_cmd(true), + Some("admission-check") => admission_check(&args[1..]), + Some("ledger") => ledger_cmd(&args[1..]), + Some("ladder") => ladder_cmd(&args[1..]), + Some("ratchet") => ratchet_cmd(&args[1..]), + Some("challenge") => challenge_cmd(&args[1..]), + Some("b2") => b2_cmd(&args[1..]), + Some("lane-kappa") => lane_kappa_cmd(&args[1..]), + Some("workload") => workload_cmd(&args[1..]), + Some(c @ ("review" | "not-run" | "score" | "admit")) => { + match flags(&args[1..]).and_then(|f| match c { + "review" => review(&f), + "not-run" => not_run(&f), + "admit" => admit(&f), + _ => score_cmd(&f), + }) { + Ok(()) => ExitCode::SUCCESS, + Err(e) => { + eprintln!("rex {c}: {e}"); + ExitCode::from(1) + } + } + } + Some("corpus-build") if args.len() >= 5 => match corpus_build(&args[1..]) { + Ok(()) => ExitCode::SUCCESS, + Err(e) => { + eprintln!("rex corpus-build: {e}"); + ExitCode::from(1) + } + }, + _ => { + eprintln!( + "usage: rex (see the example docs)" + ); + ExitCode::from(2) + } + } +} + +fn prereg_cmd(check: bool) -> ExitCode { + let Some(c) = prereg::Components::in_tree() else { + eprintln!("rex: spec lacks a §2..§6 span"); + return ExitCode::from(2); + }; + if !check { + print!("{}", c.render_lock()); + return ExitCode::SUCCESS; + } + let bad = prereg::verify(prereg::LOCK, &c); + if bad.is_empty() { + println!("{} OK prereg_sha={}", prereg::SCHEME, c.prereg_sha()); + return ExitCode::SUCCESS; + } + for b in &bad { + eprintln!("{} DRIFT {b}", prereg::SCHEME); + } + ExitCode::from(1) +} + +fn read_json(path: &str) -> Result { + let text = std::fs::read_to_string(path).map_err(|e| format!("{path}: {e}"))?; + serde_json::from_str(&text).map_err(|e| format!("{path}: {e}")) +} + +fn corpus_build(a: &[String]) -> Result<(), String> { + let (prs_path, cutoff, items_dir) = (&a[0], &a[1], &a[2]); + let prs: Vec = read_json(prs_path)?; + let mut mutants: Vec = Vec::new(); + for m in &a[3..] { + mutants.extend(read_json::>(m)?); + } + let base = String::from_utf8_lossy( + &std::process::Command::new("git") + .args(["rev-parse", "--short=9", "HEAD"]) + .output() + .map_err(|e| e.to_string())? + .stdout, + ) + .trim() + .to_string(); + + let r = choose(r_candidates(&prs), PER_CLASS, SEED); + let g_pool: Vec<_> = choose(g_candidates(&prs, cutoff), PER_CLASS * 2, SEED) + .into_iter() + .filter(|(i, _)| i.id[4..].parse().is_ok_and(is_green)) + .collect(); + let g = choose(g_pool, PER_CLASS, SEED); + let p = p_candidates(&mutants, &base, PER_CLASS, SEED); + + let mut all: Vec<(Item, String)> = p.into_iter().chain(r).chain(g).collect(); + let mut items: Vec = all.iter().map(|(i, _)| i.clone()).collect(); + assign_splits(&mut items, SEED); + let split: BTreeMap = items.iter().map(|i| (i.id.clone(), i.split)).collect(); + for (i, _) in &mut all { + i.split = split[&i.id]; + } + all.sort_by(|x, y| x.0.id.cmp(&y.0.id)); + write_outputs(&all, items_dir) +} + +fn write_outputs(all: &[(Item, String)], items_dir: &str) -> Result<(), String> { + std::fs::create_dir_all(items_dir).map_err(|e| e.to_string())?; + let mut jsonl = String::new(); + let mut counts: BTreeMap = BTreeMap::new(); + for (i, d) in all { + std::fs::write(format!("{items_dir}/{}.diff", i.id), d).map_err(|e| e.to_string())?; + jsonl.push_str(&serde_json::to_string(i).map_err(|e| e.to_string())?); + jsonl.push('\n'); + *counts + .entry(format!("{:?} {:?} {:?}", i.class, i.stratum, i.split)) + .or_default() += 1; + } + let manifest = render_manifest(&seal(all)); + let sketches: Vec<(String, Vec)> = all + .iter() + .filter(|(i, _)| i.split == Split::Test) + .filter_map(|(i, d)| Some((i.id.clone(), sketch(d)?))) + .collect(); + std::fs::write( + format!("{CORPUS_DIR}/test-sketch-v1.txt"), + render_sketches(&sketches), + ) + .map_err(|e| e.to_string())?; + std::fs::write(format!("{CORPUS_DIR}/corpus-v1.jsonl"), &jsonl).map_err(|e| e.to_string())?; + std::fs::write(format!("{CORPUS_DIR}/test-manifest-v1.txt"), &manifest) + .map_err(|e| e.to_string())?; + for (k, n) in &counts { + println!("{k} {n}"); + } + println!( + "items {} corpus_version {}", + all.len(), + corpus_version(&manifest) + ); + Ok(()) +} + +type Flags = BTreeMap; + +/// `--key value` pairs; a flag followed by another flag (or nothing) is `"1"`. +fn flags(a: &[String]) -> Result { + let mut f = Flags::new(); + let mut i = 0; + while i < a.len() { + let k = a[i] + .strip_prefix("--") + .ok_or_else(|| format!("expected --flag, got {:?}", a[i]))?; + match a.get(i + 1).filter(|v| !v.starts_with("--")) { + Some(v) => { + f.insert(k.into(), v.clone()); + i += 2; + } + None => { + f.insert(k.into(), "1".into()); + i += 1; + } + } + } + Ok(f) +} + +fn need<'a>(f: &'a Flags, k: &str) -> Result<&'a str, String> { + f.get(k) + .map(String::as_str) + .ok_or_else(|| format!("--{k} is required")) +} + +fn split_of(f: &Flags) -> Result { + match need(f, "split")? { + "dev" => Ok(Split::Dev), + "test" => Ok(Split::Test), + s => Err(format!("--split {s}: dev or test")), + } +} + +fn arm_of(s: &str) -> Result { + serde_json::from_value(serde_json::Value::String(s.into())) + .map_err(|_| format!("--arm {s}: apr-4b|apr-9b|haiku|agy")) +} + +/// The corpus items and the analysis identity (prereg sha, corpus version). +fn corpus() -> Result<(Vec, String, String), String> { + let jsonl = std::fs::read_to_string(format!("{CORPUS_DIR}/corpus-v1.jsonl")) + .map_err(|e| e.to_string())?; + let items = jsonl + .lines() + .map(|l| serde_json::from_str(l).map_err(|e| e.to_string())) + .collect::, _>>()?; + let manifest = std::fs::read_to_string(format!("{CORPUS_DIR}/test-manifest-v1.txt")) + .map_err(|e| e.to_string())?; + let prereg_sha = prereg::locked_prereg_sha() + .ok_or("no locked prereg sha")? + .to_string(); + Ok((items, prereg_sha, corpus_version(&manifest))) +} + +fn run_of(f: &Flags, prereg_sha: String, corpus_version: String) -> Result { + Ok(Run { + cell: need(f, "cell")?.into(), + host: need(f, "host")?.into(), + backend: need(f, "backend")?.into(), + arm: arm_of(need(f, "arm")?)?, + apr_tag: need(f, "apr-tag")?.into(), + apr_sha256: need(f, "apr-sha")?.into(), + model_id: need(f, "model-id")?.into(), + weights_sha256: need(f, "weights-sha")?.into(), + prompt: prereg::PROMPT_V1.into(), + corpus_version, + prereg_sha, + }) +} + +/// Items of the split in the seeded run order (the rerun subset with --rerun). +fn ordered(items: &[Item], split: Split, rerun: bool) -> Vec<&Item> { + let by_id: BTreeMap<&str, &Item> = items + .iter() + .filter(|i| i.split == split) + .map(|i| (i.id.as_str(), i)) + .collect(); + let ids: Vec = by_id.keys().map(|k| (*k).to_string()).collect(); + let mut order = run_order(&ids, SEED); + if rerun { + order = rerun_subset(&order); + } + order.iter().map(|id| by_id[id.as_str()]).collect() +} + +fn append(path: &str, r: &Receipt) -> Result<(), String> { + use std::io::Write; + let mut f = std::fs::OpenOptions::new() + .create(true) + .append(true) + .open(path) + .map_err(|e| e.to_string())?; + writeln!( + f, + "{}", + serde_json::to_string(r).map_err(|e| e.to_string())? + ) + .map_err(|e| e.to_string()) +} + +fn review(f: &Flags) -> Result<(), String> { + let (items, prereg_sha, cv) = corpus()?; + let run = run_of(f, prereg_sha, cv)?; + let (url, items_dir, out) = (need(f, "url")?, need(f, "items")?, need(f, "out")?); + let rerun = f.contains_key("rerun"); + let mut todo = ordered(&items, split_of(f)?, rerun); + if let Some(n) = f.get("limit") { + todo.truncate(n.parse().map_err(|_| "--limit N")?); + } + let raw_dir = format!("raw/{}/{}", run.cell, need(f, "arm")?); + std::fs::create_dir_all(format!("{out}/{raw_dir}")).map_err(|e| e.to_string())?; + let mut tally: BTreeMap = BTreeMap::new(); + for (n, item) in todo.iter().enumerate() { + let diff = std::fs::read_to_string(format!("{items_dir}/{}.diff", item.id)) + .map_err(|e| format!("{}: {e}", item.id))?; + if sha256_hex(diff.as_bytes()) != item.diff_sha256 { + return Err(format!( + "{}: diff sha differs from the corpus (wrong items tar?)", + item.id + )); + } + let body = request_body(&run.model_id, &run.prompt, &diff); + let when = utc_now(); + let reply = post(url, &body)?; + let parsed = classify(&reply); + let raw = format!( + "{raw_dir}/{}{}.txt", + item.id, + if rerun { ".rerun" } else { "" } + ); + std::fs::write(format!("{out}/{raw}"), &parsed.text).map_err(|e| e.to_string())?; + let mut r = run.receipt(item, &body, &parsed, reply.wall_ms, &raw, &when); + r.cold = n == 0 && f.contains_key("cold-first"); + r.rerun = rerun; + append(&format!("{out}/receipts.jsonl"), &r)?; + *tally.entry(format!("{:?}", r.verdict)).or_default() += 1; + } + println!("reviewed {} {tally:?}", todo.len()); + Ok(()) +} + +fn not_run(f: &Flags) -> Result<(), String> { + let (items, prereg_sha, cv) = corpus()?; + let run = run_of(f, prereg_sha, cv)?; + let why: NotRun = serde_json::from_value(serde_json::Value::String(need(f, "why")?.into())) + .map_err(|_| "--why NoDeclaredExecutor|Refused|ContextOverflow|ServeError")?; + let out = need(f, "out")?; + std::fs::create_dir_all(out).map_err(|e| e.to_string())?; + let when = utc_now(); + let todo = ordered(&items, split_of(f)?, false); + for item in &todo { + append( + &format!("{out}/receipts.jsonl"), + &run.not_run(item, why, &when), + )?; + } + println!("not-run {} {why:?}", todo.len()); + Ok(()) +} + +/// What `score` prints. +#[derive(serde::Serialize)] +struct Report<'a> { + cell: &'a str, + arm: Arm, + provenance: &'a str, + prereg_sha: &'a str, + corpus_version: &'a str, + rejected_rows: usize, + score: aprender_review_experiment::score::LaneScore, + #[serde(skip_serializing_if = "Option::is_none")] + pilot: Option, +} + +fn score_cmd(f: &Flags) -> Result<(), String> { + let (items, prereg_sha, cv) = corpus()?; + let path = need(f, "receipts")?; + let signed = match f.get("pubkey") { + Some(pk) => { + verify_signed(path, pk)?; + "signed" + } + None => "unsigned (exploratory)", + }; + let (cell, arm, rerun) = ( + need(f, "cell")?.to_string(), + arm_of(need(f, "arm")?)?, + f.contains_key("rerun"), + ); + let expected = ordered(&items, split_of(f)?, rerun); + let lines = std::fs::read_to_string(path).map_err(|e| e.to_string())?; + let root = std::path::Path::new(path) + .parent() + .map(std::path::Path::to_path_buf) + .unwrap_or_default(); + let expect = Expect { + prereg_sha: &prereg_sha, + corpus_version: &cv, + }; + let (rows, rejected) = collect( + &lines, + expect, + &expected, + |r| r.cell == cell && r.arm == arm && r.rerun == rerun, + |p| std::fs::read_to_string(root.join(p)).ok(), + ); + let sc = score(&rows); + let pilot = if f.contains_key("pilot") { + if split_of(f)? != Split::Dev { + return Err("--pilot projects from dev items: pass --split dev".into()); + } + let ran: std::collections::BTreeSet<&str> = rows + .iter() + .filter(|s| s.verdict.executed()) + .map(|s| s.id.as_str()) + .collect(); + let (mut warm, mut cold) = (Vec::new(), Vec::new()); + for r in lines.lines().filter_map(|l| admissible(l, expect).ok()) { + let mine = r.cell == cell && r.arm == arm && !r.rerun; + if let (true, Some(t)) = (mine && ran.contains(r.item_id.as_str()), r.timings) { + if r.cold { + cold.push(t.wall_ms); + } else { + warm.push(t.wall_ms); + } + } + } + let test: Vec<&Item> = items.iter().filter(|i| i.split == Split::Test).collect(); + let defects = test.iter().filter(|i| i.class.is_defect()).count() as u64; + Some(project(sc.recall, &warm, &cold, defects, test.len() as u64)) + } else { + None + }; + let out = Report { + cell: &cell, + arm, + provenance: signed, + prereg_sha: &prereg_sha, + corpus_version: &cv, + rejected_rows: rejected.len(), + score: sc, + pilot, + }; + println!( + "{}", + serde_json::to_string_pretty(&out).map_err(|e| e.to_string())? + ); + for r in rejected.iter().take(5) { + eprintln!("rejected line {}: {}", r.line, r.reasons.join("; ")); + } + Ok(()) +} + +/// REX-04: append admission rows for one cell or all six. +fn admit(f: &Flags) -> Result<(), String> { + use aprender_review_experiment::admission::{ + parity_from_receipt, Expect, Row, Status, CELLS, SCHEME, + }; + let prereg_sha = prereg::locked_prereg_sha() + .ok_or("no locked prereg sha")? + .to_string(); + let which = need(f, "cell")?; + let status = match (f.get("parity-receipt"), f.get("removed-by")) { + (Some(path), _) => { + if which == "all" { + return Err("--parity-receipt admits one --cell, not all".into()); + } + let num = |k: &str| -> Result { + need(f, k)? + .parse() + .map_err(|_| format!("--{k}: not a number")) + }; + let bytes = std::fs::read(path).map_err(|e| format!("{path}: {e}"))?; + let x = Expect { + apr_sha256: need(f, "apr-sha")?, + weights_sha256: need(f, "weights-sha")?, + threshold: num("threshold")?, + threshold_basis: need(f, "threshold-basis")?, + min_positions: num("min-positions")? as usize, + }; + let parity = parity_from_receipt(&bytes, &x) + .map_err(|e| format!("{which}: {path} does not admit: {}", e.join("; ")))?; + Status::Admitted { parity } + } + (None, Some(r)) => Status::Refused { + removed_by: r.clone(), + }, + (None, None) => Status::NotRun { + reason: serde_json::from_value(serde_json::Value::String(need(f, "why")?.into())) + .map_err(|_| "--why NoDeclaredExecutor|ServeError|…")?, + }, + }; + let cells: Vec<_> = CELLS + .iter() + .filter(|c| which == "all" || c.cell == which) + .collect(); + if cells.is_empty() { + return Err(format!("--cell {which}: not a §2.1 cell")); + } + let out = need(f, "out")?; + let at = utc_now(); + let mut text = String::new(); + for c in &cells { + let row = Row { + schema: SCHEME.into(), + cell: c.cell.into(), + host: c.host.into(), + backend: c.backend.into(), + apr_tag: need(f, "apr-tag")?.into(), + apr_sha256: need(f, "apr-sha")?.into(), + model_id: need(f, "model-id")?.into(), + weights_sha256: need(f, "weights-sha")?.into(), + prereg_sha: prereg_sha.clone(), + at: at.clone(), + status: status.clone(), + }; + text += &(serde_json::to_string(&row).map_err(|e| e.to_string())? + "\n"); + } + use std::io::Write; + std::fs::OpenOptions::new() + .create(true) + .append(true) + .open(out) + .and_then(|mut h| h.write_all(text.as_bytes())) + .map_err(|e| e.to_string())?; + println!("admit {} row(s)", cells.len()); + Ok(()) +} + +/// REX-04: resolve an admission file. Exit 10 (an alarm, never a crash code) +/// when it is admissible but no cell is admitted (S-7). +fn admission_check(a: &[String]) -> ExitCode { + let run = || -> Result> { + let f = flags(a).map_err(|e| vec![e])?; + let path = need(&f, "file").map_err(|e| vec![e])?; + let text = std::fs::read_to_string(path).map_err(|e| vec![e.to_string()])?; + let lock = + prereg::locked_prereg_sha().ok_or_else(|| vec!["no locked prereg sha".into()])?; + aprender_review_experiment::admission::check(&text, lock) + }; + match run() { + Ok(s) => { + match serde_json::to_string(&s) { + Ok(j) => println!("{j}"), + Err(e) => eprintln!("rex admission-check: {e}"), + } + if s.s7 { + eprintln!("S-7: no cell is admitted — no admissible cell for (A)"); + ExitCode::from(10) + } else { + ExitCode::SUCCESS + } + } + Err(es) => { + for e in es { + eprintln!("rex admission-check: {e}"); + } + ExitCode::from(1) + } + } +} + +fn ledger_cmd(a: &[String]) -> ExitCode { + let (Some(repo), Some(out), receipts) = (a.first(), a.get(1), a.get(2..).unwrap_or(&[])) else { + eprintln!("usage: rex ledger REPO OUT_JSONL QUORUM_RECEIPT..."); + return ExitCode::from(2); + }; + let mut read = Vec::new(); + for p in receipts { + match std::fs::read_to_string(p) { + Ok(t) => read.push((p.clone(), t)), + Err(e) => { + eprintln!("rex ledger: {p}: {e}"); + return ExitCode::from(1); + } + } + } + let (rows, c) = aprender_review_experiment::ledger::build(&read, repo); + let lines: Result = rows + .iter() + .map(|r| serde_json::to_string(r).map(|j| j + "\n")) + .collect(); + match lines + .map_err(|e| e.to_string()) + .and_then(|l| std::fs::write(out, l).map_err(|e| format!("{out}: {e}"))) + { + Ok(()) => {} + Err(e) => { + eprintln!("rex ledger: {e}"); + return ExitCode::from(1); + } + } + match serde_json::to_string(&c) { + Ok(j) => println!("{j}"), + Err(e) => eprintln!("rex ledger: {e}"), + } + if c.holds() { + ExitCode::SUCCESS + } else { + eprintln!( + "REX-07: {} of {} quorums carry a shadow row, {} violations", + c.carried, + c.quorums, + c.violations.len() + ); + ExitCode::from(10) + } +} + +fn ladder_cmd(a: &[String]) -> ExitCode { + use aprender_review_experiment::ladder::{decide, Mode}; + let usage = || { + eprintln!("usage: rex ladder REPORT_JSON [--current shadow|tripwire|tie-breaker|vote]"); + ExitCode::from(2) + }; + let (path, current) = match a { + [p] => (p, Mode::Shadow), + [p, f, r] if f == "--current" => { + match serde_json::from_value(serde_json::Value::String(r.clone())) { + Ok(m) => (p, m), + Err(_) => return usage(), + } + } + _ => return usage(), + }; + let Some(lock) = prereg::locked_prereg_sha() else { + eprintln!("rex ladder: no locked prereg sha"); + return ExitCode::from(1); + }; + let text = match std::fs::read_to_string(path) { + Ok(t) => t, + Err(e) => { + eprintln!("rex ladder: {path}: {e}"); + return ExitCode::from(1); + } + }; + let d = decide(&text, lock, current); + match serde_json::to_string(&d) { + Ok(j) => println!("{j}"), + Err(e) => eprintln!("rex ladder: {e}"), + } + ExitCode::SUCCESS +} + +/// Exit code for a lane-independence andon (S-14). +const ANDON_KAPPA: u8 = 12; + +fn lane_kappa_cmd(a: &[String]) -> ExitCode { + use aprender_review_experiment::kappa_probe::{gate, parse_manifest, parse_rows, probe}; + const USAGE: &str = "usage: rex lane-kappa ROWS --split val [--shadow L] (--min-n N | --manifest M --candidate ROWS2)"; + let (Some(path), Ok(f)) = (a.first(), flags(a.get(1..).unwrap_or_default())) else { + eprintln!("{USAGE}"); + return ExitCode::from(2); + }; + let Some(split) = f.get("split") else { + eprintln!("{USAGE}"); + return ExitCode::from(2); + }; + let shadow = f.get("shadow").map_or("qwen-shadow", String::as_str); + let load = |p: &str| { + std::fs::read_to_string(p) + .map_err(|e| format!("{p}: {e}")) + .and_then(|t| parse_rows(&t).map_err(|e| format!("{p}: {e}"))) + }; + let andon = |why: String| { + eprintln!("rex lane-kappa: ANDON {why}"); + ExitCode::from(ANDON_KAPPA) + }; + let manifest = match f.get("manifest").map(|m| { + std::fs::read_to_string(m) + .map_err(|e| format!("{m}: {e}")) + .and_then(|t| parse_manifest(&t)) + }) { + None => None, + Some(Ok(m)) => Some(m), + Some(Err(e)) => return andon(format!("S-14 manifest: {e}")), + }; + if f.contains_key("candidate") && manifest.is_none() { + return andon( + "S-14 --candidate needs --manifest: the voters and min_n are not pre-registered".into(), + ); + } + let min_n = match (&manifest, f.get("min-n").map(|n| n.parse::())) { + (Some(m), _) => m.min_n, + (None, Some(Ok(n))) if n > 0 => n, + _ => { + eprintln!("{USAGE}"); + return ExitCode::from(2); + } + }; + let baseline = match load(path) { + Ok(r) => probe(&r, split, shadow, min_n), + Err(e) => { + eprintln!("rex lane-kappa: {e}"); + return ExitCode::from(1); + } + }; + let mut out = serde_json::Map::new(); + out.insert("baseline".into(), to_json(&baseline)); + let mut code = ExitCode::SUCCESS; + if let (Some(m), Some(c)) = (&manifest, f.get("candidate")) { + let rows = match load(c) { + Ok(r) => r, + Err(e) => { + eprintln!("rex lane-kappa: {e}"); + return ExitCode::from(1); + } + }; + let candidate = probe(&rows, split, shadow, min_n); + let v = gate(&rows, split, shadow, m); + if !v.pass { + eprintln!("rex lane-kappa: ANDON {}", v.refusals.join("; ")); + code = ExitCode::from(ANDON_KAPPA); + } + out.insert("candidate".into(), to_json(&candidate)); + out.insert("manifest".into(), to_json(m)); + out.insert("gate".into(), to_json(&v)); + } + println!("{}", serde_json::Value::Object(out)); + code +} + +fn to_json(x: &T) -> serde_json::Value { + serde_json::to_value(x).unwrap_or(serde_json::Value::Null) +} + +/// Warm, non-rerun, admissible timings of one cell (and tag, when given). +fn warm_timings( + path: &str, + cell: &str, + tag: Option<&str>, +) -> Result<(Vec, String), String> { + let (_, prereg_sha, cv) = corpus()?; + let text = std::fs::read_to_string(path).map_err(|e| format!("{path}: {e}"))?; + let expect = Expect { + prereg_sha: &prereg_sha, + corpus_version: &cv, + }; + let rows = text + .lines() + .filter_map(|l| admissible(l, expect).ok()) + .filter(|r| { + r.cell == cell + && tag.is_none_or(|t| r.apr_tag == t) + && !r.cold + && !r.rerun + && r.timings.is_some() + }) + .collect(); + Ok((rows, sha256_hex(text.as_bytes()))) +} + +fn samples(rows: &[Receipt]) -> Vec> { + rows.iter() + .filter_map(|r| { + r.timings.map(|t| ratchet::Sample { + item_id: &r.item_id, + apr_sha256: &r.apr_sha256, + wall_ms: t.wall_ms, + }) + }) + .collect() +} + +fn ratchet_entries(file: &str) -> Result, String> { + match std::fs::read_to_string(file) { + Ok(t) => t + .lines() + .filter(|l| !l.trim().is_empty()) + .map(|l| serde_json::from_str(l).map_err(|e| format!("{file}: {e}"))) + .collect(), + Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(Vec::new()), + Err(e) => Err(format!("{file}: {e}")), + } +} + +fn ratchet_record(f: &Flags) -> Result<(), String> { + let (file, tag, cell) = (need(f, "file")?, need(f, "tag")?, need(f, "cell")?); + let (rows, sha) = warm_timings(need(f, "receipts")?, cell, Some(tag))?; + let llama = match f.get("llama-receipts") { + Some(p) => warm_timings(p, need(f, "llama-cell")?, None)?.0, + None => Vec::new(), + }; + let entry = ratchet::record(tag, cell, &samples(&rows), &samples(&llama), &sha)?; + let mut all = ratchet_entries(file)?; + all.push(entry.clone()); + let line = serde_json::to_string(&entry).map_err(|e| e.to_string())?; + let mut out = std::fs::OpenOptions::new() + .create(true) + .append(true) + .open(file) + .map_err(|e| format!("{file}: {e}"))?; + std::io::Write::write_all(&mut out, format!("{line}\n").as_bytes()) + .map_err(|e| e.to_string())?; + println!( + "recorded {tag} on {cell}: {} items, p95 {:.0} ms; andon {:?}", + entry.items.len(), + entry.p95_ms, + ratchet::check(&all).andon + ); + Ok(()) +} + +fn ratchet_cmd(a: &[String]) -> ExitCode { + let run = |sub: &str| -> Result { + let f = flags(&a[1..])?; + if sub == "record" { + return ratchet_record(&f).map(|()| ExitCode::SUCCESS); + } + let v = ratchet::check(&ratchet_entries(need(&f, "file")?)?); + println!( + "{}", + serde_json::to_string_pretty(&v).map_err(|e| e.to_string())? + ); + Ok(if v.andon == ratchet::Andon::Red { + ExitCode::from(10) + } else { + ExitCode::SUCCESS + }) + }; + match a.first().map(String::as_str) { + Some(s @ ("record" | "check")) => run(s).unwrap_or_else(|e| { + eprintln!("rex ratchet {s}: {e}"); + ExitCode::from(1) + }), + _ => { + eprintln!("usage: rex ratchet record|check --file F ... (see the example docs)"); + ExitCode::from(2) + } + } +} + +/// REX-12: `b2 status` (exit 11 while any B2 row is NotRun) and +/// `b2 teacher-receipt`. +fn b2_cmd(a: &[String]) -> ExitCode { + let run = |sub: &str| -> Result { + let f = flags(&a[1..])?; + if sub == "status" { + let path = f + .get("refusals") + .map_or("evidence/verbs/refusals.json", String::as_str); + let text = std::fs::read_to_string(path).map_err(|e| format!("{path}: {e}"))?; + let ledger = b2::parse_ledger(&text).map_err(|e| format!("{path}: {e}"))?; + let mut accepted = BTreeMap::new(); + for kv in f.get("accepted").map_or("", String::as_str).split(',') { + if kv.is_empty() { + continue; + } + let (verb, receipt) = kv + .split_once('=') + .ok_or_else(|| format!("--accepted {kv}: verb=receipt"))?; + accepted.insert(verb.to_string(), receipt.to_string()); + } + let rows = b2::status(&ledger, &accepted)?; + for r in &rows { + println!("{}", serde_json::to_string(r).map_err(|e| e.to_string())?); + } + let ready = rows + .iter() + .all(|r| matches!(r.state, b2::State::Ready { .. })); + return Ok(if ready { + ExitCode::SUCCESS + } else { + ExitCode::from(11) + }); + } + let (items, _, _) = corpus()?; + let manifest = std::fs::read_to_string(format!("{CORPUS_DIR}/test-manifest-v1.txt")) + .map_err(|e| e.to_string())?; + let sealed = parse_manifest(&manifest).ok_or("test manifest does not parse")?; + let index = match std::fs::read_to_string(format!("{CORPUS_DIR}/test-sketch-v1.txt")) { + Ok(t) => Index::new(&sealed) + .with_sketches(parse_sketches(&t).ok_or("test sketch file does not parse")?), + Err(_) => { + eprintln!( + "no test-sketch-v1.txt: exact-hash contamination only (no cluster check)" + ); + Index::new(&sealed) + } + }; + let path = need(&f, "logits")?; + let jsonl = std::fs::read_to_string(path).map_err(|e| format!("{path}: {e}"))?; + let k = need(&f, "k")?.parse().map_err(|e| format!("--k: {e}"))?; + match b2::teacher_receipt(&jsonl, need(&f, "teacher-sha")?, k, &items, &index) { + Ok(r) => { + println!( + "{}", + serde_json::to_string_pretty(&r).map_err(|e| e.to_string())? + ); + Ok(ExitCode::SUCCESS) + } + Err(e) => Err(e.join("\n")), + } + }; + match a.first().map(String::as_str) { + Some(s @ ("status" | "teacher-receipt")) => run(s).unwrap_or_else(|e| { + eprintln!("rex b2 {s}: {e}"); + ExitCode::from(1) + }), + _ => { + eprintln!("usage: rex b2 status|teacher-receipt ... (see the example docs)"); + ExitCode::from(2) + } + } +} + +/// `minisign -V` must pass on `path` under `pk`. +fn verify_signed(path: &str, pk: &str) -> Result<(), String> { + let ok = std::process::Command::new("minisign") + .args(["-Vqm", path, "-p", pk]) + .status() + .map_err(|e| format!("minisign: {e}"))? + .success(); + if ok { + Ok(()) + } else { + Err(format!("{path}: signature does not verify under {pk}")) + } +} + +/// One lane's scored rows on the sealed test split (first runs only). +fn test_rows(path: &str, pk: &str, cell: &str, arm: Arm) -> Result, String> { + verify_signed(path, pk)?; + let (items, prereg_sha, cv) = corpus()?; + let lines = std::fs::read_to_string(path).map_err(|e| format!("{path}: {e}"))?; + let root = std::path::Path::new(path) + .parent() + .map(std::path::Path::to_path_buf) + .unwrap_or_default(); + let expect = Expect { + prereg_sha: &prereg_sha, + corpus_version: &cv, + }; + let (rows, rejected) = collect( + &lines, + expect, + &ordered(&items, Split::Test, false), + |r| r.cell == cell && r.arm == arm && !r.rerun, + |p| std::fs::read_to_string(root.join(p)).ok(), + ); + if !rejected.is_empty() { + eprintln!("{path}: {} rejected rows", rejected.len()); + } + Ok(rows) +} + +fn opt_bool(f: &Flags, k: &str) -> Result, String> { + match f.get(k).map(String::as_str) { + None => Ok(None), + Some("holds") => Ok(Some(true)), + Some("fails") => Ok(Some(false)), + Some(v) => Err(format!("--{k} {v}: holds|fails")), + } +} + +fn opt_num(f: &Flags, k: &str) -> Result, String> { + f.get(k) + .map(|v| v.parse().map_err(|_| format!("--{k} {v}: not a number"))) + .transpose() +} + +fn challenge(f: &Flags) -> Result { + let file = need(f, "ledger")?; + let ledger: Vec = match std::fs::read_to_string(file) { + Ok(t) => t + .lines() + .filter(|l| !l.trim().is_empty()) + .map(|l| serde_json::from_str(l).map_err(|e| format!("{file}: {e}"))) + .collect::>()?, + Err(e) if e.kind() == std::io::ErrorKind::NotFound => Vec::new(), + Err(e) => return Err(format!("{file}: {e}")), + }; + let pin = |w: &str, p: &str, a: &str| -> Result { + Ok(champion::Pin { + weights_sha256: need(f, w)?.into(), + prompt_sha256: need(f, p)?.into(), + adapter_sha256: f.get(a).cloned(), + }) + }; + let initial = pin( + "initial-weights-sha", + "initial-prompt-sha", + "initial-adapter-sha", + )?; + let (pk, cell) = (need(f, "pubkey")?, need(f, "cell")?); + let ch = test_rows( + need(f, "challenger-receipts")?, + pk, + cell, + arm_of(need(f, "challenger-arm")?)?, + )?; + let cm = test_rows( + need(f, "champion-receipts")?, + pk, + cell, + arm_of(need(f, "champion-arm")?)?, + )?; + let e = champion::Evidence { + id: need(f, "id")?, + tier: need(f, "tier")?, + pin: pin("weights-sha", "prompt-sha", "adapter-sha")?, + test_version: need(f, "test-version")?, + challenger: &ch, + champion: &cm, + h1_holds: opt_bool(f, "h1")?, + h2_holds: opt_bool(f, "h2")?, + contamination_hits: opt_num(f, "contamination-hits")?, + p95_s: opt_num(f, "p95-s")?, + queue_budget_p95_s: opt_num(f, "budget-p95-s")?, + }; + let d = champion::decide(&ledger, &initial, &e); + let line = serde_json::to_string(&d).map_err(|e| e.to_string())?; + let mut out = std::fs::OpenOptions::new() + .create(true) + .append(true) + .open(file) + .map_err(|e| format!("{file}: {e}"))?; + std::io::Write::write_all(&mut out, format!("{line}\n").as_bytes()) + .map_err(|e| e.to_string())?; + println!( + "{}", + serde_json::to_string_pretty(&d).map_err(|e| e.to_string())? + ); + Ok(d.outcome) +} + +fn challenge_cmd(a: &[String]) -> ExitCode { + match flags(a).and_then(|f| challenge(&f)) { + Ok(champion::Outcome::Promoted) => ExitCode::SUCCESS, + Ok(champion::Outcome::Rejected) => ExitCode::from(10), + Ok(champion::Outcome::Refused) => ExitCode::from(11), + Err(e) => { + eprintln!("rex challenge: {e}"); + ExitCode::from(1) + } + } +} + +const ANDON_WORKLOAD: u8 = 13; + +fn workload_cmd(a: &[String]) -> ExitCode { + use aprender_review_experiment::workload::{profile, render}; + const USAGE: &str = "usage: rex workload ROWS --min-n N"; + let (Some(path), Ok(f)) = (a.first(), flags(a.get(1..).unwrap_or_default())) else { + eprintln!("{USAGE}"); + return ExitCode::from(2); + }; + let Some(Ok(min_n)) = f.get("min-n").map(|n| n.parse::()) else { + eprintln!("{USAGE}"); + return ExitCode::from(2); + }; + if min_n == 0 { + eprintln!("{USAGE}"); + return ExitCode::from(2); + } + let rows = match std::fs::read_to_string(path) { + Ok(t) => t, + Err(e) => { + eprintln!("rex workload: {path}: {e}"); + return ExitCode::from(1); + } + }; + let p = profile(&rows, min_n); + println!("{}", render(&p)); + if p.claim.is_none() { + eprintln!("rex workload: ANDON {}", p.refusals.join("; ")); + return ExitCode::from(ANDON_WORKLOAD); + } + ExitCode::SUCCESS +} diff --git a/crates/aprender-review-experiment/src/admission.rs b/crates/aprender-review-experiment/src/admission.rs new file mode 100644 index 0000000000..2ac743618e --- /dev/null +++ b/crates/aprender-review-experiment/src/admission.rs @@ -0,0 +1,338 @@ +//! REX-04 cell admission (`rex-cell-admission-v1`). +//! +//! Every §2.1 cell gets exactly one row: `Admitted` with a parity receipt, +//! `Refused{removed_by}`, or `NotRun{…}`. A cell with no row is silent, and a +//! silent cell makes the whole admission file inadmissible. When no cell is +//! `Admitted`, the summary raises S-7. + +use crate::receipt::{is_hex64, NotRun}; +use serde::{Deserialize, Serialize}; +use std::collections::BTreeMap; + +pub const SCHEME: &str = "rex-cell-admission-v1"; + +/// The §7 REX-04 oracle `[X]`. The released `apr parity` (v0.69.3) compares +/// GPU with CPU only; a llama.cpp comparison is recorded under this name. +pub const LLAMA_CPP: &str = "llama.cpp@d1d3c3396"; + +/// The interim oracle of §4.1 rule 1 until llama.cpp is a declared input: the +/// released `apr parity` GPU-vs-CPU comparison (plus H1 in the analysis). +pub const APR_GPU_CPU: &str = "apr-parity-gpu-cpu"; + +/// One §2.1 device cell. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct Cell { + pub cell: &'static str, + pub host: &'static str, + pub backend: &'static str, +} + +/// The six §2.1 cells, in table order. +pub const CELLS: [Cell; 6] = [ + Cell { + cell: "C1", + host: "intel", + backend: "wgpu", + }, + Cell { + cell: "C2", + host: "intel", + backend: "cpu", + }, + Cell { + cell: "C3", + host: "lambda-labs", + backend: "cpu", + }, + Cell { + cell: "C4", + host: "gx10", + backend: "cuda", + }, + Cell { + cell: "C5a", + host: "mini", + backend: "cpu", + }, + Cell { + cell: "C5b", + host: "mini", + backend: "metal", + }, +]; + +/// A parity measurement against the oracle on the exact weights. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Parity { + pub oracle: String, + pub cosine: f64, + pub threshold: f64, + /// Where the threshold comes from (R-7: no invented thresholds). + pub threshold_basis: String, + pub receipt_sha256: String, +} + +/// A cell's admission state. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(tag = "state")] +pub enum Status { + Admitted { parity: Parity }, + Refused { removed_by: String }, + NotRun { reason: NotRun }, +} + +/// One admission row. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Row { + pub schema: String, + pub cell: String, + pub host: String, + pub backend: String, + pub apr_tag: String, + pub apr_sha256: String, + pub model_id: String, + pub weights_sha256: String, + pub prereg_sha: String, + pub at: String, + pub status: Status, +} + +/// What an admission file resolves to. +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +pub struct Summary { + pub admitted: Vec, + pub refused: Vec, + pub not_run: Vec, + /// S-7: no cell is admissible for (A). + pub s7: bool, +} + +fn row_errors(r: &Row, prereg_sha: &str) -> Vec { + let mut e = Vec::new(); + let id = &r.cell; + if r.schema != SCHEME { + e.push(format!("{id}: schema {}", r.schema)); + } + match CELLS.iter().find(|c| c.cell == r.cell) { + None => e.push(format!("{id}: not a §2.1 cell")), + Some(c) if c.host != r.host || c.backend != r.backend => e.push(format!( + "{id}: {}/{} is not the declared {}/{}", + r.host, r.backend, c.host, c.backend + )), + Some(_) => {} + } + for (k, v) in [ + ("apr_tag", &r.apr_tag), + ("model_id", &r.model_id), + ("at", &r.at), + ] { + if v.trim().is_empty() || v == "unknown" { + e.push(format!("{id}: {k} missing")); + } + } + for (k, v) in [ + ("apr_sha256", &r.apr_sha256), + ("weights_sha256", &r.weights_sha256), + ] { + if !is_hex64(v) { + e.push(format!("{id}: {k} is not a sha256")); + } + } + if r.prereg_sha != prereg_sha { + e.push(format!("{id}: prereg_sha differs from the lock")); + } + match &r.status { + Status::Admitted { parity: p } => { + if p.oracle.trim().is_empty() || p.threshold_basis.trim().is_empty() { + e.push(format!("{id}: Admitted without oracle or threshold basis")); + } + if !is_hex64(&p.receipt_sha256) { + e.push(format!("{id}: Admitted without a parity receipt sha")); + } + if !(p.cosine.is_finite() && p.threshold.is_finite() && p.cosine >= p.threshold) { + e.push(format!( + "{id}: Admitted at cosine {} below threshold {}", + p.cosine, p.threshold + )); + } + } + Status::Refused { removed_by } if removed_by.trim().is_empty() => { + e.push(format!("{id}: Refused without removed_by")); + } + _ => {} + } + e +} + +/// Check an admission file (JSON lines). Every §2.1 cell must appear exactly +/// once with a well-formed row; otherwise the file is rejected as a whole. +/// +/// # Errors +/// Every reason the file is inadmissible (a silent cell is one of them). +pub fn check(lines: &str, prereg_sha: &str) -> Result> { + let mut errors = Vec::new(); + let mut rows: BTreeMap> = BTreeMap::new(); + for (ix, line) in lines + .lines() + .enumerate() + .filter(|(_, l)| !l.trim().is_empty()) + { + match serde_json::from_str::(line) { + Ok(r) => { + errors.extend(row_errors(&r, prereg_sha)); + rows.entry(r.cell.clone()).or_default().push(r); + } + Err(err) => errors.push(format!("line {}: {err}", ix + 1)), + } + } + let mut s = Summary { + admitted: Vec::new(), + refused: Vec::new(), + not_run: Vec::new(), + s7: true, + }; + for c in CELLS { + match rows.get(c.cell).map(Vec::as_slice) { + None | Some([]) => errors.push(format!("{}: silent (no admission row)", c.cell)), + Some([r]) => match r.status { + Status::Admitted { .. } => s.admitted.push(c.cell.into()), + Status::Refused { .. } => s.refused.push(c.cell.into()), + Status::NotRun { .. } => s.not_run.push(c.cell.into()), + }, + Some(_) => errors.push(format!("{}: more than one admission row", c.cell)), + } + } + s.s7 = s.admitted.is_empty(); + if errors.is_empty() { + Ok(s) + } else { + Err(errors) + } +} + +/// What a parity receipt must match before it admits a cell: the row's own +/// binary and weights, the declared threshold and its basis, and the evidence +/// floor (`min_positions` of `evidence/parity/thresholds.yaml`). +#[derive(Debug, Clone, Copy)] +pub struct Expect<'a> { + pub apr_sha256: &'a str, + pub weights_sha256: &'a str, + pub threshold: f64, + pub threshold_basis: &'a str, + pub min_positions: usize, +} + +/// Derive an `Admitted` parity block from the bytes of a parity receipt. +/// +/// # Errors +/// Every reason the receipt cannot admit the cell. +pub fn parity_from_receipt(receipt: &[u8], x: &Expect<'_>) -> Result> { + use serde_json::Value; + let v: Value = + serde_json::from_slice(receipt).map_err(|e| vec![format!("receipt is not JSON: {e}")])?; + let mut e = Vec::new(); + // (oracle, cosines, weights sha, binary sha) per receipt shape + let (oracle, cos, weights, binary): (String, Vec>, Option<&str>, Option<&str>) = + if v["schema"].as_str() == Some("apr-parity-oracle/v1") { + if v["verdict"].as_str() != Some("GREEN") { + e.push(format!("verdict {} is not GREEN", v["verdict"])); + } + let per = v["per_position"].as_array().map_or(&[][..], Vec::as_slice); + ( + v["oracle"].as_str().unwrap_or_default().to_string(), + per.iter().map(|p| p["cosine"].as_f64()).collect(), + v["subject"]["producer"]["model_sha256"].as_str(), + None, + ) + } else if v.get("parity").is_some() && v.get("comparator").is_some() { + if v["exit"].as_i64() != Some(0) { + e.push(format!("exit {} is not 0", v["exit"])); + } + if v["verdict"].as_str() != Some("pass") { + e.push(format!("verdict {} is not pass", v["verdict"])); + } + let p = &v["parity"]; + if p["failed"].as_u64() != Some(0) || p["parity"].as_bool() != Some(true) { + e.push(format!( + "parity failed {} (parity {})", + p["failed"], p["parity"] + )); + } + let comparator = v["comparator"].as_str().unwrap_or_default(); + let oracle = if comparator.starts_with("apr-cpu") { + APR_GPU_CPU.to_string() + } else { + comparator.to_string() + }; + let metrics = p["metrics"].as_array().map_or(&[][..], Vec::as_slice); + ( + oracle, + metrics + .iter() + .map(|m| m["cosine_similarity"].as_f64()) + .collect(), + v["weights_sha256"].as_str(), + Some(v["binary_sha256"].as_str().unwrap_or_default()), + ) + } else { + return Err(vec![ + "not a parity receipt (apr-parity-oracle/v1 or apr-review-serve parity)".into(), + ]); + }; + if oracle != LLAMA_CPP && oracle != APR_GPU_CPU { + e.push(format!("oracle {oracle:?} is not a declared parity.oracle")); + } + if weights != Some(x.weights_sha256) { + e.push(format!( + "weights sha {weights:?} is not the row's {}", + x.weights_sha256 + )); + } + if binary.is_some_and(|b| b != x.apr_sha256) { + e.push(format!( + "binary sha {binary:?} is not the row's {}", + x.apr_sha256 + )); + } + if cos.len() < x.min_positions { + e.push(format!( + "{} positions < min_positions {}", + cos.len(), + x.min_positions + )); + } + if !x.threshold.is_finite() { + e.push(format!("threshold {} is not finite", x.threshold)); + } + if x.threshold_basis.trim().is_empty() { + e.push("no threshold basis declared".into()); + } + let mut min = f64::INFINITY; + for (i, c) in cos.iter().enumerate() { + match c { + // JSON carries no NaN or infinity: a number is finite + Some(c) => min = min.min(*c), + None => e.push(format!("position {i}: no numeric cosine")), + } + } + if min < x.threshold { + e.push(format!("min cosine {min} below threshold {}", x.threshold)); + } + if e.is_empty() { + Ok(Parity { + oracle, + cosine: min, + threshold: x.threshold, + threshold_basis: x.threshold_basis.to_string(), + receipt_sha256: crate::corpus::sha256_hex(receipt), + }) + } else { + Err(e) + } +} + +#[cfg(test)] +#[path = "admission_tests.rs"] +mod tests; diff --git a/crates/aprender-review-experiment/src/admission_tests.rs b/crates/aprender-review-experiment/src/admission_tests.rs new file mode 100644 index 0000000000..33a94946a8 --- /dev/null +++ b/crates/aprender-review-experiment/src/admission_tests.rs @@ -0,0 +1,339 @@ +// `json!` expands to an `unwrap` of an infallible `to_value`. +#![allow(clippy::disallowed_methods)] + +use super::*; + +const PREREG: &str = "ef51087dc79bab0ad160e8a14f5b13e2ea43986b30c05dafc63c84c2dc21cdc0"; + +fn h(c: char) -> String { + std::iter::repeat_n(c, 64).collect() +} + +fn row(c: &Cell, status: Status) -> Row { + Row { + schema: SCHEME.into(), + cell: c.cell.into(), + host: c.host.into(), + backend: c.backend.into(), + apr_tag: "v0.69.3".into(), + apr_sha256: h('a'), + model_id: "Qwen3.5-4B-Q4_K_M".into(), + weights_sha256: h('b'), + prereg_sha: PREREG.into(), + at: "2026-09-25T12:00:00Z".into(), + status, + } +} + +fn admitted(cosine: f64) -> Status { + Status::Admitted { + parity: Parity { + oracle: LLAMA_CPP.into(), + cosine, + threshold: 0.98, + threshold_basis: "evidence/parity/thresholds.yaml".into(), + receipt_sha256: h('c'), + }, + } +} + +fn not_run() -> Status { + Status::NotRun { + reason: NotRun::NoDeclaredExecutor, + } +} + +fn file(rows: &[Row]) -> String { + rows.iter() + .map(|r| serde_json::to_string(r).expect("json") + "\n") + .collect() +} + +/// C2 admitted, C1/C5b refused, the rest not run. +fn mixed() -> Vec { + CELLS + .iter() + .map(|c| match c.cell { + "C2" => row(c, admitted(0.995)), + "C1" | "C5b" => row( + c, + Status::Refused { + removed_by: "aprender#9999: backend refuses qwen35".into(), + }, + ), + _ => row(c, not_run()), + }) + .collect() +} + +#[test] +fn a_complete_file_resolves_every_cell() { + let s = check(&file(&mixed()), PREREG).expect("admissible"); + assert_eq!(s.admitted, ["C2"]); + assert_eq!(s.refused, ["C1", "C5b"]); + assert_eq!(s.not_run, ["C3", "C4", "C5a"]); + assert!(!s.s7); +} + +#[test] +fn falsify_rca_001_a_silent_cell_rejects_the_file() { + for drop in 0..CELLS.len() { + let mut rows = mixed(); + let gone = rows.remove(drop).cell; + let e = check(&file(&rows), PREREG).expect_err("a silent cell must reject the file"); + assert!( + e.iter() + .any(|m| m == &format!("{gone}: silent (no admission row)")), + "{e:?}" + ); + } + let mut dup = mixed(); + dup.push(dup[3].clone()); + assert!(check(&file(&dup), PREREG).is_err(), "a cell admitted twice"); +} + +#[test] +fn falsify_rca_002_admitted_needs_a_passing_parity_receipt() { + let cases: Vec<(&str, Box)> = vec![ + ("below threshold", Box::new(|p| p.cosine = 0.97)), + ("NaN cosine", Box::new(|p| p.cosine = f64::NAN)), + ("no receipt", Box::new(|p| p.receipt_sha256 = String::new())), + ("no basis", Box::new(|p| p.threshold_basis = " ".into())), + ("no oracle", Box::new(|p| p.oracle = String::new())), + ]; + for (name, bad) in cases { + let mut rows = mixed(); + if let Status::Admitted { parity } = &mut rows[1].status { + bad(parity); + } + assert!(check(&file(&rows), PREREG).is_err(), "{name}"); + } + let mut rows = mixed(); + rows[0].status = Status::Refused { + removed_by: String::new(), + }; + assert!( + check(&file(&rows), PREREG).is_err(), + "Refused needs removed_by" + ); +} + +#[test] +fn falsify_rca_003_no_admitted_cell_raises_s7() { + let rows: Vec = CELLS.iter().map(|c| row(c, not_run())).collect(); + let s = check(&file(&rows), PREREG).expect("admissible"); + assert!(s.s7, "all NotRun is S-7"); + let mut rows = mixed(); + rows[1].status = Status::Refused { + removed_by: "x".into(), + }; + assert!( + check(&file(&rows), PREREG).expect("ok").s7, + "Refused + NotRun is S-7" + ); +} + +#[test] +fn identity_and_cell_declaration_are_enforced() { + let cases: Vec<(&str, Box)> = vec![ + ("backend swap", Box::new(|r| r.backend = "cuda".into())), + ("host", Box::new(|r| r.host = "lambda-labs".into())), + ("cell name", Box::new(|r| r.cell = "C9".into())), + ("apr sha", Box::new(|r| r.apr_sha256 = "unknown".into())), + ( + "weights sha", + Box::new(|r| r.weights_sha256 = h('b')[..12].into()), + ), + ("model", Box::new(|r| r.model_id = "unknown".into())), + ("prereg", Box::new(|r| r.prereg_sha = h('9'))), + ("schema", Box::new(|r| r.schema = "v0".into())), + ]; + for (name, bad) in cases { + let mut rows = mixed(); + bad(&mut rows[4]); + assert!(check(&file(&rows), PREREG).is_err(), "{name}"); + } + assert!(check("{not json}\n", PREREG).is_err()); +} + +/// A `apr-review-serve parity` receipt (the declared C4 oneshot's shape) with +/// `n` positions; position `low_at` carries cosine `low`, the rest 0.9999. +fn serve_receipt(n: usize, low_at: usize, low: f64) -> serde_json::Value { + let metrics: Vec<_> = (0..n) + .map(|p| { + let c = if p == low_at { low } else { 0.9999 }; + serde_json::json!({"position": p, "cosine_similarity": c, "verdict": "Pass"}) + }) + .collect(); + serde_json::json!({ + "host": "gx10-a5b5", "apr_tag": "v0.69.3", + "binary_sha256": h('a'), "weights_sha256": h('b'), + "comparator": "apr-cpu (apr parity: GPU vs CPU; no llama.cpp comparator in this apr, aprender#3576)", + "exit": 0, "verdict": "pass", + "parity": {"tokens": n, "passed": n, "failed": 0, "parity": true, "metrics": metrics} + }) +} + +/// An `apr-parity-oracle/v1` receipt (`apr parity-oracle`, #4444) with `n` +/// positions; its top-level `cosine` claims 0.9999 whatever the positions say. +fn oracle_receipt(n: usize, low_at: usize, low: f64) -> serde_json::Value { + let per: Vec<_> = (0..n) + .map(|p| { + let c = if p == low_at { low } else { 0.9999 }; + serde_json::json!({"pos": p, "cosine": c}) + }) + .collect(); + serde_json::json!({ + "schema": "apr-parity-oracle/v1", "oracle": LLAMA_CPP, "verdict": "GREEN", + "cosine": 0.9999, "threshold": 0.5, "threshold_basis": "self-declared", + "n_positions": n, + "subject": {"producer": {"model_sha256": h('b')}}, + "per_position": per + }) +} + +const BASIS: &str = "evidence/parity/thresholds.yaml default.min_cosine"; + +fn expect() -> Expect<'static> { + Expect { + apr_sha256: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + weights_sha256: "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + threshold: 0.98, + threshold_basis: BASIS, + min_positions: 64, + } +} + +fn bytes(v: &serde_json::Value) -> Vec { + serde_json::to_vec_pretty(v).expect("serialize receipt") +} + +/// FALSIFY-RCA-004: an Admitted parity block is derived from the receipt — +/// the minimum cosine over its positions (never a summary field), the sha256 +/// of its bytes, the oracle it measured against, and the declared threshold +/// and basis (never the receipt's own). +#[test] +fn falsify_rca_004_admitted_is_derived_from_the_receipt() { + let b = bytes(&serve_receipt(64, 17, 0.985)); + let p = parity_from_receipt(&b, &expect()).expect("admits"); + assert_eq!(p.oracle, APR_GPU_CPU); + assert_eq!(p.cosine, 0.985); + assert_eq!(p.threshold, 0.98); + assert_eq!(p.threshold_basis, BASIS); + assert_eq!(p.receipt_sha256, crate::corpus::sha256_hex(&b)); + + let o = bytes(&oracle_receipt(78, 40, 0.981)); + let q = parity_from_receipt(&o, &expect()).expect("admits"); + assert_eq!(q.oracle, LLAMA_CPP); + assert_eq!(q.cosine, 0.981, "the summary `cosine` must not be read"); + assert_eq!( + q.threshold, 0.98, + "the receipt's own threshold must not be read" + ); + assert_eq!(q.threshold_basis, BASIS); + + // exactly min_positions and exactly the threshold admit + assert!(parity_from_receipt(&bytes(&serve_receipt(64, 0, 0.98)), &expect()).is_ok()); + + // the derived block is a well-formed Admitted row + let mut rows: Vec = CELLS.iter().map(|c| row(c, not_run())).collect(); + rows[3] = row(&CELLS[3], Status::Admitted { parity: p }); + let s = check(&file(&rows), PREREG).expect("admissible"); + assert_eq!((s.admitted, s.s7), (vec!["C4".to_string()], false)); +} + +/// FALSIFY-RCA-005: a receipt with too few positions, a failing or +/// non-zero-exit run, a position below threshold, a missing or non-finite +/// cosine, or no declared basis does not admit, and says why. +#[test] +fn falsify_rca_005_a_short_or_failing_receipt_does_not_admit() { + let refuse = |v: &serde_json::Value, x: &Expect<'_>, why: &str| { + let e = parity_from_receipt(&bytes(v), x).expect_err(why); + assert!(e.iter().any(|m| m.contains(why)), "{why}: {e:?}"); + }; + let x = expect(); + // the declared gx10 oneshot measured 7 positions: under the 64 floor + refuse(&serve_receipt(7, 0, 0.9997), &x, "min_positions"); + refuse(&serve_receipt(63, 0, 0.99), &x, "min_positions"); + refuse(&oracle_receipt(63, 0, 0.99), &x, "min_positions"); + refuse(&serve_receipt(64, 5, 0.979_999), &x, "below threshold"); + refuse(&oracle_receipt(78, 5, 0.97), &x, "below threshold"); + let mut v = serve_receipt(64, 0, 0.99); + v["exit"] = 1.into(); + refuse(&v, &x, "exit"); + let mut v = serve_receipt(64, 0, 0.99); + v["verdict"] = "fail".into(); + refuse(&v, &x, "verdict"); + let mut v = oracle_receipt(64, 0, 0.99); + v["verdict"] = "RED".into(); + refuse(&v, &x, "verdict"); + let mut v = serve_receipt(64, 0, 0.99); + v["parity"]["failed"] = 1.into(); + refuse(&v, &x, "failed"); + let mut v = serve_receipt(64, 0, 0.99); + v["parity"]["parity"] = false.into(); + refuse(&v, &x, "failed"); + let mut v = serve_receipt(64, 0, 0.99); + v["parity"]["metrics"][3] = serde_json::json!({"position": 3}); + refuse(&v, &x, "cosine"); + let mut v = oracle_receipt(64, 0, 0.99); + v["per_position"][3]["cosine"] = "NaN".into(); + refuse(&v, &x, "cosine"); + let mut v = oracle_receipt(64, 0, 0.99); + v["oracle"] = "ollama".into(); + refuse(&v, &x, "oracle"); + let mut v = serve_receipt(64, 0, 0.99); + v["comparator"] = "hf-transformers".into(); + refuse(&v, &x, "oracle"); + refuse( + &serve_receipt(64, 0, 0.99), + &Expect { + threshold_basis: " ", + ..x + }, + "basis", + ); + refuse( + &serve_receipt(64, 0, 0.99), + &Expect { + threshold: f64::NAN, + ..x + }, + "threshold", + ); + assert!(parity_from_receipt(b"{not json", &x).is_err()); + assert!(parity_from_receipt(b"{\"schema\": \"other\"}", &x).is_err()); + // a half-shaped receipt (parity block, no comparator, or the reverse) is + // not a parity receipt at all, not an oracle mismatch + for key in ["comparator", "parity"] { + let mut v = serve_receipt(64, 0, 0.99); + v.as_object_mut().expect("object").remove(key); + refuse(&v, &x, "not a parity receipt"); + } +} + +/// FALSIFY-RCA-006: the receipt must be of the row's own binary and weights — +/// a passing receipt of another build or another model does not admit. +#[test] +fn falsify_rca_006_the_receipt_must_be_the_rows_binary_and_weights() { + let x = expect(); + let wrong = "c".repeat(64); + let mut v = serve_receipt(64, 0, 0.99); + v["binary_sha256"] = wrong.clone().into(); + let e = parity_from_receipt(&bytes(&v), &x).expect_err("binary"); + assert!(e.iter().any(|m| m.contains("binary")), "{e:?}"); + let mut v = serve_receipt(64, 0, 0.99); + v["weights_sha256"] = wrong.clone().into(); + let e = parity_from_receipt(&bytes(&v), &x).expect_err("weights"); + assert!(e.iter().any(|m| m.contains("weights")), "{e:?}"); + let mut v = oracle_receipt(64, 0, 0.99); + v["subject"]["producer"]["model_sha256"] = wrong.into(); + let e = parity_from_receipt(&bytes(&v), &x).expect_err("weights"); + assert!(e.iter().any(|m| m.contains("weights")), "{e:?}"); + let mut v = serve_receipt(64, 0, 0.99); + v.as_object_mut().expect("object").remove("binary_sha256"); + assert!( + parity_from_receipt(&bytes(&v), &x).is_err(), + "a missing sha must not match" + ); +} diff --git a/crates/aprender-review-experiment/src/b2.rs b/crates/aprender-review-experiment/src/b2.rs new file mode 100644 index 0000000000..427bca374d --- /dev/null +++ b/crates/aprender-review-experiment/src/b2.rs @@ -0,0 +1,251 @@ +//! REX-12 B2 loop (§5.3 B2a..B2d), `review-b2-loop-v1`. +//! +//! - **Row status.** Each B2 tier waits on one `apr` verb accepting qwen35. +//! A tier is `Ready` only on a positive acceptance receipt for its verb. A +//! verb with a `refusal-receipt-v1` row in the refusal ledger reads +//! `VerbRefused{removed_by}`. A verb with neither is `Unledgered`: nobody +//! measured it, so it is NotRun too. Absence of a refusal is not +//! acceptance; the ledger is a bucket, not a census. +//! - **Teacher dataset.** The 27B teacher writes top-k logits at m = 1 over +//! the train pool (every non-test item). [`teacher_receipt`] refuses the +//! dataset unless every row is a known non-test item, appears once, carries +//! exactly k finite logits per position, and the file text contains no +//! sealed diff sha or hunk. The receipt carries sha, count and 0 test hits. + +use std::collections::{BTreeMap, BTreeSet}; + +use serde::{Deserialize, Serialize}; + +use crate::contamination::Index; +use crate::corpus::{sha256_hex, Item, Split}; + +pub const SCHEME: &str = "review-b2-loop-v1"; +pub const LEDGER_SCHEMA: &str = "apr-refusal-ledger/v1"; + +/// The §5.3 B2 tiers and the verb each waits on. +pub const TIERS: [(&str, &str); 4] = [ + ("B2a", "distill"), + ("B2b", "finetune"), + ("B2c", "merge"), + ("B2d", "quantize"), +]; + +/// One `refusal-receipt-v1` row. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Refusal { + pub verb: String, + pub reason: String, + pub exit_code: i32, + pub removed_by: String, +} + +/// `evidence/verbs/refusals.json`. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct Ledger { + pub schema: String, + #[serde(default)] + pub note: String, + pub refusals: Vec, +} + +/// `refusal-receipt-v1`: `v.`, `never` or `unscheduled`. +#[must_use] +pub fn valid_removed_by(s: &str) -> bool { + if matches!(s, "never" | "unscheduled") { + return true; + } + let Some(v) = s.strip_prefix('v') else { + return false; + }; + let parts: Vec<&str> = v.split('.').collect(); + parts.len() == 2 + && parts + .iter() + .all(|p| !p.is_empty() && p.bytes().all(|b| b.is_ascii_digit())) +} + +/// Parse and validate a refusal ledger; one bad row rejects the file. +pub fn parse_ledger(text: &str) -> Result { + let l: Ledger = serde_json::from_str(text).map_err(|e| e.to_string())?; + if l.schema != LEDGER_SCHEMA { + return Err(format!("schema {:?} is not {LEDGER_SCHEMA}", l.schema)); + } + let mut seen = BTreeSet::new(); + for r in &l.refusals { + if !valid_removed_by(&r.removed_by) { + return Err(format!( + "{}: removed_by {:?} is not v., never or unscheduled", + r.verb, r.removed_by + )); + } + if r.reason.trim().is_empty() || r.exit_code == 0 { + return Err(format!( + "{}: a refusal needs a reason and a non-zero exit", + r.verb + )); + } + if !seen.insert(r.verb.as_str()) { + return Err(format!("{}: two refusal rows", r.verb)); + } + } + Ok(l) +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "state", rename_all = "snake_case")] +pub enum State { + /// The verb accepted qwen35: the tier's challengers go to §5.4. + Ready { + acceptance: String, + }, + VerbRefused { + removed_by: String, + exit_code: i32, + }, + /// No refusal row and no acceptance: unmeasured, so NotRun. + Unledgered, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct Row { + pub schema: String, + pub tier: String, + pub verb: String, + #[serde(flatten)] + pub state: State, +} + +/// The B2 rows. `accepted` maps a verb to its acceptance receipt (a sha or a +/// path); a verb that is both accepted and refused is an error, since one of +/// the two receipts is stale. +pub fn status(ledger: &Ledger, accepted: &BTreeMap) -> Result, String> { + let refused: BTreeMap<&str, &Refusal> = ledger + .refusals + .iter() + .map(|r| (r.verb.as_str(), r)) + .collect(); + TIERS + .iter() + .map(|&(tier, verb)| { + let state = match (accepted.get(verb), refused.get(verb)) { + (Some(_), Some(_)) => { + return Err(format!( + "{verb}: both accepted and refused; one receipt is stale" + )) + } + (Some(a), None) => State::Ready { + acceptance: a.clone(), + }, + (None, Some(r)) => State::VerbRefused { + removed_by: r.removed_by.clone(), + exit_code: r.exit_code, + }, + (None, None) => State::Unledgered, + }; + Ok(Row { + schema: SCHEME.into(), + tier: tier.into(), + verb: verb.into(), + state, + }) + }) + .collect() +} + +/// One teacher row: an item's top-k `(token, logit)` pairs per position. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct TeacherRow { + pub item_id: String, + pub top_k: Vec>, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct TeacherReceipt { + pub schema: String, + pub teacher_weights_sha256: String, + pub k: usize, + /// Samples per item (§5.3 B2a: m = 1). + pub m: usize, + pub count: usize, + pub sha256: String, + pub sealed_items_indexed: usize, + pub test_hits: usize, +} + +/// Receipt a teacher-logit JSONL, or list every reason it is refused. +pub fn teacher_receipt( + jsonl: &str, + teacher_weights_sha256: &str, + k: usize, + items: &[Item], + sealed: &Index, +) -> Result> { + let mut e = Vec::new(); + if sealed.is_empty() { + e.push("no sealed manifest indexed: 0 test hits would prove nothing".into()); + } + if k == 0 { + e.push("k = 0".into()); + } + let split: BTreeMap<&str, Split> = items.iter().map(|i| (i.id.as_str(), i.split)).collect(); + let mut seen = BTreeSet::new(); + for (n, line) in jsonl + .lines() + .enumerate() + .filter(|(_, l)| !l.trim().is_empty()) + { + let row: TeacherRow = match serde_json::from_str(line) { + Ok(r) => r, + Err(x) => { + e.push(format!("line {}: {x}", n + 1)); + continue; + } + }; + let id = row.item_id.as_str(); + match split.get(id) { + None => e.push(format!("{id}: not a corpus item")), + Some(Split::Test) => e.push(format!("{id}: a sealed test item (R-2)")), + Some(Split::Dev) => {} + } + if !seen.insert(row.item_id.clone()) { + e.push(format!("{id}: more than one row (m = 1)")); + } + if row.top_k.is_empty() { + e.push(format!("{id}: no positions")); + } + if let Some(p) = row + .top_k + .iter() + .position(|pos| pos.len() != k || pos.iter().any(|(_, x)| !x.is_finite())) + { + e.push(format!("{id}: position {p} is not {k} finite logits")); + } + } + if seen.is_empty() { + e.push("no teacher rows".into()); + } + let hits = sealed.scan(jsonl, "teacher"); + if !hits.is_empty() { + e.push(format!("{} sealed test leak(s): {hits:?}", hits.len())); + } + if !e.is_empty() { + return Err(e); + } + Ok(TeacherReceipt { + schema: SCHEME.into(), + teacher_weights_sha256: teacher_weights_sha256.into(), + k, + m: 1, + count: seen.len(), + sha256: sha256_hex(jsonl.as_bytes()), + sealed_items_indexed: sealed.len(), + test_hits: 0, + }) +} + +#[cfg(test)] +#[path = "b2_tests.rs"] +mod tests; diff --git a/crates/aprender-review-experiment/src/b2_tests.rs b/crates/aprender-review-experiment/src/b2_tests.rs new file mode 100644 index 0000000000..37fd589381 --- /dev/null +++ b/crates/aprender-review-experiment/src/b2_tests.rs @@ -0,0 +1,146 @@ +use super::*; +use crate::corpus::{hunk_fingerprints, Class, Sealed}; + +const IN_TREE: &str = include_str!("../../../evidence/verbs/refusals.json"); + +fn ledger(rows: &str) -> String { + format!(r#"{{"schema":"apr-refusal-ledger/v1","refusals":[{rows}]}}"#) +} + +fn refusal(verb: &str, removed_by: &str) -> String { + format!( + r#"{{"verb":"{verb}","reason":"refuses qwen35","exit_code":8,"removed_by":"{removed_by}"}}"# + ) +} + +#[test] +fn falsify_rxb_001_no_b2_row_is_ready_without_an_acceptance_receipt() { + // The in-tree ledger at HEAD: every B2 row is NotRun, none silently Ready. + let l = parse_ledger(IN_TREE).expect("in-tree ledger parses"); + let rows = status(&l, &BTreeMap::new()).expect("status"); + assert_eq!(rows.len(), 4); + assert!(rows.iter().all(|r| !matches!(r.state, State::Ready { .. }))); + + let l = parse_ledger(&ledger(&refusal("distill", "v0.75"))).expect("parses"); + let acc = BTreeMap::from([("finetune".to_string(), "sha-ft".to_string())]); + let rows = status(&l, &acc).expect("status"); + let by: BTreeMap<&str, &State> = rows.iter().map(|r| (r.tier.as_str(), &r.state)).collect(); + assert_eq!( + by["B2a"], + &State::VerbRefused { + removed_by: "v0.75".into(), + exit_code: 8 + } + ); + assert_eq!( + by["B2b"], + &State::Ready { + acceptance: "sha-ft".into() + } + ); + assert_eq!( + by["B2c"], + &State::Unledgered, + "no refusal row is not acceptance" + ); + + let both = BTreeMap::from([("distill".to_string(), "sha".to_string())]); + assert!(status(&l, &both).is_err(), "accepted and refused at once"); +} + +#[test] +fn falsify_rxb_002_the_ledger_is_refusal_receipt_v1_or_rejected() { + for good in ["v0.75", "v1.0", "never", "unscheduled"] { + assert!(valid_removed_by(good), "{good}"); + } + for bad in [ + "tbd", "soon", "0.70", "v0.70.1", "v0.", "v.7", "vx.y", "", "9f3c2a1", + ] { + assert!(!valid_removed_by(bad), "{bad}"); + assert!( + parse_ledger(&ledger(&refusal("merge", bad))).is_err(), + "{bad}" + ); + } + let dup = format!( + "{},{}", + refusal("merge", "never"), + refusal("merge", "v0.75") + ); + assert!( + parse_ledger(&ledger(&dup)).is_err(), + "two rows for one verb" + ); + assert!(parse_ledger(&ledger(&refusal("merge", "v0.75")).replace(":8", ":0")).is_err()); + assert!(parse_ledger(&ledger("").replace("ledger/v1", "ledger/v2")).is_err()); +} + +fn diff(body: &str) -> String { + format!("--- a/src/a.rs\n+++ b/src/a.rs\n@@ -10,5 +10,5 @@\n fn f(x: u32) -> bool {{\n{body} }}\n \n") +} + +fn fixture() -> (Vec, Index, String) { + let mk = |id: &str, split: Split, d: &str| { + let mut i = Item::new(id.into(), Class::R, "src".into(), d); + i.split = split; + i + }; + let test_diff = diff("- z * 2\n+ z * 3\n"); + let items = vec![ + mk("D01", Split::Dev, &diff("- a\n+ b\n")), + mk("D02", Split::Dev, &diff("- c\n+ d\n")), + mk("T01", Split::Test, &test_diff), + ]; + let ix = Index::new(&[Sealed { + id: "T01".into(), + diff_sha256: sha256_hex(test_diff.as_bytes()), + hunks: hunk_fingerprints(&test_diff), + }]); + (items, ix, test_diff) +} + +fn row(id: &str, k: usize) -> String { + let pos: Vec<(u32, f32)> = (0..k).map(|t| (t as u32, -(t as f32))).collect(); + serde_json::to_string(&TeacherRow { + item_id: id.into(), + top_k: vec![pos.clone(), pos], + }) + .expect("serializes") +} + +#[test] +fn falsify_rxb_003_the_teacher_dataset_is_receipted_with_zero_test_hashes() { + let (items, ix, test_diff) = fixture(); + let good = format!("{}\n{}\n", row("D01", 4), row("D02", 4)); + let r = teacher_receipt(&good, "w27b", 4, &items, &ix).expect("receipted"); + assert_eq!((r.count, r.m, r.k, r.test_hits), (2, 1, 4, 0)); + assert_eq!(r.sha256, sha256_hex(good.as_bytes())); + assert_eq!(r.sealed_items_indexed, 1); + + let refused = |text: &str, why: &str| { + let e = teacher_receipt(text, "w27b", 4, &items, &ix).expect_err(why); + assert!(e.iter().any(|x| x.contains(why)), "{why}: {e:?}"); + }; + refused( + &format!("{good}{}\n", row("T01", 4)), + "T01: a sealed test item", + ); + refused(&format!("{good}{}\n", row("D01", 4)), "more than one row"); + refused(&format!("{good}{}\n", row("X9", 4)), "not a corpus item"); + refused(&format!("{}\n", row("D01", 3)), "not 4 finite logits"); + refused(&row("D01", 4).replace("-1.0", "NaN"), "line 1"); + // A sealed diff carried in any string field is a hit. + let smuggled = format!( + "{good}{}\n", + format!( + r#"{{"item_id":"D02","top_k":[],"note":{}}}"#, + serde_json::to_string(&test_diff).expect("serializes") + ) + ); + refused(&smuggled, "sealed test leak"); + let leak = format!(r#"{good}"{}""#, sha256_hex(test_diff.as_bytes())); + refused(&leak, "sealed test leak"); + refused("", "no teacher rows"); + let e = teacher_receipt(&good, "w27b", 4, &items, &Index::default()).expect_err("empty"); + assert!(e[0].contains("prove nothing"), "{e:?}"); +} diff --git a/crates/aprender-review-experiment/src/build_corpus.rs b/crates/aprender-review-experiment/src/build_corpus.rs new file mode 100644 index 0000000000..fcb88ee559 --- /dev/null +++ b/crates/aprender-review-experiment/src/build_corpus.rs @@ -0,0 +1,325 @@ +//! REX-02 corpus builder: turns merged PRs and cargo-mutants listings into +//! corpus items. Reads git (and `gh` for the G-class green check); all +//! selection is seeded and ordered, so a rebuild from the same inputs yields +//! the same items and the same manifest. + +use crate::corpus::{ + drop_comment_only_hunks, hunk_fingerprints, is_review_path, names_the_mutation, pick_balanced, + sanitize_mutant_diff, sha256_hex, Class, Item, Sealed, Split, +}; +use crate::stats::SplitMix64; +use serde::Deserialize; +use std::collections::BTreeSet; +use std::process::Command; + +/// Target n per class (§2.2). +pub const PER_CLASS: usize = 50; +/// Candidate cap `[A]`: a diff over ~32k proxy tokens is dropped at +/// construction (release merge-backs, vendored drops). Anything kept is never +/// truncated; one that overflows a model's context is `NotRun{ContextOverflow}`. +pub const MAX_TOKENS: u64 = 32_000; + +/// The subset of `gh pr list --json` the builder reads. +#[derive(Debug, Clone, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct Pr { + pub number: u64, + pub title: String, + pub merged_at: String, + pub merge_commit: Option, + #[serde(default)] + pub labels: Vec, + #[serde(default)] + pub closing_issues_references: Vec, +} + +#[derive(Debug, Clone, Deserialize)] +pub struct Oid { + pub oid: String, +} + +#[derive(Debug, Clone, Deserialize)] +pub struct Named { + pub name: String, +} + +#[derive(Debug, Clone, Deserialize)] +pub struct IssueRef { + pub number: u64, +} + +/// The subset of a `cargo mutants --list --diff --json` row the builder reads. +#[derive(Debug, Clone, Deserialize)] +pub struct Mutant { + pub name: String, + pub file: String, + pub diff: String, + pub package: String, +} + +/// A fix title (R class needs a fix PR, not a feature that closed an issue). +#[must_use] +pub fn is_fix_title(title: &str) -> bool { + let t = title.to_lowercase(); + t.starts_with("fix") + || t.starts_with("bug") + || [ + "fix(", + ": fix", + "defect", + "regression", + "broken", + "wrong", + "crash", + "panic", + ] + .iter() + .any(|k| t.contains(k)) +} + +/// PR numbers named by a merged `Revert …` PR (`#N` in its title). +#[must_use] +pub fn reverted_prs(prs: &[Pr]) -> BTreeSet { + prs.iter() + .filter(|p| p.title.to_lowercase().starts_with("revert")) + .flat_map(|p| { + p.title + .split('#') + .skip(1) + .filter_map(|s| { + let d: String = s.chars().take_while(char::is_ascii_digit).collect(); + d.parse().ok() + }) + .collect::>() + }) + .collect() +} + +fn git(args: &[&str]) -> Option { + let out = Command::new("git").args(args).output().ok()?; + out.status + .success() + .then(|| String::from_utf8_lossy(&out.stdout).into_owned()) +} + +/// Review paths a commit touched. +fn review_paths(oid: &str) -> Vec { + let parent = format!("{oid}^"); + git(&["diff", "--name-only", &parent, oid]) + .unwrap_or_default() + .lines() + .filter(|p| is_review_path(p)) + .map(str::to_string) + .collect() +} + +/// `git diff -U3 from to -- paths`, or `None` when empty or over the cap. +fn diff_between(from: &str, to: &str, paths: &[String]) -> Option { + if paths.is_empty() { + return None; + } + let mut args = vec!["diff", "--no-color", "--no-ext-diff", "-U3", from, to, "--"]; + args.extend(paths.iter().map(String::as_str)); + let d = git(&args)?; + (!d.is_empty() && (d.len() as u64).div_ceil(4) <= MAX_TOKENS).then_some(d) +} + +/// R candidates: fix PRs with a linked issue, fix hunks reverse-applied. +#[must_use] +pub fn r_candidates(prs: &[Pr]) -> Vec<(Item, String)> { + prs.iter() + .filter(|p| !p.closing_issues_references.is_empty() && is_fix_title(&p.title)) + .filter_map(|p| { + let oid = &p.merge_commit.as_ref()?.oid; + let paths = review_paths(oid); + let diff = drop_comment_only_hunks(&diff_between(oid, &format!("{oid}^"), &paths)?); + if diff.is_empty() { + return None; + } + let issues: Vec = p + .closing_issues_references + .iter() + .map(|i| format!("#{}", i.number)) + .collect(); + let source = format!("pr#{} {oid} reversed; fixes {}", p.number, issues.join(",")); + let item = Item::new(format!("R-pr{}", p.number), Class::R, source, &diff); + Some((item, diff)) + }) + .collect() +} + +/// G candidates: merged before `cutoff`, not a fix, not reverted, no +/// `regression` label. The green check is applied afterwards ([`is_green`]). +#[must_use] +pub fn g_candidates(prs: &[Pr], cutoff: &str) -> Vec<(Item, String)> { + let reverted = reverted_prs(prs); + prs.iter() + .filter(|p| p.merged_at.as_str() < cutoff) + .filter(|p| !is_fix_title(&p.title) && !p.title.to_lowercase().starts_with("revert")) + .filter(|p| !reverted.contains(&p.number)) + .filter(|p| !p.labels.iter().any(|l| l.name == "regression")) + .filter_map(|p| { + let oid = &p.merge_commit.as_ref()?.oid; + let paths = review_paths(oid); + let diff = diff_between(&format!("{oid}^"), oid, &paths)?; + let source = format!("pr#{} {oid}", p.number); + let item = Item::new(format!("G-pr{}", p.number), Class::G, source, &diff); + Some((item, diff)) + }) + .collect() +} + +/// Green clean-room: `ci / gate` and `workspace-test` both SUCCESS and no +/// check FAILED. Unknown (gh error) is not green. +#[must_use] +pub fn is_green(pr: u64) -> bool { + let q = ".statusCheckRollup[]|\"\\(.name // .context)|\\(.conclusion // .state)\""; + let Ok(out) = Command::new("gh") + .args(["pr", "view", &pr.to_string(), "-R", "paiml/aprender"]) + .args(["--json", "statusCheckRollup", "-q", q]) + .output() + else { + return false; + }; + out.status.success() && rollup_is_green(&String::from_utf8_lossy(&out.stdout)) +} + +/// [`is_green`] on the `name|conclusion` lines. +#[must_use] +pub fn rollup_is_green(lines: &str) -> bool { + let ok = |n: &str| lines.lines().any(|l| l == format!("{n}|SUCCESS")); + let failed = lines + .lines() + .any(|l| l.ends_with("|FAILURE") || l.ends_with("|TIMED_OUT") || l.ends_with("|ERROR")); + ok("ci / gate") && ok("workspace-test") && !failed +} + +/// P candidates: a seeded sample of mutants on review paths, one per file, +/// with every mutation marker stripped. +#[must_use] +pub fn p_candidates(mutants: &[Mutant], base: &str, n: usize, seed: u64) -> Vec<(Item, String)> { + let mut ms: Vec<&Mutant> = mutants.iter().filter(|m| is_review_path(&m.file)).collect(); + ms.sort_by(|a, b| (&a.package, &a.name).cmp(&(&b.package, &b.name))); + let mut rng = SplitMix64::new(seed); + let keys: Vec = ms.iter().map(|_| rng.next_u64()).collect(); + let mut order: Vec = (0..ms.len()).collect(); + order.sort_by_key(|&i| keys[i]); + let mut files = BTreeSet::new(); + let mut out = Vec::new(); + for i in order { + let m = ms[i]; + if out.len() == n || !files.insert(m.file.clone()) { + continue; + } + let diff = sanitize_mutant_diff(&m.diff, &m.file); + if names_the_mutation(&diff) { + continue; + } + let id = format!("P-{}", &sha256_hex(m.name.as_bytes())[..12]); + let source = format!("cargo-mutants 27.0.0 @ {base}: {} :: {}", m.package, m.name); + out.push((Item::new(id, Class::P, source, &diff), diff)); + } + out +} + +/// Keep the diffs of the items `pick_balanced` chose. +#[must_use] +pub fn choose(cands: Vec<(Item, String)>, n: usize, seed: u64) -> Vec<(Item, String)> { + let items: Vec = cands.iter().map(|(i, _)| i.clone()).collect(); + let keep: BTreeSet = pick_balanced(items, n, seed) + .into_iter() + .map(|i| i.id) + .collect(); + cands + .into_iter() + .filter(|(i, _)| keep.contains(&i.id)) + .collect() +} + +/// Sealed rows for the test split. +#[must_use] +pub fn seal(items: &[(Item, String)]) -> Vec { + items + .iter() + .filter(|(i, _)| i.split == Split::Test) + .map(|(i, d)| Sealed { + id: i.id.clone(), + diff_sha256: i.diff_sha256.clone(), + hunks: hunk_fingerprints(d), + }) + .collect() +} + +#[cfg(test)] +mod tests { + use super::*; + + fn pr(n: u64, title: &str) -> Pr { + Pr { + number: n, + title: title.into(), + merged_at: "2026-09-01T00:00:00Z".into(), + merge_commit: None, + labels: vec![], + closing_issues_references: vec![], + } + } + + #[test] + fn fix_titles() { + for t in [ + "fix(guard): x", + "Fix x", + "feat: y — regression in z", + "cgp: wrong device", + ] { + assert!(is_fix_title(t), "{t}"); + } + for t in ["feat(apr): add serve", "docs: prefix", "chore: bump"] { + assert!(!is_fix_title(t), "{t}"); + } + } + + #[test] + fn reverts_name_their_targets() { + let prs = vec![ + pr(9, "Revert \"feat: a (#1234)\" (#1240)"), + pr(1, "feat #77"), + ]; + assert_eq!(reverted_prs(&prs), BTreeSet::from([1234, 1240])); + } + + #[test] + fn green_needs_both_required_checks_and_no_failure() { + let g = "ci / gate|SUCCESS\nworkspace-test|SUCCESS\nci / bench|SKIPPED\n"; + assert!(rollup_is_green(g)); + assert!(!rollup_is_green("ci / gate|SUCCESS\n")); + assert!(!rollup_is_green(&format!("{g}mutants|FAILURE\n"))); + assert!(!rollup_is_green("")); + } + + #[test] + fn p_sample_is_seeded_one_per_file_and_marker_free() { + let m = |name: &str, file: &str| { + Mutant { + name: name.into(), + file: file.into(), + package: "p".into(), + diff: format!("--- {file}\n+++ replace {name}\n@@ -1,4 +1,4 @@\n a\n-b\n+c /* ~ changed by cargo-mutants ~ */\n d\n"), + } + }; + let ms = vec![ + m("a1", "src/a.rs"), + m("a2", "src/a.rs"), + m("b1", "src/b.rs"), + m("t", "tests/t.rs"), + ]; + let p = p_candidates(&ms, "base", 10, 4354); + assert_eq!(p.len(), 2, "one per file, tests excluded"); + assert!(p + .iter() + .all(|(i, d)| !names_the_mutation(d) && i.class == Class::P)); + let ids = |v: &Vec<(Item, String)>| v.iter().map(|(i, _)| i.id.clone()).collect::>(); + assert_eq!(ids(&p), ids(&p_candidates(&ms, "base", 10, 4354))); + } +} diff --git a/crates/aprender-review-experiment/src/champion.rs b/crates/aprender-review-experiment/src/champion.rs new file mode 100644 index 0000000000..432cea6e72 --- /dev/null +++ b/crates/aprender-review-experiment/src/champion.rs @@ -0,0 +1,217 @@ +//! REX-11 champion/challenger promotion (§5.4), `review-champion-challenger-v1`. +//! +//! A challenger (any §5.3 candidate: a prompt version, retrieval few-shot, a +//! B2 checkpoint) replaces the champion only if all five §5.4 gates hold on +//! the current sealed test version: +//! 1. per-item correctness improves: McNemar exact, paired, one-sided, α; +//! 2. the precision point estimate is not lower than the champion's; +//! 3. H1 (invariance) and H2 (determinism) hold on the primary cell; +//! 4. the contamination contract is green: 0 test hashes in its training data; +//! 5. its p95 on the primary cell meets the §4 queue budget. +//! +//! Every gate fails closed: an unmeasured gate is a failed gate. Each decision +//! consumes one of the test version's [`EVALUATIONS_PER_VERSION`] evaluations +//! (§2.2); once they are spent the next challenger is **refused**, consumes +//! nothing, and the test set must rotate. A challenger that never ran is +//! refused too (R-6): it is not an evaluation. Demotion (§5.4): two +//! consecutive gold-labelled escapes where the champion said PASS and a voter +//! said FAIL trigger an HRQ and re-evaluation. + +use serde::{Deserialize, Serialize}; + +use crate::score::{correctness_pairs, score, Scored}; +use crate::stats::{discordant, mcnemar_exact_one_sided, ALPHA}; + +pub const SCHEME: &str = "review-champion-challenger-v1"; +/// §2.2 `[A]`: promotion decisions a sealed test version supports. +pub const EVALUATIONS_PER_VERSION: usize = 20; + +/// What serves the lane: promotion is a forjar pin bump of these shas. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Pin { + pub weights_sha256: String, + pub prompt_sha256: String, + pub adapter_sha256: Option, +} + +/// Everything §5.4 reads about one challenger. `None` is "not measured". +#[derive(Debug, Clone)] +pub struct Evidence<'a> { + pub id: &'a str, + /// The §5.3 tier (B1a, B1b, B2a…). + pub tier: &'a str, + pub pin: Pin, + pub test_version: &'a str, + pub challenger: &'a [Scored], + pub champion: &'a [Scored], + pub h1_holds: Option, + pub h2_holds: Option, + pub contamination_hits: Option, + pub p95_s: Option, + /// §4 step 5: the slowest voting lane's measured p95 on the same items. + pub queue_budget_p95_s: Option, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "lowercase")] +pub enum Outcome { + Promoted, + Rejected, + /// Not an evaluation: nothing consumed. + Refused, +} + +/// One ledger row. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Decision { + pub schema: String, + pub test_version: String, + pub challenger: String, + pub tier: String, + pub outcome: Outcome, + /// The test version's evaluations used after this row. + pub evaluations_used: usize, + pub champion_before: Pin, + pub pin: Pin, + pub mcnemar_b: u64, + pub mcnemar_c: u64, + pub mcnemar_p: Option, + pub precision_champion: Option, + pub precision_challenger: Option, + pub precision_delta: Option, + pub reasons: Vec, +} + +/// The serving champion: the pin of the last promotion, else the initial one. +#[must_use] +pub fn champion<'a>(ledger: &'a [Decision], initial: &'a Pin) -> &'a Pin { + ledger + .iter() + .rev() + .find(|d| d.outcome == Outcome::Promoted) + .map_or(initial, |d| &d.pin) +} + +/// Evaluations the ledger has spent on `test_version`. +#[must_use] +pub fn evaluations_used(ledger: &[Decision], test_version: &str) -> usize { + ledger + .iter() + .filter(|d| d.test_version == test_version && d.outcome != Outcome::Refused) + .count() +} + +fn gate(reasons: &mut Vec, ok: bool, why: impl FnOnce() -> String) { + if !ok { + reasons.push(why()); + } +} + +/// Decide one challenger against the ledger's champion. Pure: the caller +/// appends the returned row. +#[must_use] +pub fn decide(ledger: &[Decision], initial: &Pin, e: &Evidence<'_>) -> Decision { + let used = evaluations_used(ledger, e.test_version); + let champ = champion(ledger, initial).clone(); + let (sc, sh) = (score(e.challenger), score(e.champion)); + let (b, c) = discordant(&correctness_pairs(e.challenger, e.champion)); + let mut d = Decision { + schema: SCHEME.into(), + test_version: e.test_version.into(), + challenger: e.id.into(), + tier: e.tier.into(), + outcome: Outcome::Refused, + evaluations_used: used, + champion_before: champ.clone(), + pin: e.pin.clone(), + mcnemar_b: b, + mcnemar_c: c, + mcnemar_p: None, + precision_champion: sh.precision.value(), + precision_challenger: sc.precision.value(), + precision_delta: sc + .precision + .value() + .zip(sh.precision.value()) + .map(|(a, b)| a - b), + reasons: Vec::new(), + }; + let ran = e.challenger.iter().any(|s| s.verdict.executed()); + let refusals = [ + (used >= EVALUATIONS_PER_VERSION).then(|| { + format!("test version {} has spent its {EVALUATIONS_PER_VERSION} evaluations: rotate it (§2.2)", e.test_version) + }), + (!ran).then(|| "the challenger has no executed row on the test split (R-6)".to_string()), + e.champion.is_empty().then(|| "no champion rows on this test version".to_string()), + (e.pin == champ).then(|| "the challenger is the champion".to_string()), + ]; + d.reasons = refusals.into_iter().flatten().collect(); + if !d.reasons.is_empty() { + return d; + } + let p = mcnemar_exact_one_sided(b, c); + d.mcnemar_p = Some(p); + d.evaluations_used = used + 1; + let r = &mut d.reasons; + gate(r, p < ALPHA, || { + format!("1: McNemar one-sided p = {p:.4} (b={b}, c={c}) is not < {ALPHA}") + }); + gate(r, d.precision_delta.is_some_and(|x| x >= 0.0), || { + format!( + "2: precision {:?} is lower than the champion's {:?}, or unmeasured", + d.precision_challenger, d.precision_champion + ) + }); + gate(r, e.h1_holds == Some(true), || { + format!("3: H1 invariance {:?}", e.h1_holds) + }); + gate(r, e.h2_holds == Some(true), || { + format!("3: H2 determinism {:?}", e.h2_holds) + }); + gate(r, e.contamination_hits == Some(0), || { + format!( + "4: contamination hits {:?} (must be 0)", + e.contamination_hits + ) + }); + let fits = matches!((e.p95_s, e.queue_budget_p95_s), + (Some(p), Some(q)) if p.is_finite() && q.is_finite() && p <= q); + gate(r, fits, || { + format!( + "5: p95 {:?} s vs queue budget {:?} s", + e.p95_s, e.queue_budget_p95_s + ) + }); + d.outcome = if d.reasons.is_empty() { + Outcome::Promoted + } else { + Outcome::Rejected + }; + d +} + +/// A gold-labelled escape joined back to the PR that let it through (§5.2). +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct Escape { + pub pr: String, + pub gold: bool, + pub champion_passed: bool, + pub a_voter_failed: bool, +} + +/// §5.4 demotion: the first pair of consecutive gold escapes where the +/// champion said PASS and a voter said FAIL, if any. Silver rows are not +/// evidence and neither break nor extend a run. +#[must_use] +pub fn demotion(escapes: &[Escape]) -> Option<(String, String)> { + let gold: Vec<&Escape> = escapes.iter().filter(|e| e.gold).collect(); + gold.windows(2) + .find(|w| w.iter().all(|e| e.champion_passed && e.a_voter_failed)) + .map(|w| (w[0].pr.clone(), w[1].pr.clone())) +} + +#[cfg(test)] +#[path = "champion_tests.rs"] +mod tests; diff --git a/crates/aprender-review-experiment/src/champion_tests.rs b/crates/aprender-review-experiment/src/champion_tests.rs new file mode 100644 index 0000000000..bb4e37efd5 --- /dev/null +++ b/crates/aprender-review-experiment/src/champion_tests.rs @@ -0,0 +1,241 @@ +use super::*; +use crate::receipt::{NotRun, Verdict}; + +/// 60 test items: 0..30 defects, 30..60 good. +fn rows(verdict: impl Fn(usize) -> Verdict) -> Vec { + (0..60) + .map(|i| { + let v = verdict(i); + Scored { + id: format!("t{i:02}"), + class: if i < 30 { + crate::corpus::Class::P + } else { + crate::corpus::Class::G + }, + defect: i < 30, + verdict: v, + localized: false, + output_sha: v.executed().then(|| format!("o{i}")), + } + }) + .collect() +} + +/// The right answer on item `i`, or the wrong one. +fn answer(i: usize, right: bool) -> Verdict { + if (i < 30) == right { + Verdict::Fail + } else { + Verdict::Pass + } +} + +fn pin(w: &str) -> Pin { + Pin { + weights_sha256: w.into(), + prompt_sha256: "p0".into(), + adapter_sha256: None, + } +} + +/// Champion: flags defects 0..10 only, never a good item (40 correct, precision 1). +fn champ() -> Vec { + rows(|i| answer(i, !(10..30).contains(&i))) +} + +/// Challenger: flags every defect, no good item (60 correct; b = 20, c = 0). +fn better() -> Vec { + rows(|i| answer(i, true)) +} + +fn evidence<'a>(ch: &'a [Scored], cm: &'a [Scored]) -> Evidence<'a> { + Evidence { + id: "prompt-v2", + tier: "B1a", + pin: Pin { + prompt_sha256: "p2".into(), + ..pin("w0") + }, + test_version: "test-v1", + challenger: ch, + champion: cm, + h1_holds: Some(true), + h2_holds: Some(true), + contamination_hits: Some(0), + p95_s: Some(40.0), + queue_budget_p95_s: Some(60.0), + } +} + +#[test] +fn falsify_rxc_001_all_five_gates_promote_and_each_one_alone_rejects() { + let (ch, cm) = (better(), champ()); + let d = decide(&[], &pin("w0"), &evidence(&ch, &cm)); + assert_eq!(d.outcome, Outcome::Promoted, "{:?}", d.reasons); + assert_eq!((d.mcnemar_b, d.mcnemar_c, d.evaluations_used), (20, 0, 1)); + + let same = champ(); + let mut flips: Vec<(&str, Evidence<'_>)> = Vec::new(); + flips.push(( + "1", + Evidence { + challenger: &same, + ..evidence(&ch, &cm) + }, + )); + flips.push(( + "3: H1", + Evidence { + h1_holds: None, + ..evidence(&ch, &cm) + }, + )); + flips.push(( + "3: H2", + Evidence { + h2_holds: Some(false), + ..evidence(&ch, &cm) + }, + )); + flips.push(( + "4:", + Evidence { + contamination_hits: Some(1), + ..evidence(&ch, &cm) + }, + )); + flips.push(( + "4:", + Evidence { + contamination_hits: None, + ..evidence(&ch, &cm) + }, + )); + flips.push(( + "5:", + Evidence { + p95_s: Some(61.0), + ..evidence(&ch, &cm) + }, + )); + flips.push(( + "5:", + Evidence { + queue_budget_p95_s: None, + ..evidence(&ch, &cm) + }, + )); + for (gate, e) in flips { + let d = decide(&[], &pin("w0"), &e); + assert_eq!(d.outcome, Outcome::Rejected, "gate {gate}"); + assert_eq!(d.reasons.len(), 1, "gate {gate}: {:?}", d.reasons); + assert!(d.reasons[0].starts_with(gate), "{:?}", d.reasons); + assert_eq!( + d.evaluations_used, 1, + "a rejection still spends an evaluation" + ); + } +} + +#[test] +fn falsify_rxc_001_more_correct_but_less_precise_is_rejected() { + // Every defect flagged, plus 5 false FAILs: b = 20, c = 5, p ≈ 0.002. + let (ch, cm) = (rows(|i| answer(i, !(30..35).contains(&i))), champ()); + let d = decide(&[], &pin("w0"), &evidence(&ch, &cm)); + assert!(d.mcnemar_p.is_some_and(|p| p < ALPHA), "{:?}", d.mcnemar_p); + assert_eq!(d.outcome, Outcome::Rejected); + assert_eq!(d.reasons.len(), 1); + assert!(d.reasons[0].starts_with("2:"), "{:?}", d.reasons); + assert!(d.precision_delta.is_some_and(|x| x < 0.0)); +} + +#[test] +fn falsify_rxc_002_the_budget_counts_and_refusals_spend_nothing() { + let (ch, cm) = (better(), champ()); + let mut ledger = Vec::new(); + for n in 0..EVALUATIONS_PER_VERSION { + let e = Evidence { + h1_holds: Some(false), + ..evidence(&ch, &cm) + }; + let d = decide(&ledger, &pin("w0"), &e); + assert_eq!(d.evaluations_used, n + 1); + ledger.push(d); + } + let d = decide(&ledger, &pin("w0"), &evidence(&ch, &cm)); + assert_eq!(d.outcome, Outcome::Refused, "the 21st evaluation"); + assert_eq!(d.evaluations_used, EVALUATIONS_PER_VERSION); + let fresh = decide( + &ledger, + &pin("w0"), + &Evidence { + test_version: "test-v2", + ..evidence(&ch, &cm) + }, + ); + assert_eq!( + fresh.outcome, + Outcome::Promoted, + "a rotated version has its own budget" + ); + + let never = rows(|_| Verdict::NotRun(NotRun::NoDeclaredExecutor)); + let d = decide(&[], &pin("w0"), &evidence(&never, &cm)); + assert_eq!( + (d.outcome, d.evaluations_used), + (Outcome::Refused, 0), + "R-6" + ); + let d = decide( + &[], + &pin("w0"), + &Evidence { + pin: pin("w0"), + ..evidence(&ch, &cm) + }, + ); + assert_eq!( + d.outcome, + Outcome::Refused, + "the champion cannot challenge itself" + ); +} + +#[test] +fn falsify_rxc_002_promotion_moves_the_pin_and_rollback_is_the_previous_sha() { + let (ch, cm) = (better(), champ()); + let d = decide(&[], &pin("w0"), &evidence(&ch, &cm)); + let ledger = vec![d]; + assert_eq!(champion(&ledger, &pin("w0")).prompt_sha256, "p2"); + assert_eq!(ledger[0].champion_before, pin("w0"), "rollback target"); +} + +#[test] +fn falsify_rxc_003_two_consecutive_gold_escapes_demote() { + let esc = |pr: &str, gold: bool, hit: bool| Escape { + pr: pr.into(), + gold, + champion_passed: hit, + a_voter_failed: hit, + }; + assert_eq!( + demotion(&[ + esc("#1", true, true), + esc("#2", false, false), + esc("#3", true, true) + ]), + Some(("#1".into(), "#3".into())), + "a silver row neither breaks nor extends the run" + ); + assert_eq!( + demotion(&[ + esc("#1", true, true), + esc("#2", true, false), + esc("#3", true, true) + ]), + None, + "a gold escape the champion caught breaks the run" + ); + assert_eq!(demotion(&[esc("#1", true, true)]), None); +} diff --git a/crates/aprender-review-experiment/src/cluster.rs b/crates/aprender-review-experiment/src/cluster.rs new file mode 100644 index 0000000000..8432d4a1b4 --- /dev/null +++ b/crates/aprender-review-experiment/src/cluster.rs @@ -0,0 +1,179 @@ +//! Near-duplicate cluster check for sealed test items (PRA-001 T7, gate G-CON; +//! contract `review-corpus-contamination-v1`, FALSIFY-RCC-004..006). +//! +//! The exact checks in [`crate::contamination`] (diff sha, hunk sha) miss a +//! sealed diff that was re-indented or had an identifier renamed. This module +//! catches it with a bottom-k MinHash sketch over normalised token shingles: +//! +//! - **whitespace**: tokens are taken with all whitespace dropped, and blank +//! lines produce no tokens, so re-indenting or re-spacing changes nothing; +//! - **renames**: every identifier becomes `$` (the MOSS normalisation), so any +//! rename, even a partial one, yields the same token stream; keywords and +//! literals are kept, so `x += 1` and `x += 2` still differ. +//! +//! A sealed item's sketch is the `SKETCH` smallest shingle hashes. A text leaks +//! the item when at least `CONTAINMENT` of that sketch is among the text's own +//! shingle hashes: an unbiased estimate of how much of the item the text holds, +//! so a sealed diff embedded in a longer prompt is still seen. + +use std::collections::HashSet; + +/// Tokens per shingle. +pub const SHINGLE: usize = 12; +/// Hashes kept per sealed item (bottom-k). +pub const SKETCH: usize = 64; +/// Items with fewer distinct shingles are too generic to cluster on. +pub const MIN_SHINGLES: usize = 32; +/// Fraction of a sealed sketch that must appear in a text for a hit. +pub const CONTAINMENT: f64 = 0.7; +/// Header line of a rendered sketch file. +pub const SKETCH_SCHEME: &str = "review-corpus-v1 cluster-sketch shingle=12 k=64"; + +const HEADERS: &[&str] = &["diff --git", "index ", "--- ", "+++ ", "@@"]; + +const KEYWORDS: &[&str] = &[ + "as", "async", "await", "break", "const", "continue", "crate", "dyn", "else", "enum", "extern", + "false", "fn", "for", "if", "impl", "in", "let", "loop", "match", "mod", "move", "mut", "pub", + "ref", "return", "self", "Self", "static", "struct", "super", "trait", "true", "type", + "unsafe", "use", "where", "while", +]; + +/// Normalised tokens of one hunk body (or any text): whitespace dropped, a +/// `+`/`-` line marker kept as a token, identifiers replaced by `$`. +#[must_use] +pub fn tokens(body: &str) -> Vec { + let mut out = Vec::new(); + for line in body.lines() { + let (marker, rest) = match line.chars().next() { + Some(c @ ('+' | '-')) => (Some(c), &line[1..]), + Some(' ') => (None, &line[1..]), + _ => (None, line), + }; + if rest.trim().is_empty() { + continue; + } + out.extend(marker.map(String::from)); + let mut chars = rest.chars().peekable(); + while let Some(c) = chars.next() { + if c.is_whitespace() { + continue; + } + if c.is_alphanumeric() || c == '_' { + let mut w = String::from(c); + while let Some(&n) = chars.peek() { + if n.is_alphanumeric() || n == '_' { + w.push(n); + chars.next(); + } else { + break; + } + } + out.push(ident(w)); + } else { + out.push(c.to_string()); + } + } + } + out +} + +fn ident(w: String) -> String { + let lead = w.chars().next().unwrap_or('0'); + if lead.is_ascii_digit() || KEYWORDS.contains(&w.as_str()) { + w + } else { + "$".into() + } +} + +fn mix(mut z: u64) -> u64 { + z = (z ^ (z >> 30)).wrapping_mul(0xbf58_476d_1ce4_e5b9); + z = (z ^ (z >> 27)).wrapping_mul(0x94d0_49bb_1331_11eb); + z ^ (z >> 31) +} + +fn hash(shingle: &[String]) -> u64 { + let mut h: u64 = 0xcbf2_9ce4_8422_2325; + for t in shingle { + for b in t.bytes().chain([0xff]) { + h = (h ^ u64::from(b)).wrapping_mul(0x0100_0000_01b3); + } + } + mix(h) +} + +/// Distinct shingle hashes of a text. Diff header lines (`diff --git`, `index`, +/// `---`, `+++`, `@@`) are skipped, so a re-based or re-pathed copy matches; +/// every other line is tokenised, so a copy whose line markers were mangled (a +/// context space eaten by re-indenting) or that sits inside a prompt is seen. +#[must_use] +pub fn shingles(text: &str) -> HashSet { + let body: Vec<&str> = text + .lines() + .filter(|l| !HEADERS.iter().any(|h| l.starts_with(h))) + .collect(); + tokens(&body.join("\n")) + .windows(SHINGLE) + .map(hash) + .collect() +} + +/// Bottom-k sketch of a sealed diff, ascending; `None` below `MIN_SHINGLES`. +#[must_use] +pub fn sketch(diff: &str) -> Option> { + let mut s: Vec = shingles(diff).into_iter().collect(); + if s.len() < MIN_SHINGLES { + return None; + } + s.sort_unstable(); + s.truncate(SKETCH); + Some(s) +} + +/// Fraction of `sketch` present in `set` (0 for an empty sketch). +#[must_use] +pub fn containment(sketch: &[u64], set: &HashSet) -> f64 { + if sketch.is_empty() { + return 0.0; + } + let n = sketch.iter().filter(|h| set.contains(h)).count(); + n as f64 / sketch.len() as f64 +} + +/// Render sealed sketches: the scheme line, then `id hex,hex,…` sorted by id. +#[must_use] +pub fn render_sketches(items: &[(String, Vec)]) -> String { + let mut rows: Vec<&(String, Vec)> = items.iter().collect(); + rows.sort_by(|a, b| a.0.cmp(&b.0)); + let mut out = format!("{SKETCH_SCHEME}\n"); + for (id, s) in rows { + let hex: Vec = s.iter().map(|h| format!("{h:016x}")).collect(); + out.push_str(&format!("{id} {}\n", hex.join(","))); + } + out +} + +/// Parse a sketch file. `None` if the scheme line is missing or any row is +/// malformed: a sketch file that cannot be read clusters nothing. +#[must_use] +pub fn parse_sketches(text: &str) -> Option)>> { + let mut lines = text.lines(); + if lines.next()? != SKETCH_SCHEME { + return None; + } + lines + .filter(|l| !l.is_empty()) + .map(|l| { + let (id, hex) = l.split_once(' ')?; + let s = hex + .split(',') + .map(|h| u64::from_str_radix(h, 16).ok()) + .collect::>>()?; + Some((id.to_string(), s)) + }) + .collect() +} + +#[cfg(test)] +#[path = "cluster_tests.rs"] +mod tests; diff --git a/crates/aprender-review-experiment/src/cluster_tests.rs b/crates/aprender-review-experiment/src/cluster_tests.rs new file mode 100644 index 0000000000..1c97765236 --- /dev/null +++ b/crates/aprender-review-experiment/src/cluster_tests.rs @@ -0,0 +1,140 @@ +// `json!` expands to an `unwrap` of an infallible `to_value`. +#![allow(clippy::disallowed_methods)] + +use super::*; +use crate::contamination::{Hit, Index}; +use crate::corpus::{hunk_fingerprints, sha256_hex, Sealed}; +use crate::pool::{admit, local_tagged, Verdict}; + +const SEALED: &str = "--- a/crates/x/src/sum.rs\n+++ b/crates/x/src/sum.rs\n@@ -40,12 +40,14 @@ impl Ledger {\n pub fn total(&self, rows: &[Row]) -> u64 {\n let mut total = 0u64;\n for row in rows {\n- if row.amount > 0 {\n- total += row.amount;\n+ if row.amount > 0 && !row.voided {\n+ total = total.saturating_add(row.amount);\n }\n }\n+ debug_assert!(total >= self.floor);\n total\n }\n"; + +/// The same change as `SEALED`, re-indented with tabs, re-spaced, with blank +/// lines added, the local `total` renamed to `acc` (the method keeps its name, +/// so the rename is partial) and `row` to `r`, and re-based. +fn perturbed() -> String { + SEALED + .replace("@@ -40,12 +40,14 @@ impl Ledger {", "@@ -212,12 +219,14 @@") + .replace(" ", "\t") + .replace(" > 0", ">0") + .replace(" }\n", " }\n \n") + .replace("total", "acc") + .replace("row", "r") + .replace("fn acc", "fn total") +} + +const UNRELATED: &[&str] = &[ + "--- a/crates/y/src/parse.rs\n+++ b/crates/y/src/parse.rs\n@@ -8,9 +8,11 @@\n pub fn parse(s: &str) -> Result {\n let mut it = s.split(':');\n- let key = it.next().ok_or(Error::Empty)?;\n+ let key = it.next().ok_or(Error::Empty)?.trim();\n+ if key.is_empty() {\n+ return Err(Error::Empty);\n+ }\n let value = it.next().unwrap_or_default();\n Ok(Header { key: key.into(), value: value.into() })\n }\n", + "--- a/crates/z/src/cache.rs\n+++ b/crates/z/src/cache.rs\n@@ -30,8 +30,10 @@\n impl Cache {\n pub fn get(&mut self, k: &Key) -> Option<&Value> {\n- self.hits += 1;\n- self.map.get(k)\n+ let v = self.map.get(k);\n+ if v.is_some() { self.hits += 1 } else { self.misses += 1 }\n+ v\n }\n pub fn clear(&mut self) { self.map.clear(); }\n }\n", + "--- a/crates/w/src/sum.rs\n+++ b/crates/w/src/sum.rs\n@@ -40,12 +40,12 @@ impl Ledger {\n pub fn count(&self, rows: &[Row]) -> usize {\n let mut n = 0usize;\n for row in rows {\n- if row.kind == Kind::Debit {\n+ if matches!(row.kind, Kind::Debit | Kind::Fee) {\n n += 1;\n }\n }\n n\n }\n", +]; + +fn sealed_index() -> Index { + Index::new(&[Sealed { + id: "R042".into(), + diff_sha256: sha256_hex(SEALED.as_bytes()), + hunks: hunk_fingerprints(SEALED), + }]) +} + +fn sketches() -> Vec<(String, Vec)> { + vec![( + "R042".into(), + sketch(SEALED).expect("SEALED is above MIN_SHINGLES"), + )] +} + +/// FALSIFY-RCC-004 (G-CON): a sealed item with whitespace and rename +/// perturbation, inserted into a candidate pool, passes the exact-hash checks +/// and is caught once the cluster check lands: a `cluster` hit, and the row is +/// refused, bare and inside a JSONL field beside other prompt text. +#[test] +fn falsify_rcc_004_perturbed_sealed_item_is_a_cluster_hit() { + let p = perturbed(); + assert_ne!(p, SEALED); + assert!( + sealed_index().scan(&p, "pool").is_empty(), + "the perturbation must defeat the exact checks, or this test proves nothing" + ); + let ix = sealed_index().with_sketches(sketches()); + assert_eq!( + ix.scan(&p, "pool"), + vec![Hit { + item: "R042".into(), + file: "pool".into(), + how: "cluster" + }] + ); + let row = serde_json::json!({"lane": "sonnet-5", "input": format!("Review this diff.\n\n{p}\nBe terse.")}) + .to_string(); + let clean = local_tagged(serde_json::json!({"lane": "qwen", "input": UNRELATED[0]})); + let a = admit(&format!("{row}\n{clean}\n"), &ix); + assert_eq!( + a.verdicts[0].1, + Verdict::Refused { + items: vec!["R042".into()] + } + ); + assert_eq!(a.pool(), vec![clean.as_str()]); + // a backport that also gained one line is still the same cluster + let backport = p.replace("\t let mut", "+\t let _guard = lock();\n\t let mut"); + assert_ne!(backport, p); + assert_eq!(ix.scan(&backport, "pool").len(), 1); +} + +/// FALSIFY-RCC-005: unrelated diffs, including one over the same file shape +/// with the same loop skeleton, are not clustered with the sealed item, and a +/// diff too short to fingerprint has no sketch. +#[test] +fn falsify_rcc_005_unrelated_diffs_are_not_clustered() { + let ix = sealed_index().with_sketches(sketches()); + let s = &sketches()[0].1; + for u in UNRELATED { + let c = containment(s, &shingles(u)); + assert!(c < CONTAINMENT, "containment {c} for {u}"); + assert!(ix.scan(u, "pool").is_empty(), "{u}"); + } + let short = "@@ -1,4 +1,4 @@\n fn f(x: u32) -> bool {\n- x > 1\n+ x >= 1\n }\n"; + assert_eq!(sketch(short), None); +} + +/// FALSIFY-RCC-006: the sketch file round-trips, and a file that is not a +/// sketch file (wrong scheme, bad or missing hex) parses to `None`. +#[test] +fn falsify_rcc_006_sketch_file_round_trips_and_fails_closed() { + let items = sketches(); + assert_eq!(parse_sketches(&render_sketches(&items)), Some(items)); + for bad in [ + "", + "review-corpus-v1 test-manifest\nR1 00ff\n", + &format!("{SKETCH_SCHEME}\nR1 zz\n"), + &format!("{SKETCH_SCHEME}\nR1\n"), + &format!("{SKETCH_SCHEME}\nR1 \n"), + ] { + assert_eq!(parse_sketches(bad), None, "{bad:?}"); + } +} + +#[test] +fn tokens_case_table() { + let t = |s: &str| tokens(s).join(" "); + // whitespace and blank lines vanish; a +/- marker is a token + assert_eq!(t("+ x >= 1;\n \n"), "+ $ > = 1 ;"); + assert_eq!(t("+x>=1;"), "+ $ > = 1 ;"); + assert_eq!(t("+x;\n+\n-\t\n+y;"), t("+x;\n+y;")); + // any rename gives the same stream; keywords and literals stay + assert_eq!( + t(" let mut total = 0u64;\n+total += n;"), + t(" let mut acc = 0u64;\n+acc += n;") + ); + assert_eq!(t(" let mut total = 0u64;"), "let mut $ = 0u64 ;"); + assert_ne!(t("+x += 1;"), t("+x += 2;")); + assert_ne!(t("+x += 1;"), t("-x += 1;")); +} + +#[test] +fn sketch_is_bottom_k_and_perturbation_invariant() { + let s = sketch(SEALED).expect("sketch"); + assert!(s.len() <= SKETCH && s.len() >= MIN_SHINGLES); + assert!(s.windows(2).all(|w| w[0] < w[1])); + assert_eq!(sketch(&perturbed()), Some(s)); +} diff --git a/crates/aprender-review-experiment/src/contamination.rs b/crates/aprender-review-experiment/src/contamination.rs new file mode 100644 index 0000000000..742ab6613f --- /dev/null +++ b/crates/aprender-review-experiment/src/contamination.rs @@ -0,0 +1,132 @@ +//! REX-02 contamination check (contract `review-corpus-contamination-v1`, +//! spec §2.2 and rule R-2). +//! +//! A sealed test item leaks into a training file when that file carries the +//! item's diff sha as a literal, or carries any of the item's hunks (header +//! stripped, so a re-based copy still matches). JSON string values are scanned +//! after unescaping, so a diff stored in a JSONL `"diff"` field is seen as the +//! diff it is, not as one escaped line. +//! +//! With [`Index::with_sketches`], a re-spaced, renamed or lightly edited copy +//! is a `cluster` hit too (PRA-001 T7, [`crate::cluster`]). + +use crate::cluster::{containment, shingles, CONTAINMENT}; +use crate::corpus::{hunk_fingerprints, Sealed}; +use std::collections::BTreeMap; + +/// Directories that hold training data for this spec (B2 teacher sets, +/// challenger few-shot pools). Every file under them is scanned. +pub const TRAIN_ROOTS: &[&str] = &["docs/audits/review-corpus/train"]; + +/// One leak: which test item, which file, and how it was seen. +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)] +pub struct Hit { + pub item: String, + pub file: String, + pub how: &'static str, +} + +/// Index of the sealed items: diff sha → id, hunk fingerprint → id, and +/// (optionally) each item's near-dup sketch from [`crate::cluster`]. +#[derive(Debug, Default)] +pub struct Index { + shas: BTreeMap, + hunks: BTreeMap, + sketches: Vec<(String, Vec)>, +} + +impl Index { + #[must_use] + pub fn new(sealed: &[Sealed]) -> Self { + let mut ix = Self::default(); + for s in sealed { + ix.shas.insert(s.diff_sha256.clone(), s.id.clone()); + for h in &s.hunks { + ix.hunks.insert(h.clone(), s.id.clone()); + } + } + ix + } + + /// Add sealed near-dup sketches (`id`, bottom-k hashes), so a re-spaced or + /// renamed copy of a sealed diff is a `cluster` hit. + #[must_use] + pub fn with_sketches(mut self, sketches: Vec<(String, Vec)>) -> Self { + self.sketches = sketches; + self + } + + /// Number of sealed diff shas indexed (0 means the scan proves nothing). + #[must_use] + pub fn len(&self) -> usize { + self.shas.len() + } + + /// True when nothing is sealed. + #[must_use] + pub fn is_empty(&self) -> bool { + self.shas.is_empty() + } + + fn scan_str(&self, text: &str, file: &str, hits: &mut Vec) { + for (sha, id) in &self.shas { + if text.contains(sha.as_str()) { + hits.push(hit(id, file, "diff-sha")); + } + } + for fp in hunk_fingerprints(text) { + if let Some(id) = self.hunks.get(&fp) { + hits.push(hit(id, file, "hunk")); + } + } + if self.sketches.is_empty() { + return; + } + let set = shingles(text); + for (id, s) in &self.sketches { + if containment(s, &set) >= CONTAINMENT { + hits.push(hit(id, file, "cluster")); + } + } + } + + /// Every leak of a sealed item into `text` (the contents of `file`). + #[must_use] + pub fn scan(&self, text: &str, file: &str) -> Vec { + let mut hits = Vec::new(); + self.scan_str(text, file, &mut hits); + for line in text.lines() { + if let Ok(v) = serde_json::from_str::(line) { + let mut strings = Vec::new(); + collect_strings(&v, &mut strings); + for s in strings { + self.scan_str(s, file, &mut hits); + } + } + } + hits.sort(); + hits.dedup(); + hits + } +} + +fn hit(id: &str, file: &str, how: &'static str) -> Hit { + Hit { + item: id.to_string(), + file: file.to_string(), + how, + } +} + +fn collect_strings<'a>(v: &'a serde_json::Value, out: &mut Vec<&'a str>) { + match v { + serde_json::Value::String(s) => out.push(s), + serde_json::Value::Array(a) => a.iter().for_each(|x| collect_strings(x, out)), + serde_json::Value::Object(o) => o.values().for_each(|x| collect_strings(x, out)), + _ => {} + } +} + +#[cfg(test)] +#[path = "contamination_tests.rs"] +mod tests; diff --git a/crates/aprender-review-experiment/src/contamination_tests.rs b/crates/aprender-review-experiment/src/contamination_tests.rs new file mode 100644 index 0000000000..d3a3a6994d --- /dev/null +++ b/crates/aprender-review-experiment/src/contamination_tests.rs @@ -0,0 +1,98 @@ +// `json!` expands to an `unwrap` of an infallible `to_value`. +#![allow(clippy::disallowed_methods)] + +use super::*; +use crate::corpus::{hunk_fingerprints, parse_manifest, sha256_hex, Sealed}; +use std::path::{Path, PathBuf}; + +const TEST_DIFF: &str = "--- a/crates/x/src/a.rs\n+++ b/crates/x/src/a.rs\n@@ -10,5 +10,5 @@\n fn f(x: u32) -> bool {\n- x > 1\n+ x >= 1\n }\n \n"; + +fn index() -> Index { + Index::new(&[Sealed { + id: "R007".into(), + diff_sha256: sha256_hex(TEST_DIFF.as_bytes()), + hunks: hunk_fingerprints(TEST_DIFF), + }]) +} + +/// FALSIFY-RCC-001: a test diff planted into a JSONL training record is caught. +#[test] +fn falsify_rcc_001_planted_jsonl_leak_is_caught() { + let rec = serde_json::json!({"prompt": "review", "diff": TEST_DIFF, "label": "FAIL"}); + let hits = index().scan(&format!("{rec}\n"), "train/teacher.jsonl"); + assert_eq!( + hits, + vec![Hit { + item: "R007".into(), + file: "train/teacher.jsonl".into(), + how: "hunk" + }] + ); +} + +/// FALSIFY-RCC-002: a re-based copy (other line numbers) and a bare sha both leak. +#[test] +fn falsify_rcc_002_rebased_copy_and_sha_literal_leak() { + let rebased = TEST_DIFF.replace("@@ -10,5 +10,5 @@", "@@ -300,5 +301,5 @@ impl A"); + assert!(!index().scan(&rebased, "t.txt").is_empty()); + let sha = sha256_hex(TEST_DIFF.as_bytes()); + assert_eq!( + index().scan(&format!("seen {sha}\n"), "t.txt")[0].how, + "diff-sha" + ); +} + +/// A different diff over the same file is not a leak. +#[test] +fn unrelated_diff_is_clean() { + let other = TEST_DIFF.replace("x >= 1", "x == 1"); + assert!(index().scan(&other, "t.txt").is_empty()); +} + +fn files_under(dir: &Path, out: &mut Vec) { + let Ok(rd) = std::fs::read_dir(dir) else { + return; + }; + for e in rd.flatten() { + let p = e.path(); + if p.is_dir() { + files_under(&p, out); + } else { + out.push(p); + } + } +} + +/// FALSIFY-RCC-003: the committed training roots carry none of the sealed +/// test items. Seals a non-empty manifest only; before REX-02 seals, it is +/// the placeholder and there is nothing to leak. +#[test] +fn falsify_rcc_003_train_roots_are_clean() { + let manifest = include_str!("../../../docs/audits/review-corpus/test-manifest-v1.txt"); + let Some(sealed) = parse_manifest(manifest) else { + assert!( + manifest.contains("PLACEHOLDER (REX-00)"), + "manifest is neither sealed nor the placeholder" + ); + return; + }; + let ix = Index::new(&sealed); + assert!( + !ix.is_empty(), + "a sealed manifest with 0 items seals nothing" + ); + let root = Path::new(env!("CARGO_MANIFEST_DIR")).join("../.."); + let mut files = Vec::new(); + for r in TRAIN_ROOTS { + files_under(&root.join(r), &mut files); + } + let hits: Vec = files + .iter() + .filter_map(|f| Some((f, std::fs::read_to_string(f).ok()?))) + .flat_map(|(f, t)| ix.scan(&t, &f.display().to_string())) + .collect(); + assert!( + hits.is_empty(), + "sealed test items leaked into training data: {hits:#?}" + ); +} diff --git a/crates/aprender-review-experiment/src/corpus.rs b/crates/aprender-review-experiment/src/corpus.rs new file mode 100644 index 0000000000..edf2be9cbd --- /dev/null +++ b/crates/aprender-review-experiment/src/corpus.rs @@ -0,0 +1,414 @@ +//! REX-02 review corpus (contract `review-corpus-v1`, spec §2.2). +//! +//! Pure functions only: classifying paths, sizing and stratifying diffs, +//! sanitising planted-mutation diffs, locating the defect lines, assigning the +//! seeded dev/test split and rendering the sealed test manifest. The builder +//! that talks to git and gh lives in `examples/rex.rs`. + +use crate::stats::SplitMix64; +use serde::{Deserialize, Serialize}; + +/// Corpus scheme id; also the first line of the sealed manifest. +pub const SCHEME: &str = "review-corpus-v1"; +/// Share of each (class, stratum) cell that goes to `dev` (§2.2). +pub const DEV_SHARE: f64 = 0.3; +/// Token proxy `[A]`: one token per 4 diff bytes, rounded up. Strata are +/// decided on this proxy, never on a model tokenizer, so they do not move when +/// the model changes. +pub const BYTES_PER_TOKEN: u64 = 4; + +/// Item class (§2.2). +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)] +pub enum Class { + /// Planted: a mutation fixture applied as a diff. + P, + /// Real: a merged fix, reverse-applied. + R, + /// Good: a merged PR that stayed clean. + G, +} + +impl Class { + /// P and R carry a defect; G does not. + #[must_use] + pub fn is_defect(self) -> bool { + !matches!(self, Self::G) + } +} + +/// Diff-size stratum on the token proxy: S < 2k, M 2k–8k, L > 8k. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)] +pub enum Stratum { + S, + M, + L, +} + +/// Split (§2.2): `dev` may be read by prompt authors; `test` is sealed. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)] +pub enum Split { + Dev, + Test, +} + +/// A defect location: the file and the new-side line of the diff. +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)] +pub struct Loc { + pub file: String, + pub line: u32, +} + +/// One corpus item. `defect` is empty exactly for class G. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct Item { + pub id: String, + pub class: Class, + pub stratum: Stratum, + pub split: Split, + pub diff_sha256: String, + pub bytes: u64, + pub approx_tokens: u64, + /// Where the item came from (PR number + commit, or mutant identity). + pub source: String, + pub defect: Vec, +} + +impl Item { + /// Build an item from its diff text. The split starts as `Test` and is set + /// by [`assign_splits`]. + #[must_use] + pub fn new(id: String, class: Class, source: String, diff: &str) -> Self { + let bytes = diff.len() as u64; + let approx_tokens = bytes.div_ceil(BYTES_PER_TOKEN); + let defect = if class.is_defect() { + defect_locs(diff) + } else { + Vec::new() + }; + Self { + id, + class, + stratum: stratum(approx_tokens), + split: Split::Test, + diff_sha256: sha256_hex(diff.as_bytes()), + bytes, + approx_tokens, + source, + defect, + } + } +} + +/// Lowercase hex sha256. +#[must_use] +pub fn sha256_hex(bytes: &[u8]) -> String { + crate::prereg::sha256_hex(bytes) +} + +/// Stratum of a token count. +#[must_use] +pub fn stratum(tokens: u64) -> Stratum { + match tokens { + 0..2000 => Stratum::S, + 2000..=8000 => Stratum::M, + _ => Stratum::L, + } +} + +/// A file a reviewer is shown: Rust or shell source, not a test, bench or +/// example. Test hunks are excluded so that R items reintroduce the defect +/// without also deleting the test that names it. +#[must_use] +pub fn is_review_path(path: &str) -> bool { + let code = path.ends_with(".rs") || path.ends_with(".sh"); + let test_like = path.starts_with("tests/") + || ["/tests/", "/test/", "/benches/", "/examples/"] + .iter() + .any(|d| path.contains(d)) + || path.ends_with("_tests.rs") + || path.ends_with("_test.rs") + || path.ends_with("/tests.rs"); + code && !test_like +} + +/// Strip everything in a cargo-mutants diff that names the mutation: the +/// `+++ replace … with …` header and the `/* ~ changed by cargo-mutants ~ */` +/// marker. The result is a plain `a/`/`b/` unified diff of `file`. +#[must_use] +pub fn sanitize_mutant_diff(diff: &str, file: &str) -> String { + let mut out = format!("--- a/{file}\n+++ b/{file}\n"); + for line in diff.lines().skip_while(|l| !l.starts_with("@@")) { + out.push_str(&line.replace(" /* ~ changed by cargo-mutants ~ */", "")); + out.push('\n'); + } + out +} + +/// True when `text` still names the mutation (a leak into the prompt). +#[must_use] +pub fn names_the_mutation(text: &str) -> bool { + text.contains("cargo-mutants") || text.contains("+++ replace ") +} + +fn hunk_new_start(header: &str) -> Option { + let plus = header.split_whitespace().find(|t| t.starts_with('+'))?; + plus[1..].split(',').next()?.parse().ok() +} + +/// Defect locations: every added line (new-side line number); for a hunk that +/// only deletes, the new-side line where the deletion sits. +#[must_use] +pub fn defect_locs(diff: &str) -> Vec { + let mut locs = Vec::new(); + let mut file = String::new(); + let mut line = 0u32; + let mut hunk_added = false; + for l in diff.lines() { + if let Some(p) = l.strip_prefix("+++ ") { + file = p.strip_prefix("b/").unwrap_or(p).to_string(); + } else if l.starts_with("@@") { + line = hunk_new_start(l).unwrap_or(0); + hunk_added = false; + } else if l.starts_with('+') { + locs.push(Loc { + file: file.clone(), + line, + }); + line += 1; + hunk_added = true; + } else if l.starts_with('-') { + if !hunk_added && !l.starts_with("---") { + locs.push(Loc { + file: file.clone(), + line, + }); + hunk_added = true; + } + } else { + line += 1; + } + } + locs.sort(); + locs.dedup(); + locs +} + +fn is_comment(path: &str, changed: &str) -> bool { + let t = changed.trim(); + t.is_empty() || t.starts_with("//") || (path.ends_with(".sh") && t.starts_with('#')) +} + +/// Keep only the hunks that change code. A fix PR's comment-only hunks are not +/// the fix, and reverse-applying them labels a comment as the defect. Files +/// left with no hunk are dropped whole. +#[must_use] +pub fn drop_comment_only_hunks(diff: &str) -> String { + let mut out = String::new(); + let mut header = String::new(); + let mut hunk = String::new(); + let mut path = String::new(); + let mut code = false; + let mut flush = |header: &mut String, hunk: &mut String, code: bool| { + if code { + out.push_str(header); + header.clear(); + out.push_str(hunk); + } + hunk.clear(); + }; + for l in diff.lines() { + if l.starts_with("diff --git") { + flush(&mut header, &mut hunk, code); + header.clear(); + code = false; + header.push_str(l); + header.push('\n'); + } else if l.starts_with("@@") { + flush(&mut header, &mut hunk, code); + code = false; + hunk.push_str(l); + hunk.push('\n'); + } else if hunk.is_empty() { + if let Some(p) = l.strip_prefix("+++ ") { + path = p.to_string(); + } + header.push_str(l); + header.push('\n'); + } else { + let changed = l.strip_prefix(['+', '-']); + code |= changed.is_some_and(|c| !is_comment(&path, c)); + hunk.push_str(l); + hunk.push('\n'); + } + } + flush(&mut header, &mut hunk, code); + out +} + +/// Seeded, stratified split: within each (class, stratum) cell, items are +/// ordered by SplitMix64 keys drawn in id order, and the first +/// `round(DEV_SHARE × n)` become `dev`. +pub fn assign_splits(items: &mut [Item], seed: u64) { + items.sort_by(|a, b| a.id.cmp(&b.id)); + let mut rng = SplitMix64::new(seed); + let keys: Vec = items.iter().map(|_| rng.next_u64()).collect(); + let mut cells: Vec<(Class, Stratum)> = items.iter().map(|i| (i.class, i.stratum)).collect(); + cells.sort(); + cells.dedup(); + for cell in cells { + let mut idx: Vec = (0..items.len()) + .filter(|&i| (items[i].class, items[i].stratum) == cell) + .collect(); + idx.sort_by_key(|&i| keys[i]); + let dev = dev_count(idx.len()); + for (rank, &i) in idx.iter().enumerate() { + items[i].split = if rank < dev { Split::Dev } else { Split::Test }; + } + } +} + +/// `round(DEV_SHARE × n)`. +#[must_use] +pub fn dev_count(n: usize) -> usize { + #[allow( + clippy::cast_possible_truncation, + clippy::cast_sign_loss, + clippy::cast_precision_loss + )] + let d = (DEV_SHARE * n as f64).round() as usize; + d +} + +/// Normalised hunk bodies: each `@@` hunk's lines without its header, so a +/// diff re-based to other line numbers still matches. +#[must_use] +pub fn hunks(diff: &str) -> Vec { + let mut out = Vec::new(); + let mut cur: Option = None; + for l in diff.lines() { + if l.starts_with("@@") || l.starts_with("--- ") || l.starts_with("diff --git") { + out.extend(cur.take()); + if l.starts_with("@@") { + cur = Some(String::new()); + } + } else if let Some(c) = cur.as_mut() { + if l.starts_with([' ', '+', '-']) { + c.push_str(l.trim_end()); + c.push('\n'); + } + } + } + out.extend(cur); + out +} + +/// Minimum hunk length (lines) that is fingerprinted. Shorter hunks (`-}`/`+}`) +/// are too generic to prove a leak. +pub const MIN_HUNK_LINES: usize = 4; + +/// sha256 of every fingerprintable hunk of a diff, sorted and deduplicated. +#[must_use] +pub fn hunk_fingerprints(diff: &str) -> Vec { + let mut f: Vec = hunks(diff) + .iter() + .filter(|h| h.lines().count() >= MIN_HUNK_LINES && h.lines().any(|l| !l.starts_with(' '))) + .map(|h| sha256_hex(h.as_bytes())) + .collect(); + f.sort(); + f.dedup(); + f +} + +/// One sealed test item: id, diff sha, hunk fingerprints. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Sealed { + pub id: String, + pub diff_sha256: String, + pub hunks: Vec, +} + +/// Render the sealed manifest: the scheme line, then one line per test item +/// (`id diff_sha hunk,hunk,…`), sorted by id. +#[must_use] +pub fn render_manifest(sealed: &[Sealed]) -> String { + let mut rows: Vec<&Sealed> = sealed.iter().collect(); + rows.sort_by(|a, b| a.id.cmp(&b.id)); + let mut out = format!("{SCHEME} test-manifest\n"); + for s in rows { + out.push_str(&format!( + "{} {} {}\n", + s.id, + s.diff_sha256, + s.hunks.join(",") + )); + } + out +} + +/// Parse a sealed manifest. `None` if the scheme line is missing or a row is +/// malformed — a manifest that cannot be read seals nothing. +#[must_use] +pub fn parse_manifest(text: &str) -> Option> { + let mut lines = text.lines(); + if lines.next()? != format!("{SCHEME} test-manifest") { + return None; + } + lines + .filter(|l| !l.is_empty()) + .map(|l| { + let mut f = l.split(' '); + let id = f.next()?.to_string(); + let diff_sha256 = f.next()?.to_string(); + let hunks = f + .next() + .unwrap_or("") + .split(',') + .filter(|h| !h.is_empty()) + .map(str::to_string) + .collect(); + (diff_sha256.len() == 64).then_some(Sealed { + id, + diff_sha256, + hunks, + }) + }) + .collect() +} + +/// Corpus version: sha256 of the sealed manifest bytes. +#[must_use] +pub fn corpus_version(manifest: &str) -> String { + format!("{SCHEME}@{}", &sha256_hex(manifest.as_bytes())[..16]) +} + +/// Deterministic pick of up to `n` items, balanced across strata where the +/// population allows: candidates are shuffled per stratum with `seed`, then +/// taken round-robin S, M, L until `n` are chosen or all are exhausted. +#[must_use] +pub fn pick_balanced(mut cands: Vec, n: usize, seed: u64) -> Vec { + cands.sort_by(|a, b| a.id.cmp(&b.id)); + let mut rng = SplitMix64::new(seed); + let keys: Vec = cands.iter().map(|_| rng.next_u64()).collect(); + let mut order: Vec = (0..cands.len()).collect(); + order.sort_by_key(|&i| keys[i]); + let mut per: [Vec; 3] = [Vec::new(), Vec::new(), Vec::new()]; + for i in order { + per[cands[i].stratum as usize].push(i); + } + let mut chosen = Vec::new(); + let mut round = 0; + while chosen.len() < n && per.iter().any(|p| p.len() > round) { + for p in &per { + if chosen.len() < n && round < p.len() { + chosen.push(p[round]); + } + } + round += 1; + } + chosen.sort_unstable(); + chosen.into_iter().map(|i| cands[i].clone()).collect() +} + +#[cfg(test)] +#[path = "corpus_tests.rs"] +mod tests; diff --git a/crates/aprender-review-experiment/src/corpus_tests.rs b/crates/aprender-review-experiment/src/corpus_tests.rs new file mode 100644 index 0000000000..9367badf2b --- /dev/null +++ b/crates/aprender-review-experiment/src/corpus_tests.rs @@ -0,0 +1,164 @@ +use super::*; + +const MUTANT: &str = "--- crates/x/src/a.rs\n+++ replace f -> bool with true\n@@ -10,7 +10,7 @@\n fn f() -> bool {\n- x > 1\n+ true /* ~ changed by cargo-mutants ~ */\n }\n"; + +#[test] +fn sanitize_removes_every_mutation_marker() { + let s = sanitize_mutant_diff(MUTANT, "crates/x/src/a.rs"); + assert!(names_the_mutation(MUTANT)); + assert!(!names_the_mutation(&s), "{s}"); + assert!(s.starts_with("--- a/crates/x/src/a.rs\n+++ b/crates/x/src/a.rs\n@@ -10,7 +10,7 @@\n")); + assert!(s.contains("\n+ true\n")); +} + +#[test] +fn defect_locs_uses_new_side_lines() { + let s = sanitize_mutant_diff(MUTANT, "crates/x/src/a.rs"); + // @@ +10: line 10 is context, the '-' is skipped on the new side, '+' is line 11. + assert_eq!( + defect_locs(&s), + vec![Loc { + file: "crates/x/src/a.rs".into(), + line: 11 + }] + ); + let del = "--- a/b.rs\n+++ b/b.rs\n@@ -5,3 +5,2 @@\n a\n-b\n c\n"; + assert_eq!( + defect_locs(del), + vec![Loc { + file: "b.rs".into(), + line: 6 + }] + ); +} + +#[test] +fn review_paths_exclude_tests_benches_examples() { + assert!(is_review_path("crates/a/src/lib.rs")); + assert!(is_review_path("scripts/check.sh")); + for p in [ + "crates/a/tests/x.rs", + "tests/x.rs", + "crates/a/src/x_tests.rs", + "crates/a/src/tests.rs", + "crates/a/benches/b.rs", + "crates/a/examples/e.rs", + "README.md", + "contracts/a.yaml", + ] { + assert!(!is_review_path(p), "{p}"); + } +} + +#[test] +fn strata_boundaries() { + assert_eq!(stratum(1999), Stratum::S); + assert_eq!(stratum(2000), Stratum::M); + assert_eq!(stratum(8000), Stratum::M); + assert_eq!(stratum(8001), Stratum::L); + let it = Item::new("g1".into(), Class::G, "pr".into(), &"x".repeat(8001)); + assert_eq!((it.approx_tokens, it.stratum), (2001, Stratum::M)); + assert!(it.defect.is_empty()); +} + +fn items(n: usize, class: Class) -> Vec { + (0..n) + .map(|i| { + Item::new( + format!("{class:?}{i:03}"), + class, + String::new(), + &format!("d{i}"), + ) + }) + .collect() +} + +#[test] +fn split_is_stratified_seeded_and_stable() { + let mut a = items(50, Class::R); + a.extend(items(10, Class::G)); + let mut b = a.clone(); + b.reverse(); + assign_splits(&mut a, 4354); + assign_splits(&mut b, 4354); + assert_eq!(a, b, "input order must not matter"); + let dev = |c: Class| { + a.iter() + .filter(|i| i.class == c && i.split == Split::Dev) + .count() + }; + assert_eq!((dev(Class::R), dev(Class::G)), (15, 3)); + let mut c = a.clone(); + assign_splits(&mut c, 7); + assert_ne!(a, c, "a different seed must move the split"); +} + +#[test] +fn hunks_ignore_line_numbers() { + let a = "--- a/f\n+++ b/f\n@@ -1,4 +1,4 @@\n x\n-y\n+z\n w\n"; + let b = "--- a/f\n+++ b/f\n@@ -90,4 +91,4 @@ fn g()\n x\n-y\n+z\n w\n"; + assert_eq!(hunk_fingerprints(a), hunk_fingerprints(b)); + assert_eq!(hunk_fingerprints(a).len(), 1); + assert!(hunk_fingerprints("--- a/f\n+++ b/f\n@@ -1 +1 @@\n-}\n+}\n").is_empty()); +} + +#[test] +fn manifest_round_trips_and_rejects_garbage() { + let s = vec![Sealed { + id: "r1".into(), + diff_sha256: "a".repeat(64), + hunks: vec!["b".repeat(64)], + }]; + let m = render_manifest(&s); + assert_eq!(parse_manifest(&m), Some(s)); + assert_eq!(parse_manifest("nope\n"), None); + assert_eq!( + parse_manifest(&format!("{SCHEME} test-manifest\nr1 short x\n")), + None + ); + assert!(corpus_version(&m).starts_with("review-corpus-v1@")); +} + +#[test] +fn pick_balanced_round_robins_strata() { + let mut c = items(10, Class::R); + c.push(Item::new( + "m1".into(), + Class::R, + String::new(), + &"x".repeat(9000), + )); + c.push(Item::new( + "l1".into(), + Class::R, + String::new(), + &"x".repeat(40000), + )); + let p = pick_balanced(c.clone(), 4, 1); + assert_eq!(p.len(), 4); + assert!(p.iter().any(|i| i.stratum == Stratum::M) && p.iter().any(|i| i.stratum == Stratum::L)); + assert_eq!(p, pick_balanced(c.clone(), 4, 1)); + assert_eq!(pick_balanced(c, 99, 1).len(), 12); +} + +#[test] +fn comment_only_hunks_are_not_the_fix() { + let d = "diff --git a/a.rs b/a.rs\nindex 1..2 100644\n--- a/a.rs\n+++ b/a.rs\n@@ -1,3 +1,3 @@\n //! doc\n-//! old path\n+//! new path\n@@ -20,3 +20,3 @@\n fn f() {\n- x > 1\n+ x >= 1\n }\ndiff --git a/b.rs b/b.rs\n--- a/b.rs\n+++ b/b.rs\n@@ -5,2 +5,2 @@\n-// a\n+// b\n"; + let k = drop_comment_only_hunks(d); + assert_eq!( + k, + "diff --git a/a.rs b/a.rs\nindex 1..2 100644\n--- a/a.rs\n+++ b/a.rs\n@@ -20,3 +20,3 @@\n fn f() {\n- x > 1\n+ x >= 1\n }\n" + ); + assert_eq!( + defect_locs(&k), + vec![Loc { + file: "a.rs".into(), + line: 21 + }] + ); + assert_eq!( + drop_comment_only_hunks("--- a/s.sh\n+++ b/s.sh\n@@ -1 +1 @@\n-# x\n+# y\n"), + "" + ); +} diff --git a/crates/aprender-review-experiment/src/datacard.rs b/crates/aprender-review-experiment/src/datacard.rs new file mode 100644 index 0000000000..9841629413 --- /dev/null +++ b/crates/aprender-review-experiment/src/datacard.rs @@ -0,0 +1,987 @@ +//! PRA-001 T14: `trace-datacard-v1` — the Croissant 1.0 + RAI card and the +//! Datasheet for one snapshot of the `agent-trace-v1` index (spec §2.11). +//! +//! A snapshot is the index's bytes: every `index/agent-trace-v1/YYYY/MM/DD.jsonl` +//! under the traces root, in path order, bound by a `sha256sum`-format manifest. +//! The card is generated from that snapshot and never hand-edited: +//! [`write_snapshot`] refuses to overwrite a card that differs from its +//! regeneration. The card carries aggregate counts only — no row value +//! (finding text, diff, sha, PR body) ever reaches it (PRIVATE, §6). +//! +//! The official Croissant validator (`mlcroissant`) is Python, which S-PY +//! forbids, so [`validate`] is a structural Croissant 1.0 + RAI 1.0 checker +//! in Rust: context, dataset type, conformance, resources, record set, fields, +//! references and the RAI fields the spec requires. + +use std::collections::{BTreeMap, BTreeSet}; +use std::fs; +use std::io::Write as _; +use std::path::{Path, PathBuf}; + +use serde_json::{json, Map, Value}; + +pub use crate::corpus::sha256_hex; + +pub const SCHEME: &str = "trace-datacard-v1"; +/// The index, relative to the traces root (`/almacen/traces`). +pub const INDEX_DIR: &str = "index/agent-trace-v1"; +/// Where cards land, relative to the traces root: `datacard//`. +pub const DATACARD_DIR: &str = "datacard"; +pub const ROW_SCHEMA: &str = "agent-trace-v1"; +pub const CROISSANT: &str = "http://mlcommons.org/croissant/1.0"; +pub const CROISSANT_RAI: &str = "http://mlcommons.org/croissant/RAI/1.0"; + +/// The RAI fields §2.11 requires (collection, labelling, limitations, use, PII). +pub const RAI_FIELDS: [&str; 10] = [ + "rai:dataCollection", + "rai:dataCollectionType", + "rai:dataCollectionTimeframe", + "rai:dataAnnotationProtocol", + "rai:dataPreprocessingProtocol", + "rai:dataUseCases", + "rai:dataLimitations", + "rai:dataBiases", + "rai:personalSensitiveInformation", + "rai:dataReleaseMaintenancePlan", +]; + +const DATA_TYPES: [&str; 8] = [ + "sc:Text", + "sc:Integer", + "sc:Float", + "sc:Boolean", + "sc:Date", + "sc:DateTime", + "sc:URL", + "sc:Number", +]; + +/// Composition columns counted per snapshot: (name, JSON path). +const COUNTED: [(&str, &[&str]); 7] = [ + ("provider", &["provider"]), + ("lane", &["lane"]), + ("label_tier", &["label_tier"]), + ("outcome", &["outcome"]), + ("split", &["split_guard", "split"]), + ("parse_status", &["parse_status"]), + ("secret_scan", &["secret_scan", "status"]), +]; + +/// The record-set fields: (name, jsonPath, dataType). Row-level free text +/// (`findings[].text`) is deliberately not described as a field. +const FIELDS: [(&str, &str, &str); 24] = [ + ("trace_id", "$.trace_id", "sc:Text"), + ("quorum_id", "$.quorum_id", "sc:Text"), + ("round", "$.round", "sc:Integer"), + ("lane", "$.lane", "sc:Text"), + ("counted", "$.counted", "sc:Boolean"), + ("provider", "$.provider", "sc:Text"), + ("model_id", "$.model_id", "sc:Text"), + ("access_channel", "$.access_channel", "sc:Text"), + ("repo", "$.repo", "sc:Text"), + ("pr", "$.pr", "sc:Integer"), + ("head_sha", "$.head_sha", "sc:Text"), + ("prompt_version", "$.prompt_version", "sc:Text"), + ("lane_state", "$.lane_state", "sc:Text"), + ("verdict", "$.verdict", "sc:Text"), + ("parse_status", "$.parse_status", "sc:Text"), + ("label_tier", "$.label_tier", "sc:Text"), + ("outcome", "$.outcome", "sc:Text"), + ("split", "$.split_guard.split", "sc:Text"), + ("group_key", "$.split_guard.group_key", "sc:Text"), + ("secret_scan_status", "$.secret_scan.status", "sc:Text"), + ("trace_status", "$.trace_status", "sc:Text"), + ("agreed", "$.agreed", "sc:Boolean"), + ("lane_disagreement", "$.lane_disagreement", "sc:Integer"), + ("dedup_cluster", "$.dedup_cluster", "sc:Text"), +]; + +/// Card identity that is not derived from the index. +#[derive(Debug, Clone)] +pub struct CardMeta { + pub name: String, + pub license: String, + pub url: String, +} + +impl Default for CardMeta { + fn default() -> Self { + Self { + name: "agent-trace-v1".into(), + license: "LicenseRef-paiml-private: not for public release; \ + gold rows with provider local only may be released (spec §2.11)" + .into(), + url: "https://github.com/paiml/aprender/blob/main/docs/specifications/\ + research-design-pr-agent-qwen-3.5.md" + .into(), + } + } +} + +/// One index file: its path relative to the traces root, sha256 and row count. +#[derive(Debug, Clone)] +pub struct IndexFile { + pub rel: String, + pub sha256: String, + pub bytes: u64, + pub rows: u64, +} + +/// One snapshot of the index: its files and aggregate counts, nothing else. +#[derive(Debug)] +pub struct Snapshot { + /// First 16 hex of the manifest's sha256. + pub id: String, + pub rows: u64, + /// Rows a public release may carry: gold, provider local, secret scan clean. + pub public_eligible: u64, + pub files: Vec, + counts: BTreeMap<(String, String), u64>, + /// Per index day: the G16 yield inputs. Shas only, never row content. + days: BTreeMap, +} + +/// One index day's yield: rows, distinct quorums, index bytes, blob shas. +#[derive(Debug, Default)] +struct DayYield { + rows: u64, + quorums: BTreeSet, + index_bytes: u64, + blobs: BTreeSet, +} + +impl Snapshot { + /// Read every index file under `root`. A bad line or a foreign schema is an + /// error naming `file:line` — never a skipped row. + pub fn read(root: &Path) -> Result { + let mut paths = Vec::new(); + walk(&root.join(INDEX_DIR), &mut paths)?; + paths.sort(); + if paths.is_empty() { + return Err(format!( + "no index files under {}", + root.join(INDEX_DIR).display() + )); + } + let mut snap = Snapshot { + id: String::new(), + rows: 0, + public_eligible: 0, + files: Vec::new(), + counts: BTreeMap::new(), + days: BTreeMap::new(), + }; + for p in paths { + snap.add_file(root, &p)?; + } + snap.id = sha256_hex(snap.manifest().as_bytes())[..16].to_string(); + Ok(snap) + } + + fn add_file(&mut self, root: &Path, p: &Path) -> Result<(), String> { + let rel = p + .strip_prefix(root) + .map_err(|e| format!("{}: {e}", p.display()))? + .to_string_lossy() + .into_owned(); + let day = day_of(&rel)?; + let bytes = fs::read(p).map_err(|e| format!("{rel}: {e}"))?; + let text = std::str::from_utf8(&bytes).map_err(|e| format!("{rel}: not utf-8: {e}"))?; + let mut rows = 0; + for (n, line) in text.lines().enumerate() { + if line.trim().is_empty() { + continue; + } + let row: Value = serde_json::from_str(line) + .map_err(|e| format!("{rel}:{}: bad json: {e}", n + 1))?; + let schema = row + .get("schema") + .and_then(Value::as_str) + .unwrap_or(""); + if schema != ROW_SCHEMA { + return Err(format!( + "{rel}:{}: schema {schema:?} is not {ROW_SCHEMA}", + n + 1 + )); + } + self.add_row(&row); + self.add_yield(&day, &row); + rows += 1; + } + self.days.entry(day).or_default().index_bytes += bytes.len() as u64; + self.files.push(IndexFile { + rel, + sha256: sha256_hex(&bytes), + bytes: bytes.len() as u64, + rows, + }); + Ok(()) + } + + fn add_row(&mut self, row: &Value) { + self.rows += 1; + for (col, path) in COUNTED { + let v = path + .iter() + .try_fold(row, |v, k| v.get(k)) + .and_then(Value::as_str) + .unwrap_or(""); + *self.counts.entry((col.into(), v.into())).or_default() += 1; + } + if public_eligible(row) { + self.public_eligible += 1; + } + } + + fn add_yield(&mut self, day: &str, row: &Value) { + let d = self.days.entry(day.to_string()).or_default(); + d.rows += 1; + if let Some(q) = row.get("quorum_id").and_then(Value::as_str) { + d.quorums.insert(q.to_string()); + } + for k in BLOB_SHAS { + if let Some(h) = row.get(k).and_then(Value::as_str).filter(|h| is_sha256(h)) { + d.blobs.insert(h.to_string()); + } + } + } + + /// `sha256sum` format, one line per index file, in path order. + pub fn manifest(&self) -> String { + self.files + .iter() + .map(|f| format!("{} {}\n", f.sha256, f.rel)) + .collect() + } + + pub fn count(&self, col: &str, val: &str) -> u64 { + self.counts + .get(&(col.to_string(), val.to_string())) + .copied() + .unwrap_or(0) + } + + /// (value, count) for one composition column, value order. + pub fn column(&self, col: &str) -> Vec<(&str, u64)> { + self.counts + .iter() + .filter(|((c, _), _)| c == col) + .map(|((_, v), n)| (v.as_str(), *n)) + .collect() + } + + /// First and last index day, `YYYY-MM-DD`. + pub fn timeframe(&self) -> (String, String) { + let days: Vec = self + .files + .iter() + .filter_map(|f| day_of(&f.rel).ok()) + .collect(); + ( + days.first().cloned().unwrap_or_default(), + days.last().cloned().unwrap_or_default(), + ) + } +} + +/// Row columns that address a CAS blob (`blobs/sha256/ab/cd/.zst`). +const BLOB_SHAS: [&str; 3] = ["input_sha", "output_sha", "logits_sha"]; + +fn is_sha256(h: &str) -> bool { + h.len() == 64 && h.bytes().all(|b| b.is_ascii_hexdigit()) +} + +/// §2.11: HF publication is gold-and-local-only; anthropic/google rows never, +/// and a row with any secret-scan status but `clean` never. +fn public_eligible(row: &Value) -> bool { + let s = |k: &str| row.get(k).and_then(Value::as_str); + s("label_tier") == Some("gold") + && s("provider") == Some("local") + && row.pointer("/secret_scan/status").and_then(Value::as_str) == Some("clean") +} + +fn walk(dir: &Path, out: &mut Vec) -> Result<(), String> { + if !dir.exists() { + return Ok(()); + } + for e in fs::read_dir(dir).map_err(|e| format!("{}: {e}", dir.display()))? { + let p = e.map_err(|e| format!("{}: {e}", dir.display()))?.path(); + if p.is_dir() { + walk(&p, out)?; + } else if p.extension().is_some_and(|x| x == "jsonl") { + out.push(p); + } + } + Ok(()) +} + +/// `index/agent-trace-v1/YYYY/MM/DD.jsonl` -> `YYYY-MM-DD`. +fn day_of(rel: &str) -> Result { + let tail = rel + .strip_prefix(INDEX_DIR) + .and_then(|t| t.strip_prefix('/')) + .and_then(|t| t.strip_suffix(".jsonl")) + .ok_or_else(|| format!("{rel}: not under {INDEX_DIR}/"))?; + let parts: Vec<&str> = tail.split('/').collect(); + let ok = parts.len() == 3 + && [4, 2, 2] + .iter() + .zip(&parts) + .all(|(n, p)| p.len() == *n && p.bytes().all(|b| b.is_ascii_digit())); + if !ok { + return Err(format!("{rel}: want {INDEX_DIR}/YYYY/MM/DD.jsonl")); + } + Ok(parts.join("-")) +} + +// `json!` expands to an `unwrap` of an infallible `to_value` on strings, hence the scoped allow. +#[allow(clippy::disallowed_methods)] +fn context() -> Value { + json!({ + "@language": "en", "@vocab": "https://schema.org/", + "citeAs": "cr:citeAs", "column": "cr:column", "conformsTo": "dct:conformsTo", + "cr": "http://mlcommons.org/croissant/", "rai": "http://mlcommons.org/croissant/RAI/", + "data": {"@id": "cr:data", "@type": "@json"}, + "dataType": {"@id": "cr:dataType", "@type": "@vocab"}, + "dct": "http://purl.org/dc/terms/", + "examples": {"@id": "cr:examples", "@type": "@json"}, + "extract": "cr:extract", "field": "cr:field", "fileProperty": "cr:fileProperty", + "fileObject": "cr:fileObject", "fileSet": "cr:fileSet", "format": "cr:format", + "includes": "cr:includes", "isLiveDataset": "cr:isLiveDataset", + "jsonPath": "cr:jsonPath", "key": "cr:key", "md5": "cr:md5", + "parentField": "cr:parentField", "path": "cr:path", "recordSet": "cr:recordSet", + "references": "cr:references", "regex": "cr:regex", "repeated": "cr:repeated", + "replace": "cr:replace", "sc": "https://schema.org/", "separator": "cr:separator", + "source": "cr:source", "subField": "cr:subField", "transform": "cr:transform", + }) +} + +// `json!` expands to an `unwrap` of an infallible `to_value` on strings, hence the scoped allow. +#[allow(clippy::disallowed_methods)] +fn fields() -> Vec { + FIELDS + .iter() + .map(|(name, path, ty)| { + json!({ + "@type": "cr:Field", + "@id": format!("agent-trace/{name}"), + "name": name, + "dataType": ty, + "source": {"fileSet": {"@id": "index-days"}, "extract": {"jsonPath": path}}, + }) + }) + .collect() +} + +fn composition(s: &Snapshot) -> String { + COUNTED + .iter() + .map(|(col, _)| { + let parts: Vec = s + .column(col) + .iter() + .map(|(v, n)| format!("{v}={n}")) + .collect(); + format!("{col}: {}", parts.join(", ")) + }) + .collect::>() + .join("; ") +} + +/// The Croissant 1.0 + RAI 1.0 card for one snapshot. Deterministic: the same +/// index bytes give the same card bytes (no clock is read). +#[allow(clippy::disallowed_methods)] // `json!` over strings, as above +pub fn croissant(s: &Snapshot, m: &CardMeta) -> Value { + let (first, last) = s.timeframe(); + let mut card = Map::new(); + card.insert("@context".into(), context()); + card.insert("@type".into(), json!("sc:Dataset")); + card.insert("conformsTo".into(), json!([CROISSANT, CROISSANT_RAI])); + card.insert("name".into(), json!(m.name)); + card.insert( + "description".into(), + json!(format!( + "PR-review lane traces (agent-trace-v1), snapshot {} of {} rows in {} daily \ + index files, {first}..{last}. Composition — {}.", + s.id, + s.rows, + s.files.len(), + composition(s) + )), + ); + card.insert("license".into(), json!(m.license)); + card.insert("url".into(), json!(m.url)); + card.insert("version".into(), json!(format!("0.0.0+{}", s.id))); + card.insert("datePublished".into(), json!(last)); + card.insert("isLiveDataset".into(), json!(true)); + card.insert( + "distribution".into(), + json!([ + { + "@type": "cr:FileObject", + "@id": "index-manifest", + "name": "index-manifest", + "description": "sha256sum manifest of every index file in this snapshot", + "contentUrl": "manifest.sha256", + "encodingFormat": "text/plain", + "sha256": sha256_hex(s.manifest().as_bytes()), + }, + { + "@type": "cr:FileSet", + "@id": "index-days", + "name": "index-days", + "description": "daily agent-trace-v1 index files under the traces root", + "encodingFormat": "application/jsonlines", + "includes": format!("{INDEX_DIR}/*/*/*.jsonl"), + }, + ]), + ); + card.insert( + "recordSet".into(), + json!([{ + "@type": "cr:RecordSet", + "@id": "agent-trace", + "name": "agent-trace", + "description": "one row per review lane per quorum round", + "key": {"@id": "agent-trace/trace_id"}, + "field": fields(), + }]), + ); + for (k, v) in rai(s, &first, &last) { + card.insert(k.into(), v); + } + Value::Object(card) +} + +// `json!` expands to an `unwrap` of an infallible `to_value` on strings, hence the scoped allow. +#[allow(clippy::disallowed_methods)] +fn rai(s: &Snapshot, first: &str, last: &str) -> [(&'static str, Value); 10] { + [ + ( + "rai:dataCollection", + json!( + "Captured at dispatch by the quorum harness: every review lane's full input and \ + output is written to the almacen CAS and indexed as one agent-trace-v1 row per \ + lane per round (spec §2.2). Rows are appended, never edited." + ), + ), + ( + "rai:dataCollectionType", + json!(["Machine-generated", "Software Collection"]), + ), + ("rai:dataCollectionTimeframe", json!([first, last])), + ( + "rai:dataAnnotationProtocol", + json!(format!( + "label_tier: silver = quorum-derived verdict; gold = the PR's matured outcome \ + (merged, reverted within 14 days, regression escape, closed unmerged); pending \ + until outcome_matured_at; quarantined on a secret-scan hit or parse failure. \ + This snapshot: {}.", + counts_of(s, "label_tier") + )), + ), + ( + "rai:dataPreprocessingProtocol", + json!( + "Rows are deduplicated by dedup_cluster and split by split_guard.group_key \ + (per-PR grouping, so no PR crosses train/val/test/sealed). Secret scanners run \ + before a row is indexed; this card is generated from the index by a Rust \ + binary and never hand-edited." + ), + ), + ( + "rai:dataUseCases", + json!( + "Training and evaluating a local PR-review model (Qwen 3.5) against the \ + quorum lanes; measuring lane agreement and outcome prediction. Not for \ + ranking or evaluating people." + ), + ), + ( + "rai:dataLimitations", + json!(format!( + "Single organisation's repositories and review conventions; silver labels \ + inherit quorum-lane bias; outcomes mature over 14 days so recent rows are \ + pending. Split composition: {}.", + counts_of(s, "split") + )), + ), + ( + "rai:dataBiases", + json!(format!( + "Lane and provider mix is set by quorum policy, not sampled: {}; {}.", + counts_of(s, "provider"), + counts_of(s, "lane") + )), + ), + ( + "rai:personalSensitiveInformation", + json!(format!( + "Rows may contain source code, commit authorship and review text. Every row is \ + secret-scanned; a hit sets secret_scan.status and the row is quarantined and \ + never released. Transcripts and trace blobs never enter a public repo, issue \ + or PR. Secret-scan status: {}.", + counts_of(s, "secret_scan") + )), + ), + ( + "rai:dataReleaseMaintenancePlan", + json!(format!( + "Private. A card is regenerated per index snapshot under datacard//. \ + Public (HF) release is gold rows with provider local only — provider anthropic \ + and google rows are excluded from any public release. Public-eligible rows in \ + this snapshot: {} of {}.", + s.public_eligible, s.rows + )), + ), + ] +} + +fn counts_of(s: &Snapshot, col: &str) -> String { + let parts: Vec = s + .column(col) + .iter() + .map(|(v, n)| format!("{v}={n}")) + .collect(); + format!("{col} {}", parts.join(", ")) +} + +/// Structural Croissant 1.0 + RAI 1.0 check. Empty means valid. +pub fn validate(card: &Value) -> Vec { + let mut errs = Vec::new(); + let Some(obj) = card.as_object() else { + return vec!["card is not a JSON object".into()]; + }; + check_context(obj, &mut errs); + check_dataset(obj, &mut errs); + let ids = check_ids(card, &mut errs); + check_distribution(obj, &ids, &mut errs); + check_record_sets(obj, &ids, &mut errs); + for k in RAI_FIELDS { + if !nonempty(obj.get(k)) { + errs.push(format!("{k} is missing or empty (RAI 1.0)")); + } + } + errs +} + +fn nonempty(v: Option<&Value>) -> bool { + match v { + Some(Value::String(s)) => !s.trim().is_empty(), + Some(Value::Array(a)) => !a.is_empty() && a.iter().all(|x| nonempty(Some(x))), + _ => false, + } +} + +fn check_context(obj: &Map, errs: &mut Vec) { + let want = context(); + let Some(ctx) = obj.get("@context").and_then(Value::as_object) else { + errs.push("@context is missing or not an object".into()); + return; + }; + for (k, v) in want.as_object().into_iter().flatten() { + match ctx.get(k) { + None => errs.push(format!("@context missing key {k}")), + Some(got) if got != v => errs.push(format!("@context key {k} is {got}, want {v}")), + Some(_) => {} + } + } +} + +fn check_dataset(obj: &Map, errs: &mut Vec) { + let s = |k: &str| obj.get(k).and_then(Value::as_str).unwrap_or(""); + if s("@type") != "sc:Dataset" { + errs.push(format!("@type is {:?}, want sc:Dataset", obj.get("@type"))); + } + let conforms: Vec<&str> = match obj.get("conformsTo") { + Some(Value::String(c)) => vec![c.as_str()], + Some(Value::Array(a)) => a.iter().filter_map(Value::as_str).collect(), + _ => Vec::new(), + }; + for want in [CROISSANT, CROISSANT_RAI] { + if !conforms.contains(&want) { + errs.push(format!("conformsTo does not list {want}")); + } + } + for k in ["name", "description", "license", "url", "datePublished"] { + if s(k).trim().is_empty() { + errs.push(format!("{k} is missing or empty")); + } + } +} + +/// Every `@id` defined by a resource, record set or field; duplicates are errors. +fn check_ids(card: &Value, errs: &mut Vec) -> BTreeSet { + let mut ids = BTreeSet::new(); + let defs = card["distribution"] + .as_array() + .into_iter() + .flatten() + .chain(card["recordSet"].as_array().into_iter().flatten()) + .chain( + card["recordSet"] + .as_array() + .into_iter() + .flatten() + .flat_map(|r| r["field"].as_array().into_iter().flatten()), + ); + for d in defs { + match d.get("@id").and_then(Value::as_str) { + None => errs.push(format!("{} has no @id", d["@type"])), + Some(id) if !ids.insert(id.to_string()) => errs.push(format!("duplicate @id {id}")), + Some(_) => {} + } + } + ids +} + +fn check_distribution(obj: &Map, ids: &BTreeSet, errs: &mut Vec) { + let dist = obj.get("distribution").and_then(Value::as_array); + if dist.is_none_or(|d| d.is_empty()) { + errs.push("distribution is missing or empty".into()); + } + for r in dist.into_iter().flatten() { + let id = r["@id"].as_str().unwrap_or("?"); + let has = |k: &str| { + r.get(k) + .and_then(Value::as_str) + .is_some_and(|s| !s.is_empty()) + }; + match r["@type"].as_str() { + Some("cr:FileObject") => { + for k in ["contentUrl", "encodingFormat"] { + if !has(k) { + errs.push(format!("FileObject {id}: {k} is missing")); + } + } + let sha = r["sha256"].as_str().unwrap_or(""); + if sha.len() != 64 || !sha.bytes().all(|b| b.is_ascii_hexdigit()) { + errs.push(format!("FileObject {id}: sha256 is missing or not 64 hex")); + } + } + Some("cr:FileSet") => { + for k in ["includes", "encodingFormat"] { + if !has(k) { + errs.push(format!("FileSet {id}: {k} is missing")); + } + } + } + t => errs.push(format!( + "distribution {id}: @type {t:?} is not cr:FileObject/cr:FileSet" + )), + } + if let Some(c) = r.get("containedIn") { + check_ref(c, ids, &format!("{id}.containedIn"), errs); + } + } +} + +fn check_ref(v: &Value, ids: &BTreeSet, at: &str, errs: &mut Vec) { + match v.get("@id").and_then(Value::as_str) { + Some(id) if ids.contains(id) => {} + other => errs.push(format!("{at}: unknown resource {other:?}")), + } +} + +fn check_record_sets(obj: &Map, ids: &BTreeSet, errs: &mut Vec) { + let sets = obj.get("recordSet").and_then(Value::as_array); + if sets.is_none_or(|s| s.is_empty()) { + errs.push("recordSet is missing or empty".into()); + } + for rs in sets.into_iter().flatten() { + let rid = rs["@id"].as_str().unwrap_or("?"); + if rs["@type"] != "cr:RecordSet" { + errs.push(format!("recordSet {rid}: @type is not cr:RecordSet")); + } + if let Some(k) = rs.get("key") { + check_ref(k, ids, &format!("{rid}.key"), errs); + } + let fields = rs["field"].as_array(); + if fields.is_none_or(|f| f.is_empty()) { + errs.push(format!("recordSet {rid}: field is missing or empty")); + } + for f in fields.into_iter().flatten() { + check_field(f, ids, errs); + } + } +} + +fn check_field(f: &Value, ids: &BTreeSet, errs: &mut Vec) { + let fid = f["@id"].as_str().unwrap_or("?"); + if f["@type"] != "cr:Field" { + errs.push(format!("field {fid}: @type is not cr:Field")); + } + let ty = f["dataType"].as_str().unwrap_or(""); + if !DATA_TYPES.contains(&ty) { + errs.push(format!("field {fid}: dataType {ty:?} is not a known type")); + } + let src = &f["source"]; + match (src.get("fileSet"), src.get("fileObject")) { + (Some(r), _) | (None, Some(r)) => check_ref(r, ids, &format!("{fid}.source"), errs), + (None, None) => errs.push(format!("field {fid}: source names no resource")), + } + let path = src["extract"]["jsonPath"].as_str().unwrap_or(""); + if !path.starts_with('$') { + errs.push(format!( + "field {fid}: extract.jsonPath {path:?} does not start with $" + )); + } +} + +pub const WEEKLY_SCHEME: &str = "trace-weekly-receipt-v1"; +/// G16 `[O]`: 50–200 quorums/day and at most ~0.45 TB/yr raw. The operator's +/// band, not ours; the receipt turns G16 `[U]` into `[V]` or names what is missing. +pub const G16_QUORUMS_PER_DAY: (u64, u64) = (50, 200); +pub const G16_RAW_TB_PER_YEAR: f64 = 0.45; + +/// `YYYY-MM-DD` -> ISO 8601 week `YYYY-Www` (the year of the week's Thursday). +pub fn iso_week(day: &str) -> Result { + let p: Vec = day + .split('-') + .map(str::parse) + .collect::>() + .map_err(|e| format!("{day}: {e}"))?; + let [y, m, d] = p[..] else { + return Err(format!("{day}: want YYYY-MM-DD")); + }; + let n = days_from_civil(y, m, d); + let thursday = n - (n + 3).rem_euclid(7) + 3; + let year = (y - 1..=y + 1) + .rev() + .find(|&yy| days_from_civil(yy, 1, 1) <= thursday) + .unwrap_or(y); + let week = (thursday - days_from_civil(year, 1, 1)) / 7 + 1; + Ok(format!("{year:04}-W{week:02}")) +} + +/// Days since 1970-01-01 (proleptic Gregorian; H. Hinnant's algorithm). +fn days_from_civil(y: i64, m: i64, d: i64) -> i64 { + let y = if m <= 2 { y - 1 } else { y }; + let era = y.div_euclid(400); + let yoe = y - era * 400; + let doy = (153 * (m + if m > 2 { -3 } else { 9 }) + 2) / 5 + d - 1; + era * 146_097 + yoe * 365 + yoe / 4 - yoe / 100 + doy - 719_468 +} + +/// The uncompressed size a zstd frame header declares (RFC 8878 §3.1.1.1), +/// or None when the header does not carry it. Reads the first frame only; +/// the blob store writes one frame per blob. +pub fn zstd_content_size(head: &[u8]) -> Option { + if head.get(..4)? != [0x28, 0xB5, 0x2F, 0xFD] { + return None; + } + let fhd = *head.get(4)?; + let single = fhd & 0x20 != 0; + let at = 5 + usize::from(!single) + [0, 1, 2, 4][usize::from(fhd & 3)]; + let n = match (fhd >> 6, single) { + (0, true) => 1, + (0, false) => return None, + (1, _) => 2, + (2, _) => 4, + _ => 8, + }; + let v = head + .get(at..at + n)? + .iter() + .rev() + .fold(0u64, |a, b| (a << 8) | u64::from(*b)); + Some(if n == 2 { v + 256 } else { v }) +} + +/// One blob's (stored bytes, declared raw bytes); None when it is absent. +fn blob_size(root: &Path, sha: &str) -> Option<(u64, Option)> { + use std::io::Read as _; + let p = root + .join("blobs/sha256") + .join(&sha[..2]) + .join(&sha[2..4]) + .join(format!("{sha}.zst")); + let mut f = fs::File::open(p).ok()?; + let stored = f.metadata().ok()?.len(); + let mut head = [0u8; 18]; + let got = f.read(&mut head).ok()?; + Some((stored, zstd_content_size(&head[..got]))) +} + +#[derive(Default)] +struct WeekYield { + days: Vec, + rows: u64, + index_bytes: u64, + blobs: BTreeSet, +} + +/// The weekly yield/bytes receipt (PRM-C14, G16): per ISO week, quorums/day, +/// rows, index bytes and the CAS blobs the rows address, stored and raw. +/// A week with a missing blob, or a blob whose raw size is undeclared, reads +/// `unmeasured` — never `within`. +#[allow(clippy::disallowed_methods)] // `json!` over numbers and strings +pub fn weekly(s: &Snapshot, root: &Path) -> Result { + let mut weeks: BTreeMap = BTreeMap::new(); + for (day, d) in &s.days { + let w = weeks.entry(iso_week(day)?).or_default(); + w.days.push(d.quorums.len() as u64); + w.rows += d.rows; + w.index_bytes += d.index_bytes; + w.blobs.extend(d.blobs.iter().cloned()); + } + let rows: Vec = weeks.iter().map(|(k, w)| week_row(root, k, w)).collect(); + Ok(json!({ + "schema": WEEKLY_SCHEME, + "snapshot": s.id, + "g16": { + "quorums_per_day": [G16_QUORUMS_PER_DAY.0, G16_QUORUMS_PER_DAY.1], + "raw_tb_per_year_max": G16_RAW_TB_PER_YEAR, + "provenance": "[O] PRM-001 G16", + }, + "weeks": rows, + })) +} + +#[allow(clippy::disallowed_methods)] // `json!` over numbers and strings +fn week_row(root: &Path, week: &str, w: &WeekYield) -> Value { + let (mut stored, mut raw, mut missing, mut undeclared) = (0u64, 0u64, 0u64, 0u64); + for sha in &w.blobs { + match blob_size(root, sha) { + None => missing += 1, + Some((st, r)) => { + stored += st; + match r { + Some(r) => raw += r, + None => undeclared += 1, + } + } + } + } + let n = w.days.len() as u64; + let (lo, hi) = ( + w.days.iter().copied().min().unwrap_or(0), + w.days.iter().copied().max().unwrap_or(0), + ); + let raw_total = w.index_bytes + raw; + let tb_year = raw_total as f64 / n.max(1) as f64 * 365.0 / 1e12; + let verdict = if missing + undeclared > 0 { + "unmeasured" + } else if lo >= G16_QUORUMS_PER_DAY.0 + && hi <= G16_QUORUMS_PER_DAY.1 + && tb_year <= G16_RAW_TB_PER_YEAR + { + "within" + } else { + "outside" + }; + json!({ + "week": week, + "days_with_rows": n, + "rows": w.rows, + "quorums": w.days.iter().sum::(), + "quorums_per_day_min": lo, + "quorums_per_day_max": hi, + "index_bytes": w.index_bytes, + "blobs": w.blobs.len(), + "blobs_missing": missing, + "blobs_raw_undeclared": undeclared, + "blob_bytes_stored": stored, + "blob_bytes_raw": raw, + "raw_tb_per_year": tb_year, + "g16": verdict, + }) +} + +/// Deterministic JSON bytes for a card. +pub fn render(card: &Value) -> String { + let mut s = serde_json::to_string_pretty(card).unwrap_or_default(); + s.push('\n'); + s +} + +/// The Datasheet-style card (Gebru et al. sections), aggregate counts only. +pub fn datasheet(s: &Snapshot, m: &CardMeta) -> String { + let (first, last) = s.timeframe(); + let mut o = format!( + "# Datasheet: {} (snapshot {})\n\nGenerated by `{SCHEME}` from the index; do not edit.\n\n", + m.name, s.id + ); + o += "## Motivation\n\nPR-review lane traces for training and evaluating a local \ + review model against the quorum lanes (PRA-001).\n\n"; + o += &format!( + "## Composition\n\n- rows: {}\n- index files: {} ({first}..{last})\n- public-eligible \ + (gold, provider local): {}\n", + s.rows, + s.files.len(), + s.public_eligible + ); + for (col, _) in COUNTED { + for (v, n) in s.column(col) { + o += &format!("- {col} `{v}`: {n}\n"); + } + } + o += "\n## Collection process\n\nCaptured at dispatch, one agent-trace-v1 row per lane \ + per quorum round; full lane I/O lives in the almacen CAS, never in this card.\n\n"; + o += "## Preprocessing / labelling\n\nsilver = quorum verdict; gold = matured PR outcome; \ + pending until matured; quarantined on a secret-scan hit. Split by per-PR group key; \ + deduplicated by cluster.\n\n"; + o += "## Uses\n\nTraining and evaluating the local review model; lane-agreement research. \ + Not for evaluating people.\n\n"; + o += &format!( + "## Distribution\n\nPrivate. Public release carries gold rows with provider local only \ + ({} of {} rows in this snapshot); anthropic and google rows are never released.\n\n", + s.public_eligible, s.rows + ); + o += "## Maintenance\n\nRegenerated per index snapshot under `datacard//`; a card \ + that differs from its regeneration is refused as hand-edited.\n"; + o +} + +/// Write `datacard//{croissant.json, datasheet.md, manifest.sha256, weekly.json}`. +/// Regenerating an unchanged snapshot is a no-op; a card that differs from its +/// regeneration is refused as hand-edited and never overwritten. +pub fn write_snapshot(root: &Path, m: &CardMeta) -> Result { + let s = Snapshot::read(root)?; + let card = croissant(&s, m); + let errs = validate(&card); + if !errs.is_empty() { + return Err(format!( + "generated card fails validation: {}", + errs.join("; ") + )); + } + let dir = root.join(DATACARD_DIR).join(&s.id); + let files = [ + ("croissant.json", render(&card)), + ("datasheet.md", datasheet(&s, m)), + ("manifest.sha256", s.manifest()), + ("weekly.json", render(&weekly(&s, root)?)), + ]; + for (name, body) in &files { + let p = dir.join(name); + match fs::read(&p) { + Ok(got) if got != body.as_bytes() => { + return Err(format!( + "{} is hand-edited (differs from its regeneration); refusing to overwrite", + p.display() + )) + } + _ => {} + } + } + fs::create_dir_all(&dir).map_err(|e| format!("{}: {e}", dir.display()))?; + for (name, body) in &files { + let p = dir.join(name); + if p.exists() { + continue; + } + let mut f = fs::OpenOptions::new() + .write(true) + .create_new(true) + .open(&p) + .map_err(|e| format!("{}: {e}", p.display()))?; + f.write_all(body.as_bytes()) + .map_err(|e| format!("{}: {e}", p.display()))?; + } + Ok(dir) +} + +#[cfg(test)] +#[path = "datacard_tests.rs"] +mod tests; diff --git a/crates/aprender-review-experiment/src/datacard_tests.rs b/crates/aprender-review-experiment/src/datacard_tests.rs new file mode 100644 index 0000000000..81b8a1135b --- /dev/null +++ b/crates/aprender-review-experiment/src/datacard_tests.rs @@ -0,0 +1,435 @@ +// `json!` expands to an `unwrap` of an infallible `to_value`. +#![allow(clippy::disallowed_methods)] + +use super::*; + +use std::fs; +use std::path::PathBuf; + +/// A throwaway traces root under the target dir's tmp; removed on drop. +struct Root(PathBuf); + +impl Root { + fn new(tag: &str) -> Self { + let p = std::env::temp_dir().join(format!("tdc-{tag}-{}", std::process::id())); + let _ = fs::remove_dir_all(&p); + fs::create_dir_all(&p).expect("mkdir root"); + Root(p) + } +} + +impl Drop for Root { + fn drop(&mut self) { + let _ = fs::remove_dir_all(&self.0); + } +} + +/// Synthetic rows only: no real transcript or trace ever enters a fixture. +const SECRET_TEXT: &str = "SENTINEL-FINDING-TEXT-never-in-a-card"; +const SECRET_DIFF: &str = "0badc0de0badc0de0badc0de0badc0de0badc0de0badc0de0badc0de0badc0de"; + +fn row(id: &str, lane: &str, provider: &str, tier: &str, split: &str) -> String { + serde_json::json!({ + "schema": "agent-trace-v1", + "trace_id": id, + "quorum_id": "q1", + "round": 1, + "lane": lane, + "counted": provider != "local", + "provider": provider, + "model_id": format!("{lane}-2026-09-01"), + "access_channel": if provider == "local" { "apr-serve" } else { "claude-code-cli" }, + "repo": "paiml/aprender", + "pr": 4400, + "head_sha": "a".repeat(40), + "diff_sha256": SECRET_DIFF, + "input_sha": "b".repeat(64), + "output_sha": "c".repeat(64), + "prompt_version": "1.0.0", + "lane_state": "Verdict", + "verdict": "approve", + "findings": [{"file": "src/a.rs", "line_start": 1, "line_end": 2, + "severity": "low", "category": "style", "text": SECRET_TEXT}], + "parse_status": "ok", + "label_tier": tier, + "outcome": "merged", + "split_guard": {"group_key": "paiml/aprender#4400", "split": split}, + "secret_scan": {"status": "clean", "hits": 0}, + "trace_status": "ok", + "agreed": true, + "lane_disagreement": 0, + "unique_diff": true, + "dedup_cluster": "c0", + }) + .to_string() +} + +fn day(root: &Path, ymd: (&str, &str, &str), rows: &[String]) { + let d = root.join(INDEX_DIR).join(ymd.0).join(ymd.1); + fs::create_dir_all(&d).expect("mkdir day"); + let mut body = rows.join("\n"); + body.push('\n'); + fs::write(d.join(format!("{}.jsonl", ymd.2)), body).expect("write day"); +} + +fn fixture(tag: &str) -> Root { + let r = Root::new(tag); + day( + &r.0, + ("2026", "09", "24"), + &[ + row("t1", "sonnet", "anthropic", "gold", "train"), + row("t2", "agy", "google", "gold", "train"), + ], + ); + day( + &r.0, + ("2026", "09", "25"), + &[ + row("t3", "qwen-shadow", "local", "gold", "val"), + row("t4", "qwen-shadow", "local", "silver", "train"), + row("t5", "haiku", "anthropic", "pending", "train"), + ], + ); + r +} + +fn card_of(root: &Path) -> (Snapshot, Value) { + let s = Snapshot::read(root).expect("snapshot"); + let c = croissant(&s, &CardMeta::default()); + (s, c) +} + +#[test] +fn falsify_tdc_001_the_generated_card_passes_croissant_validation() { + let r = fixture("v"); + let (_, c) = card_of(&r.0); + assert_eq!(validate(&c), Vec::::new(), "{c:#}"); + + // The validator is not vacuous: each plant is refused, and names why. + let plants: [(&str, fn(&mut Value)); 9] = [ + ("conformsTo", |c| { + c.as_object_mut().expect("obj").remove("conformsTo"); + }), + ("name", |c| c["name"] = json!("")), + ("@type", |c| c["@type"] = json!("sc:CreativeWork")), + ("sha256", |c| { + c["distribution"][0] + .as_object_mut() + .expect("obj") + .remove("sha256"); + }), + ("unknown resource", |c| { + c["recordSet"][0]["field"][0]["source"]["fileSet"]["@id"] = json!("nope") + }), + ("duplicate @id", |c| { + let f = c["recordSet"][0]["field"][0].clone(); + c["recordSet"][0]["field"] + .as_array_mut() + .expect("arr") + .push(f); + }), + ("dataType", |c| { + c["recordSet"][0]["field"][0]["dataType"] = json!("sc:Banana") + }), + ("rai:dataLimitations", |c| { + c.as_object_mut() + .expect("obj") + .remove("rai:dataLimitations"); + }), + ("@context", |c| { + c["@context"].as_object_mut().expect("obj").remove("rai"); + }), + ]; + for (why, plant) in plants { + let mut bad = c.clone(); + plant(&mut bad); + let errs = validate(&bad); + assert!( + errs.iter().any(|e| e.contains(why)), + "plant {why:?} not refused: {errs:?}" + ); + } +} + +#[test] +fn falsify_tdc_002_a_snapshot_is_deterministic_and_bound_to_the_index_bytes() { + let r = fixture("det"); + let (s1, c1) = card_of(&r.0); + let (s2, c2) = card_of(&r.0); + assert_eq!(s1.id, s2.id); + assert_eq!(render(&c1), render(&c2), "same index, same bytes"); + assert_eq!( + datasheet(&s1, &CardMeta::default()), + datasheet(&s2, &CardMeta::default()) + ); + + // The root digest is over every index file's bytes. + let manifest = s1.manifest(); + assert_eq!(manifest.lines().count(), 2, "{manifest}"); + for line in manifest.lines() { + let (sha, rel) = line.split_once(" ").expect("sha path"); + let bytes = fs::read(r.0.join(rel)).expect("read"); + assert_eq!(sha, sha256_hex(&bytes), "{rel}"); + } + assert_eq!( + c1["distribution"][0]["sha256"], + json!(sha256_hex(manifest.as_bytes())) + ); + + // One appended row is a new snapshot. + day( + &r.0, + ("2026", "09", "26"), + &[row("t6", "sonnet", "anthropic", "silver", "test")], + ); + let (s3, c3) = card_of(&r.0); + assert_ne!(s1.id, s3.id); + assert_ne!( + c1["distribution"][0]["sha256"], + c3["distribution"][0]["sha256"] + ); + assert_eq!(s3.rows, 6); +} + +#[test] +fn falsify_tdc_003_only_gold_local_rows_are_public_eligible() { + let r = fixture("pub"); + let (s, c) = card_of(&r.0); + // t1 gold anthropic, t2 gold google: out. t3 gold local: in. t4 silver local, + // t5 pending anthropic: out. + assert_eq!(s.public_eligible, 1); + assert_eq!(s.count("provider", "anthropic"), 2); + assert_eq!(s.count("provider", "google"), 1); + assert_eq!(s.count("provider", "local"), 2); + assert_eq!(s.count("label_tier", "gold"), 3); + assert_eq!(s.count("split", "val"), 1); + let sheet = datasheet(&s, &CardMeta::default()); + assert!( + sheet.contains("public-eligible (gold, provider local): 1"), + "{sheet}" + ); + assert!(c["rai:personalSensitiveInformation"] + .as_str() + .expect("str") + .contains("quarantine")); + + // A secret-hit gold local row is never eligible. + let mut hit: Value = + serde_json::from_str(&row("t7", "qwen-shadow", "local", "gold", "train")).expect("json"); + hit["secret_scan"] = json!({"status": "hit", "hits": 1}); + day(&r.0, ("2026", "09", "27"), &[hit.to_string()]); + let s = Snapshot::read(&r.0).expect("snapshot"); + assert_eq!(s.public_eligible, 1); +} + +#[test] +fn falsify_tdc_004_a_hand_edited_card_is_refused_and_regeneration_is_idempotent() { + let r = fixture("edit"); + let out = write_snapshot(&r.0, &CardMeta::default()).expect("first write"); + assert!(out.join("croissant.json").is_file()); + assert!(out.join("datasheet.md").is_file()); + assert!(out.join("manifest.sha256").is_file()); + assert!(out.join("weekly.json").is_file()); + assert_eq!(out.parent().expect("parent"), r.0.join(DATACARD_DIR)); + + // Regenerating the same snapshot is a no-op, not an error. + assert_eq!( + write_snapshot(&r.0, &CardMeta::default()).expect("again"), + out + ); + + // A hand edit is refused on the next regeneration, never overwritten. + let p = out.join("croissant.json"); + let edited = fs::read_to_string(&p) + .expect("read") + .replace("agent-trace", "agent-trice"); + fs::write(&p, &edited).expect("edit"); + let e = write_snapshot(&r.0, &CardMeta::default()).expect_err("hand edit refused"); + assert!(e.contains("hand-edited"), "{e}"); + assert_eq!( + fs::read_to_string(&p).expect("read"), + edited, + "never overwritten" + ); +} + +#[test] +fn falsify_tdc_005_no_row_content_reaches_the_card() { + let r = fixture("priv"); + let (s, c) = card_of(&r.0); + let text = format!("{}\n{}", render(&c), datasheet(&s, &CardMeta::default())); + assert!( + !text.contains(SECRET_TEXT), + "a finding's text leaked into the card" + ); + assert!( + !text.contains(SECRET_DIFF), + "a diff sha leaked into the card" + ); + assert!( + !text.contains(&"c".repeat(64)), + "an output sha leaked into the card" + ); +} + +#[test] +fn falsify_tdc_006_a_malformed_index_is_refused_not_skipped() { + let r = fixture("bad"); + day(&r.0, ("2026", "09", "28"), &["{not json".to_string()]); + let e = Snapshot::read(&r.0).expect_err("bad json refused"); + assert!( + e.contains("2026/09/28.jsonl:1") && e.contains("bad json"), + "{e}" + ); + + let r = fixture("schema"); + let foreign = row("t9", "sonnet", "anthropic", "gold", "train") + .replace("agent-trace-v1", "agent-trace-v2"); + day(&r.0, ("2026", "09", "28"), &[foreign]); + let e = Snapshot::read(&r.0).expect_err("unknown major refused"); + assert!(e.contains("agent-trace-v2"), "{e}"); + + let empty = Root::new("empty"); + let e = Snapshot::read(&empty.0).expect_err("an empty index is not a dataset"); + assert!(e.contains("no index"), "{e}"); +} + +fn rowq(id: &str, quorum: &str) -> String { + let mut v: Value = + serde_json::from_str(&row(id, "sonnet", "anthropic", "silver", "train")).expect("json"); + v["quorum_id"] = json!(quorum); + v.to_string() +} + +/// A zstd frame header declaring `raw` bytes (single segment, 4-byte FCS). +fn blob(root: &Path, sha: &str, head: &[u8]) { + let d = root.join("blobs/sha256").join(&sha[..2]).join(&sha[2..4]); + fs::create_dir_all(&d).expect("mkdir blob"); + let mut b = head.to_vec(); + b.extend_from_slice(&[0u8; 10]); + fs::write(d.join(format!("{sha}.zst")), b).expect("write blob"); +} + +const ZSTD: [u8; 4] = [0x28, 0xB5, 0x2F, 0xFD]; + +fn declared(raw: u32) -> Vec { + let mut h = ZSTD.to_vec(); + h.push(0xA0); + h.extend_from_slice(&raw.to_le_bytes()); + h +} + +#[test] +fn falsify_tdc_007_the_weekly_receipt_measures_g16_or_says_unmeasured() { + let r = Root::new("weekly"); + let (b, c) = ("b".repeat(64), "c".repeat(64)); + day( + &r.0, + ("2026", "09", "24"), + &[rowq("t1", "q1"), rowq("t2", "q1"), rowq("t3", "q2")], + ); + day(&r.0, ("2026", "09", "25"), &[rowq("t4", "q3")]); + day(&r.0, ("2026", "09", "28"), &[rowq("t5", "q4")]); + blob(&r.0, &b, &declared(1000)); + + let s = Snapshot::read(&r.0).expect("snapshot"); + let w = weekly(&s, &r.0).expect("weekly"); + assert_eq!(w["schema"], json!(WEEKLY_SCHEME)); + let w39 = &w["weeks"][0]; + assert_eq!(w39["week"], json!("2026-W39")); + assert_eq!(w39["days_with_rows"], json!(2)); + assert_eq!(w39["rows"], json!(4)); + assert_eq!(w39["quorums"], json!(3)); + assert_eq!(w39["quorums_per_day_min"], json!(1)); + assert_eq!(w39["quorums_per_day_max"], json!(2)); + let idx: u64 = s.files[..2].iter().map(|f| f.bytes).sum(); + assert_eq!(w39["index_bytes"], json!(idx)); + assert_eq!(w39["blobs"], json!(2)); + assert_eq!(w39["blobs_missing"], json!(1)); + assert_eq!(w39["blob_bytes_stored"], json!(19)); + assert_eq!(w39["blob_bytes_raw"], json!(1000)); + assert_eq!( + w39["g16"], + json!("unmeasured"), + "a missing blob is not a measurement" + ); + assert_eq!(w["weeks"][1]["week"], json!("2026-W40")); + + // A blob whose header does not declare its size is still unmeasured. + blob(&r.0, &c, &[ZSTD[0], ZSTD[1], ZSTD[2], ZSTD[3], 0x00, 0x00]); + let w = weekly(&s, &r.0).expect("weekly"); + assert_eq!(w["weeks"][0]["blobs_raw_undeclared"], json!(1)); + assert_eq!(w["weeks"][0]["g16"], json!("unmeasured")); + + // Every blob measured: 1–2 quorums/day is outside the [O] 50–200 band. + blob(&r.0, &c, &declared(500)); + let w = weekly(&s, &r.0).expect("weekly"); + assert_eq!(w["weeks"][0]["blob_bytes_raw"], json!(1500)); + assert_eq!(w["weeks"][0]["g16"], json!("outside")); + let text = render(&w); + assert!( + !text.contains(&b) && !text.contains(SECRET_TEXT), + "no sha or row text" + ); + + // 50 quorums on one day, every blob measured and tiny: within. + let r = Root::new("weekly-in"); + let rows: Vec = (0..50) + .map(|i| rowq(&format!("t{i}"), &format!("q{i}"))) + .collect(); + day(&r.0, ("2026", "09", "24"), &rows); + blob(&r.0, &b, &declared(1000)); + blob(&r.0, &c, &declared(1000)); + let s = Snapshot::read(&r.0).expect("snapshot"); + assert_eq!( + weekly(&s, &r.0).expect("weekly")["weeks"][0]["g16"], + json!("within") + ); + + // In the quorum band but ~4.29 GB/day raw ≈ 1.57 TB/yr: outside the 0.45 TB cap. + blob(&r.0, &c, &declared(u32::MAX)); + let w = weekly(&s, &r.0).expect("weekly"); + assert!(w["weeks"][0]["raw_tb_per_year"].as_f64().expect("f64") > G16_RAW_TB_PER_YEAR); + assert_eq!(w["weeks"][0]["g16"], json!("outside")); +} + +#[test] +fn falsify_tdc_008_iso_weeks_and_zstd_sizes_match_their_standards() { + for (day, want) in [ + ("2026-09-24", "2026-W39"), + ("2026-09-28", "2026-W40"), + ("2026-01-01", "2026-W01"), + ("2027-01-01", "2026-W53"), + ("2021-01-03", "2020-W53"), + ("2024-12-30", "2025-W01"), + ("2020-02-29", "2020-W09"), + ] { + assert_eq!(iso_week(day).expect("day"), want, "{day}"); + } + assert!(iso_week("2026-9").is_err()); + + let z = |tail: &[u8]| [&ZSTD[..], tail].concat(); + assert_eq!( + zstd_content_size(&z(&[0x20, 5])), + Some(5), + "single segment, 1-byte FCS" + ); + assert_eq!( + zstd_content_size(&z(&[0x40, 0x58, 0, 1])), + Some(512), + "2-byte FCS is +256" + ); + assert_eq!( + zstd_content_size(&z(&[0xA1, 7, 0x10, 0, 0, 0])), + Some(16), + "dict id skipped" + ); + assert_eq!( + zstd_content_size(&z(&[0x00, 0x58])), + None, + "no FCS declared" + ); + assert_eq!(zstd_content_size(&[0, 1, 2, 3, 0x20, 5]), None, "not zstd"); + assert_eq!(zstd_content_size(&z(&[0x80, 1, 2])), None, "truncated"); +} diff --git a/crates/aprender-review-experiment/src/dedup.rs b/crates/aprender-review-experiment/src/dedup.rs new file mode 100644 index 0000000000..fd3569a154 --- /dev/null +++ b/crates/aprender-review-experiment/src/dedup.rs @@ -0,0 +1,101 @@ +//! PRM-C8 `trace-dedup-v1`: near-duplicate clusters over trace diffs (gate +//! G-DUP; contract `trace-dedup-v1`, FALSIFY-TDD-001..004). +//! +//! Reuses the [`crate::cluster`] fingerprint (normalised token shingles, +//! bottom-k sketch, containment), so a re-indented or renamed copy of a diff +//! lands in its original's cluster, exactly as G-CON sees it. +//! +//! - **Candidates (LSH)**: an inverted index over sketch hashes; two items are +//! compared only if their sketches share a hash. Items whose sketches share +//! none are never compared — the recall bound, stated, not hidden. +//! - **Match**: either item's sketch is at least [`CONTAINMENT`] contained in +//! the other's shingle set. +//! - **Closure**: union-find, so near-dup chains close transitively. +//! - **Tiny diffs** (below `MIN_SHINGLES`, no sketch) merge only on identical +//! normalised tokens. +//! - **Cluster id**: the id of the earliest member by `(at, id)`, so the id +//! does not depend on input order. The split guard (PRM-C9) assigns a whole +//! cluster to that member's split. + +use std::collections::{BTreeMap, HashMap, HashSet}; + +use crate::cluster::{containment, shingles, tokens, CONTAINMENT, MIN_SHINGLES, SKETCH}; + +pub const SCHEME: &str = "trace-dedup-v1"; + +/// One diff to cluster: its id (a `diff_sha256`), first-seen time (RFC 3339 +/// UTC, so it orders as text) and the diff text. +#[derive(Debug, Clone, Copy)] +pub struct Item<'a> { + pub id: &'a str, + pub at: &'a str, + pub diff: &'a str, +} + +fn root(parent: &mut [usize], mut i: usize) -> usize { + while parent[i] != i { + parent[i] = parent[parent[i]]; + i = parent[i]; + } + i +} + +fn union(parent: &mut [usize], a: usize, b: usize) { + let (ra, rb) = (root(parent, a), root(parent, b)); + parent[ra.max(rb)] = ra.min(rb); +} + +/// The cluster id of each item, in input order. +#[must_use] +pub fn clusters(items: &[Item]) -> Vec { + let sets: Vec> = items.iter().map(|i| shingles(i.diff)).collect(); + let sketches: Vec>> = sets + .iter() + .map(|s| { + (s.len() >= MIN_SHINGLES).then(|| { + let mut v: Vec = s.iter().copied().collect(); + v.sort_unstable(); + v.truncate(SKETCH); + v + }) + }) + .collect(); + let mut parent: Vec = (0..items.len()).collect(); + let mut postings: HashMap> = HashMap::new(); + let mut tiny: HashMap, usize> = HashMap::new(); + for (i, it) in items.iter().enumerate() { + let Some(sk) = &sketches[i] else { + let first = *tiny.entry(tokens(it.diff)).or_insert(i); + union(&mut parent, first, i); + continue; + }; + let mut seen = HashSet::new(); + for h in sk { + let js = postings.entry(*h).or_default(); + for &j in js.iter().filter(|&&j| seen.insert(j)) { + let other = sketches[j].as_deref().unwrap_or_default(); + if containment(sk, &sets[j]) >= CONTAINMENT + || containment(other, &sets[i]) >= CONTAINMENT + { + union(&mut parent, i, j); + } + } + js.push(i); + } + } + let mut earliest: BTreeMap = BTreeMap::new(); + for i in 0..items.len() { + let r = root(&mut parent, i); + let e = earliest.entry(r).or_insert(i); + if (items[i].at, items[i].id) < (items[*e].at, items[*e].id) { + *e = i; + } + } + (0..items.len()) + .map(|i| items[earliest[&root(&mut parent, i)]].id.to_owned()) + .collect() +} + +#[cfg(test)] +#[path = "dedup_tests.rs"] +mod tests; diff --git a/crates/aprender-review-experiment/src/dedup_tests.rs b/crates/aprender-review-experiment/src/dedup_tests.rs new file mode 100644 index 0000000000..f2751ad6ba --- /dev/null +++ b/crates/aprender-review-experiment/src/dedup_tests.rs @@ -0,0 +1,106 @@ +use super::*; + +const A: &str = "--- a/crates/x/src/sum.rs\n+++ b/crates/x/src/sum.rs\n@@ -40,12 +40,14 @@ impl Ledger {\n pub fn total(&self, rows: &[Row]) -> u64 {\n let mut total = 0u64;\n for row in rows {\n- if row.amount > 0 {\n- total += row.amount;\n+ if row.amount > 0 && !row.voided {\n+ total = total.saturating_add(row.amount);\n }\n }\n+ debug_assert!(total >= self.floor);\n total\n }\n"; + +const B: &str = "--- a/crates/y/src/parse.rs\n+++ b/crates/y/src/parse.rs\n@@ -8,9 +8,11 @@\n pub fn parse(s: &str) -> Result {\n let mut it = s.split(':');\n- let key = it.next().ok_or(Error::Empty)?;\n+ let key = it.next().ok_or(Error::Empty)?.trim();\n+ if key.is_empty() {\n+ return Err(Error::Empty);\n+ }\n let value = it.next().unwrap_or_default();\n Ok(Header { key: key.into(), value: value.into() })\n }\n"; + +/// `A` re-pathed, re-indented and renamed: the same change to G-CON. +fn a_perturbed() -> String { + A.replace("crates/x/src/sum.rs", "crates/q/src/acc.rs") + .replace(" ", " ") + .replace("total", "acc") + .replace("row", "entry") +} + +fn item<'a>(id: &'a str, at: &'a str, diff: &'a str) -> Item<'a> { + Item { id, at, diff } +} + +#[test] +fn falsify_tdd_001_a_perturbed_copy_joins_its_original_and_an_unrelated_diff_does_not() { + let p = a_perturbed(); + let items = [ + item("a", "2026-09-01T00:00:00Z", A), + item("b", "2026-09-02T00:00:00Z", B), + item("p", "2026-09-03T00:00:00Z", &p), + ]; + assert_eq!(clusters(&items), ["a", "b", "a"]); + // A diff embedded in a later, larger one: either arrival order. + let big = format!("{A}{}", segs(1, 5)); + let early = [ + item("a", "2026-09-01T00:00:00Z", A), + item("g", "2026-09-02T00:00:00Z", &big), + ]; + assert_eq!(clusters(&early), ["a", "a"]); + let late = [early[1], early[0]]; + assert_eq!(clusters(&late), ["a", "a"]); +} + +#[test] +fn falsify_tdd_002_the_cluster_id_is_the_earliest_member_whatever_the_order() { + let p = a_perturbed(); + // The copy was seen first: it names the cluster, and input order is moot. + let items = [ + item("a", "2026-09-05T00:00:00Z", A), + item("b", "2026-09-02T00:00:00Z", B), + item("p", "2026-09-03T00:00:00Z", &p), + ]; + assert_eq!(clusters(&items), ["p", "b", "p"]); + let rev: Vec = items.iter().rev().copied().collect(); + assert_eq!(clusters(&rev), ["p", "b", "p"]); + // Equal times: the smaller id wins. + let tie = [ + item("z", "2026-09-01T00:00:00Z", A), + item("y", "2026-09-01T00:00:00Z", &p), + ]; + assert_eq!(clusters(&tie), ["y", "y"]); +} + +/// Segments `from..to` of a synthetic diff; each segment's literals are its own. +fn segs(from: u32, to: u32) -> String { + (from..to) + .flat_map(|k| { + (0..4).map(move |j| { + format!("+ let v = w.get({k}).map(|x| x * {k}{j}).unwrap_or({j});\n") + }) + }) + .collect() +} + +#[test] +fn falsify_tdd_003_near_dup_chains_close_transitively() { + // X~Y and Y~Z share 8 of 10 segments; X and Z share only 6. + let (x, y, z) = (segs(1, 11), segs(3, 13), segs(5, 15)); + let xz = [ + item("x", "2026-09-01T00:00:00Z", &x), + item("z", "2026-09-03T00:00:00Z", &z), + ]; + assert_eq!( + clusters(&xz), + ["x", "z"], + "premise: X and Z alone are not near-dups" + ); + let chain = [ + item("z", "2026-09-03T00:00:00Z", &z), + item("x", "2026-09-01T00:00:00Z", &x), + item("b", "2026-09-02T00:00:00Z", B), + item("y", "2026-09-04T00:00:00Z", &y), + ]; + assert_eq!(clusters(&chain), ["x", "x", "b", "x"]); +} + +#[test] +fn falsify_tdd_004_tiny_diffs_merge_only_when_identical() { + let t1 = "@@ -1,4 +1,4 @@\n fn f(x: u32) -> bool {\n- x > 1\n+ x >= 1\n }\n"; + let t1_respaced = "@@ -1,4 +1,4 @@\n fn g(y: u32) -> bool {\n- y > 1\n+ y >= 1\n }\n"; + let t2 = "@@ -1,4 +1,4 @@\n fn f(x: u32) -> bool {\n- x > 1\n+ x > 2\n }\n"; + let items = [ + item("t1", "2026-09-01T00:00:00Z", t1), + item("t2", "2026-09-02T00:00:00Z", t2), + item("t3", "2026-09-03T00:00:00Z", t1_respaced), + item("e", "2026-09-04T00:00:00Z", ""), + item("f", "2026-09-05T00:00:00Z", ""), + ]; + // Two empty diffs are the same (empty) text; a tiny edit is not. + assert_eq!(clusters(&items), ["t1", "t2", "t1", "e", "e"]); +} diff --git a/crates/aprender-review-experiment/src/fewshot.rs b/crates/aprender-review-experiment/src/fewshot.rs new file mode 100644 index 0000000000..8f0cf356ec --- /dev/null +++ b/crates/aprender-review-experiment/src/fewshot.rs @@ -0,0 +1,156 @@ +//! REX-11 B1 challengers (§5.3): prompt versions (B1a) and retrieval +//! few-shot (B1b), built so that the sealed test split cannot leak (R-2). +//! +//! - The retrieval pool is gold specimens outside the test split. A test +//! item offered to [`select`] is an error, not a skipped row: the caller +//! read a sealed diff, and that is the violation. +//! - The query never retrieves itself (same id or same diff sha), so a dev +//! evaluation is not an open-book test. +//! - Similarity is the Jaccard index of changed-line token bigrams: +//! deterministic, no model, ties broken by id. +//! - Every prompt a challenger sends, B1a file or rendered B1b prompt, passes +//! [`guard`]: any sealed diff sha or hunk inside it is a hard FAIL +//! (`review-corpus-contamination-v1`). + +use std::collections::BTreeSet; + +use crate::contamination::Index; +use crate::corpus::{Item, Split}; + +/// Token bigrams of the changed lines (`+`/`-`, not the file headers). +fn shingles(diff: &str) -> BTreeSet<(String, String)> { + let toks: Vec = diff + .lines() + .filter(|l| { + (l.starts_with('+') || l.starts_with('-')) + && !l.starts_with("+++") + && !l.starts_with("---") + }) + .flat_map(|l| { + l[1..] + .split(|c: char| !c.is_alphanumeric() && c != '_') + .filter(|t| !t.is_empty()) + .map(str::to_string) + .collect::>() + }) + .collect(); + toks.windows(2) + .map(|w| (w[0].clone(), w[1].clone())) + .collect() +} + +/// Jaccard index of two diffs' changed-line bigrams; 0 when either is empty. +#[must_use] +pub fn similarity(a: &str, b: &str) -> f64 { + let (x, y) = (shingles(a), shingles(b)); + let union = x.union(&y).count(); + if union == 0 { + return 0.0; + } + x.intersection(&y).count() as f64 / union as f64 +} + +/// The `k` pool items most similar to the query, most similar first. +pub fn select<'a>( + query: &Item, + query_diff: &str, + pool: &'a [(Item, String)], + k: usize, +) -> Result, String> { + if let Some((i, _)) = pool.iter().find(|(i, _)| i.split == Split::Test) { + return Err(format!( + "{}: a sealed test item in the retrieval pool (R-2)", + i.id + )); + } + let mut ranked: Vec<(f64, &(Item, String))> = pool + .iter() + .filter(|(i, _)| i.id != query.id && i.diff_sha256 != query.diff_sha256) + .map(|p| (similarity(query_diff, &p.1), p)) + .collect(); + ranked.sort_by(|a, b| b.0.total_cmp(&a.0).then_with(|| a.1 .0.id.cmp(&b.1 .0.id))); + Ok(ranked.into_iter().take(k).map(|(_, p)| p).collect()) +} + +/// The gold answer shown for an example: the verdict line, then the defect +/// locations for a defect. +fn gold(item: &Item) -> String { + if item.class.is_defect() { + let locs: Vec = item + .defect + .iter() + .map(|l| format!("- defect at {}:{}", l.file, l.line)) + .collect(); + format!("VERDICT: FAIL\n{}", locs.join("\n")) + } else { + "VERDICT: PASS".into() + } +} + +/// `base`, then the examples with their gold answers. The item under review +/// is appended by `harness::compose` as before. +#[must_use] +pub fn render(base: &str, examples: &[&(Item, String)]) -> String { + let mut s = base.trim_end().to_string(); + if examples.is_empty() { + return s + "\n"; + } + s.push_str("\n\nReviewed examples:\n"); + for (n, (item, diff)) in examples.iter().enumerate() { + let nl = if diff.ends_with('\n') { "" } else { "\n" }; + s.push_str(&format!( + "\nExample {}:\n```diff\n{diff}{nl}```\n{}\n", + n + 1, + gold(item) + )); + } + s.push_str("\nNow review this diff.\n"); + s +} + +/// The bodies of the prompt's fenced blocks. Text after a fence (a gold +/// `- defect at …` line, prose starting with `-`) would otherwise join the +/// last hunk and change its fingerprint, so each block is scanned alone. +fn fenced(prompt: &str) -> Vec { + let mut out = Vec::new(); + let mut cur: Option = None; + for l in prompt.lines() { + if l.trim_start().starts_with("```") { + match cur.take() { + Some(b) => out.push(b), + None => cur = Some(String::new()), + } + } else if let Some(b) = cur.as_mut() { + b.push_str(l); + b.push('\n'); + } + } + out.extend(cur); + out +} + +/// Refuse a prompt that carries any sealed item, whether in running text or +/// inside a fenced block. An empty index proves nothing, so it is refused too. +pub fn guard(prompt: &str, sealed: &Index, what: &str) -> Result<(), String> { + if sealed.is_empty() { + return Err("no sealed manifest indexed: the guard would prove nothing".into()); + } + let mut hits = sealed.scan(prompt, what); + for block in fenced(prompt) { + hits.extend(sealed.scan(&block, what)); + } + hits.sort(); + hits.dedup(); + if hits.is_empty() { + Ok(()) + } else { + Err(format!( + "{what}: {} sealed test leak(s): {hits:?}", + hits.len() + )) + } +} + +#[cfg(test)] +#[path = "fewshot_tests.rs"] +mod tests; diff --git a/crates/aprender-review-experiment/src/fewshot_tests.rs b/crates/aprender-review-experiment/src/fewshot_tests.rs new file mode 100644 index 0000000000..519a02c829 --- /dev/null +++ b/crates/aprender-review-experiment/src/fewshot_tests.rs @@ -0,0 +1,121 @@ +use super::*; +use crate::corpus::{hunk_fingerprints, sha256_hex, Class, Loc, Sealed}; + +fn diff(body: &str) -> String { + format!("--- a/src/a.rs\n+++ b/src/a.rs\n@@ -10,5 +10,5 @@\n fn f(x: u32) -> bool {{\n{body} }}\n \n") +} + +fn item(id: &str, class: Class, split: Split, d: &str) -> (Item, String) { + let mut i = Item::new(id.into(), class, "src".into(), d); + i.split = split; + if class.is_defect() { + i.defect = vec![Loc { + file: "src/a.rs".into(), + line: 11, + }]; + } + (i, d.to_string()) +} + +fn sealed(test: &[&(Item, String)]) -> Index { + Index::new( + &test + .iter() + .map(|(i, d)| Sealed { + id: i.id.clone(), + diff_sha256: sha256_hex(d.as_bytes()), + hunks: hunk_fingerprints(d), + }) + .collect::>(), + ) +} + +fn pool() -> Vec<(Item, String)> { + vec![ + item( + "P01", + Class::P, + Split::Dev, + &diff("- x > 1\n+ x >= 1\n"), + ), + item( + "G01", + Class::G, + Split::Dev, + &diff("- let y = 2;\n+ let y = 3;\n"), + ), + item( + "R01", + Class::R, + Split::Dev, + &diff("- x > 1 && x < 9\n+ x >= 1 && x < 9\n"), + ), + ] +} + +#[test] +fn falsify_rxf_001_a_test_item_in_the_pool_is_an_error() { + let mut p = pool(); + p.push(item( + "T01", + Class::R, + Split::Test, + &diff("- a\n+ b\n"), + )); + let (q, qd) = item( + "Q", + Class::P, + Split::Test, + &diff("- x > 2\n+ x >= 2\n"), + ); + let e = select(&q, &qd, &p, 2).expect_err("sealed item in pool"); + assert!(e.contains("T01") && e.contains("R-2"), "{e}"); +} + +#[test] +fn falsify_rxf_002_retrieval_ranks_by_similarity_and_never_returns_the_query() { + let p = pool(); + let (q, qd) = item( + "Q", + Class::P, + Split::Test, + &diff("- x > 1 && x < 9 || y\n+ x >= 1 && x < 9 || y\n"), + ); + let got: Vec<&str> = select(&q, &qd, &p, 2) + .expect("selects") + .iter() + .map(|(i, _)| i.id.as_str()) + .collect(); + assert_eq!(got, ["R01", "P01"]); + // A dev query retrieves neither itself nor a byte-identical copy. + let self_copy = select(&p[2].0, &p[2].1, &p, 3).expect("selects"); + assert!(self_copy.iter().all(|(i, _)| i.id != "R01")); + assert_eq!(similarity("", &p[0].1), 0.0); +} + +#[test] +fn falsify_rxf_003_a_rendered_prompt_carrying_a_test_hunk_is_refused() { + let p = pool(); + let test = item( + "T07", + Class::R, + Split::Test, + &diff("- z * 2\n+ z * 3\n"), + ); + let ix = sealed(&[&test]); + let ok = render("BASE", &[&p[0], &p[1]]); + assert!(ok.contains("VERDICT: FAIL\n- defect at src/a.rs:11")); + assert!(ok.contains("VERDICT: PASS")); + guard(&ok, &ix, "prompt").expect("clean prompt"); + // The same prompt with the sealed item smuggled in as an example. + let leaked = render("BASE", &[&p[0], &test]); + // The gold line after the fence joins the hunk in running text: the + // whole-prompt scan alone is blind here, which is why guard scans blocks. + assert!(ix.scan(&leaked, "prompt").is_empty()); + let e = guard(&leaked, &ix, "prompt").expect_err("leak"); + assert!(e.contains("T07"), "{e}"); + assert!( + guard(&ok, &Index::default(), "prompt").is_err(), + "an empty index proves nothing" + ); +} diff --git a/crates/aprender-review-experiment/src/harness.rs b/crates/aprender-review-experiment/src/harness.rs new file mode 100644 index 0000000000..def0aa5c0a --- /dev/null +++ b/crates/aprender-review-experiment/src/harness.rs @@ -0,0 +1,318 @@ +//! REX-03 harness: drives a resident `apr serve` (OpenAI-compatible +//! `/v1/chat/completions`) with the fixed prompt, one corpus item at a time, +//! and turns each reply into a `review-experiment-receipt-v1` row. +//! +//! The harness never truncates a diff: an over-long one is sent as is, and a +//! context refusal from the serve becomes `NotRun{ContextOverflow}`. + +use crate::corpus::{sha256_hex, Item}; +use crate::receipt::{ + parse_verdict, Arm, Decoding, Load, NotRun, Output, Receipt, ServerTimings, Timings, Tokens, + Verdict, SCHEME, +}; +use crate::stats::SplitMix64; +use serde_json::{json, Value}; +use std::io::Read; +use std::time::{Duration, Instant}; + +/// Largest reply body read, in bytes (ureq's `into_string` stops at 10 MB). +const MAX_BODY: u64 = 256 << 20; + +/// Fixed decoding (§2.1 `[A]`): greedy, seed = the epic number, 512 tokens +/// (the prompt asks for a verdict line and ≤ 5 short bullets). +pub const DECODING: Decoding = Decoding { + temperature: 0.0, + seed: 4354, + max_tokens: 512, +}; + +/// Per-request timeout. A review that takes longer is a `ServeError`. +pub const TIMEOUT: Duration = Duration::from_secs(900); + +/// The user message: the prompt file verbatim, a blank line, then the diff +/// in a fenced block. Fixed; its sha is part of `request_sha256`. +#[must_use] +pub fn compose(prompt: &str, diff: &str) -> String { + let nl = if diff.ends_with('\n') { "" } else { "\n" }; + format!("{}\n\n```diff\n{diff}{nl}```\n", prompt.trim_end()) +} + +/// The exact request body sent for one item. +/// (`json!` expands to an `unwrap` of an infallible `to_value` on these +/// string and number fields, hence the scoped allow.) +#[allow(clippy::disallowed_methods)] +#[must_use] +pub fn request_body(model: &str, prompt: &str, diff: &str) -> Value { + json!({ + "model": model, + "messages": [{"role": "user", "content": compose(prompt, diff)}], + "temperature": DECODING.temperature, + "seed": DECODING.seed, + "max_tokens": DECODING.max_tokens, + "stream": false, + }) +} + +/// Run order for one cell (analysis plan §Seeds): Fisher–Yates on the ids, +/// sorted first, driven by SplitMix64(seed). +#[must_use] +pub fn run_order(ids: &[String], seed: u64) -> Vec { + let mut v = ids.to_vec(); + v.sort(); + let mut rng = SplitMix64::new(seed); + for i in (1..v.len()).rev() { + v.swap(i, rng.below(i + 1)); + } + v +} + +/// The determinism re-run subset: the first 10 % (rounded up) of the +/// seed-4354 order of the test items. +#[must_use] +pub fn rerun_subset(test_order: &[String]) -> Vec { + test_order[..test_order.len().div_ceil(10)].to_vec() +} + +/// Host load now: loadavg1 from `/proc/loadavg` (Linux) or `sysctl` (macOS). +#[must_use] +pub fn load_snapshot() -> Option { + let text = std::fs::read_to_string("/proc/loadavg").ok().or_else(|| { + let o = std::process::Command::new("sysctl") + .args(["-n", "vm.loadavg"]) + .output() + .ok()?; + Some(String::from_utf8_lossy(&o.stdout).replace(['{', '}'], "")) + })?; + let loadavg1 = text.split_whitespace().next()?.parse().ok()?; + let cpus = std::thread::available_parallelism().ok()?.get() as u32; + Some(Load { + loadavg1, + cpus, + co_running: None, + peak_anon_bytes: None, + }) +} + +/// What one HTTP exchange produced. +#[derive(Debug, Clone, PartialEq)] +pub struct Reply { + pub status: u16, + pub body: String, + pub wall_ms: f64, +} + +/// POST the body; transport failures are `Err`, HTTP errors are a [`Reply`]. +/// +/// # Errors +/// The serve could not be reached or the body could not be read. +pub fn post(url: &str, body: &Value) -> Result { + let agent = ureq::AgentBuilder::new().timeout(TIMEOUT).build(); + let t = Instant::now(); + let (status, resp) = match agent.post(url).send_json(body) { + Ok(r) => (r.status(), r), + Err(ureq::Error::Status(code, r)) => (code, r), + Err(e) => return Err(e.to_string()), + }; + // `into_string` refuses bodies over 10 MB; the /tokenize reply for a large + // candidate diff exceeds that. Read bounded at MAX_BODY instead. + let body = read_bounded(resp.into_reader(), MAX_BODY)?; + Ok(Reply { + status, + body, + wall_ms: t.elapsed().as_secs_f64() * 1e3, + }) +} + +/// Read `r` to a string of at most `cap` bytes. +/// +/// # Errors +/// The read failed, or the body ran past `cap`. +pub fn read_bounded(r: impl Read, cap: u64) -> Result { + let mut limited = r.take(cap); + let mut bytes = Vec::new(); + limited.read_to_end(&mut bytes).map_err(|e| e.to_string())?; + // Probe the reader itself, not a cap + 1 limit: Ok means it hit EOF, for + // every cap, u64::MAX included. + if let Some(b) = limited.into_inner().bytes().next() { + b.map_err(|e| e.to_string())?; + return Err(format!("reply body ran past the {cap}-byte cap")); + } + String::from_utf8(bytes).map_err(|e| e.to_string()) +} + +/// The reply's text, token counts, server timings — or why none. +#[derive(Debug, Clone, PartialEq)] +pub struct Parsed { + pub verdict: Verdict, + pub text: String, + pub tokens: Option, + pub server: Option, +} + +/// Classify a reply. 200 with a message → parsed verdict; a 4xx naming the +/// context → `ContextOverflow`; anything else → `ServeError`. +#[must_use] +pub fn classify(reply: &Reply) -> Parsed { + let not_run = |why| Parsed { + verdict: Verdict::NotRun(why), + text: reply.body.clone(), + tokens: None, + server: None, + }; + if reply.status != 200 { + let lower = reply.body.to_lowercase(); + let ctx = (400..500).contains(&reply.status) + && (lower.contains("context") || lower.contains("too long")); + return not_run(if ctx { + NotRun::ContextOverflow + } else { + NotRun::ServeError + }); + } + let Ok(v) = serde_json::from_str::(&reply.body) else { + return not_run(NotRun::ServeError); + }; + let Some(text) = v["choices"][0]["message"]["content"].as_str() else { + return not_run(NotRun::ServeError); + }; + let tokens = Some(Tokens { + prompt: v["usage"]["prompt_tokens"].as_u64().unwrap_or(0), + completion: v["usage"]["completion_tokens"].as_u64().unwrap_or(0), + }) + .filter(|t| v["usage"].is_object() && t.prompt > 0); + let t = &v["timings"]; + let server = t.is_object().then(|| ServerTimings { + prompt_ms: t["prompt_ms"].as_f64().unwrap_or(f64::NAN), + predicted_ms: t["predicted_ms"].as_f64().unwrap_or(f64::NAN), + prompt_per_second: t["prompt_per_second"].as_f64(), + predicted_per_second: t["predicted_per_second"].as_f64(), + }); + Parsed { + verdict: parse_verdict(text), + text: text.to_string(), + tokens, + server, + } +} + +/// Identity of the (cell, arm) being run; copied into every receipt. +#[derive(Debug, Clone)] +pub struct Run { + pub cell: String, + pub host: String, + pub backend: String, + pub arm: Arm, + pub apr_tag: String, + pub apr_sha256: String, + pub model_id: String, + pub weights_sha256: String, + pub prompt: String, + pub corpus_version: String, + pub prereg_sha: String, +} + +impl Run { + /// A receipt for `item` from a classified reply. `raw_path` is where the + /// caller writes `parsed.text`; its sha is recorded here. A row with no + /// token counts from the serve is `ServeError` — it cannot be admitted. + #[must_use] + pub fn receipt( + &self, + item: &Item, + body: &Value, + p: &Parsed, + wall_ms: f64, + raw_path: &str, + when: &str, + ) -> Receipt { + let mut verdict = p.verdict; + if verdict.executed() && p.tokens.is_none() { + verdict = Verdict::NotRun(NotRun::ServeError); + } + let ran = verdict.executed(); + Receipt { + schema: SCHEME.into(), + item_id: item.id.clone(), + item_sha256: item.diff_sha256.clone(), + class: item.class, + stratum: item.stratum, + split: item.split, + cell: self.cell.clone(), + host: self.host.clone(), + backend: self.backend.clone(), + arm: self.arm, + apr_tag: self.apr_tag.clone(), + apr_sha256: self.apr_sha256.clone(), + model_id: self.model_id.clone(), + weights_sha256: self.weights_sha256.clone(), + prompt_sha256: sha256_hex(self.prompt.as_bytes()), + request_sha256: sha256_hex(body.to_string().as_bytes()), + decoding: DECODING, + corpus_version: self.corpus_version.clone(), + prereg_sha: self.prereg_sha.clone(), + started_at: when.to_string(), + cold: false, + rerun: false, + verdict, + tokens: p.tokens.filter(|_| ran), + timings: Some(Timings { + wall_ms, + server: p.server, + }), + output: Some(Output { + path: raw_path.to_string(), + sha256: sha256_hex(p.text.as_bytes()), + }), + load: load_snapshot(), + } + } + + /// A `NotRun` receipt with no request (e.g. `NoDeclaredExecutor`). + #[must_use] + pub fn not_run(&self, item: &Item, why: NotRun, when: &str) -> Receipt { + let p = Parsed { + verdict: Verdict::NotRun(why), + text: String::new(), + tokens: None, + server: None, + }; + let body = request_body(&self.model_id, &self.prompt, ""); + let mut r = self.receipt(item, &body, &p, 0.0, "-", when); + (r.timings, r.output, r.load) = (None, None, None); + r + } +} + +/// RFC 3339 UTC for a Unix time (civil-from-days, proleptic Gregorian). +#[must_use] +pub fn rfc3339(unix_secs: u64) -> String { + let (days, rem) = (unix_secs / 86_400, unix_secs % 86_400); + let z = days as i64 + 719_468; + let era = z.div_euclid(146_097); + let doe = z - era * 146_097; + let yoe = (doe - doe / 1460 + doe / 36_524 - doe / 146_096) / 365; + let doy = doe - (365 * yoe + yoe / 4 - yoe / 100); + let mp = (5 * doy + 2) / 153; + let d = doy - (153 * mp + 2) / 5 + 1; + let m = if mp < 10 { mp + 3 } else { mp - 9 }; + let y = yoe + era * 400 + i64::from(m <= 2); + format!( + "{y:04}-{m:02}-{d:02}T{:02}:{:02}:{:02}Z", + rem / 3600, + rem % 3600 / 60, + rem % 60 + ) +} + +/// Now, RFC 3339 UTC. +#[must_use] +pub fn utc_now() -> String { + let s = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map_or(0, |d| d.as_secs()); + rfc3339(s) +} + +#[cfg(test)] +#[path = "harness_tests.rs"] +mod tests; diff --git a/crates/aprender-review-experiment/src/harness_tests.rs b/crates/aprender-review-experiment/src/harness_tests.rs new file mode 100644 index 0000000000..d96d2f163c --- /dev/null +++ b/crates/aprender-review-experiment/src/harness_tests.rs @@ -0,0 +1,224 @@ +// `json!` expands to an `unwrap` of an infallible `to_value`. +#![allow(clippy::disallowed_methods)] + +use super::*; +use crate::corpus::Class; +use crate::receipt::{admissible, Expect}; +use crate::score::{collect, score}; +use std::io::{Read, Write}; + +const PREREG: &str = "ef51087dc79bab0ad160e8a14f5b13e2ea43986b30c05dafc63c84c2dc21cdc0"; +const CORPUS: &str = "review-corpus-v1@787d2026256cc08b"; + +fn run() -> Run { + Run { + cell: "C2".into(), + host: "intel".into(), + backend: "cpu".into(), + arm: Arm::Apr4b, + apr_tag: "v0.69.3".into(), + apr_sha256: "a".repeat(64), + model_id: "Qwen3.5-4B-Q4_K_M".into(), + weights_sha256: "b".repeat(64), + prompt: "Review. VERDICT: PASS or FAIL.\n".into(), + corpus_version: CORPUS.into(), + prereg_sha: PREREG.into(), + } +} + +fn ok_body(text: &str) -> String { + json!({ + "choices": [{"message": {"role": "assistant", "content": text}}], + "usage": {"prompt_tokens": 120, "completion_tokens": 9, "total_tokens": 129}, + "timings": {"prompt_n": 120, "prompt_ms": 80.0, "prompt_per_second": 1500.0, + "predicted_n": 9, "predicted_ms": 90.0, "predicted_per_second": 100.0, "clock": "x"} + }) + .to_string() +} + +fn reply(status: u16, body: &str) -> Reply { + Reply { + status, + body: body.into(), + wall_ms: 12.0, + } +} + +#[test] +fn compose_is_fixed_and_never_truncates() { + assert_eq!(compose("P\n", "d\n"), "P\n\n```diff\nd\n```\n"); + assert_eq!(compose("P", "d"), "P\n\n```diff\nd\n```\n"); + let big = "+x\n".repeat(50_000); + assert!(compose("P", &big).contains(&big)); +} + +#[test] +fn run_order_is_a_seeded_permutation_and_rerun_is_ten_percent() { + let ids: Vec = (0..105).map(|i| format!("i{i:03}")).collect(); + let o = run_order(&ids, 4354); + assert_eq!(o, run_order(&ids, 4354)); + assert_ne!(o, run_order(&ids, 4355)); + let mut s = o.clone(); + s.sort(); + assert_eq!(s, ids); + let mut rev = ids.clone(); + rev.reverse(); + assert_eq!(run_order(&rev, 4354), o, "input order does not matter"); + assert_eq!(rerun_subset(&o).len(), 11); + assert_eq!(rerun_subset(&o)[..], o[..11]); +} + +#[test] +fn classify_table() { + let p = classify(&reply(200, &ok_body("VERDICT: FAIL\n- x"))); + assert_eq!(p.verdict, Verdict::Fail); + assert_eq!( + p.tokens, + Some(Tokens { + prompt: 120, + completion: 9 + }) + ); + assert_eq!(p.server.map(|s| s.prompt_ms), Some(80.0)); + let cases = [ + ( + 400, + r#"{"error":"prompt exceeds context window"}"#, + Verdict::NotRun(NotRun::ContextOverflow), + ), + (413, "Too long", Verdict::NotRun(NotRun::ContextOverflow)), + (500, "context poisoned", Verdict::NotRun(NotRun::ServeError)), + (400, "bad request", Verdict::NotRun(NotRun::ServeError)), + (200, "not json", Verdict::NotRun(NotRun::ServeError)), + ( + 200, + r#"{"choices":[]}"#, + Verdict::NotRun(NotRun::ServeError), + ), + (200, &ok_body("looks fine"), Verdict::Unparsed), + ]; + for (status, body, want) in cases { + assert_eq!( + classify(&reply(status, body)).verdict, + want, + "{status} {body}" + ); + } +} + +#[test] +fn receipts_from_the_harness_are_admissible_and_score() { + let r = run(); + let d = "--- a/src/x.rs\n+++ b/src/x.rs\n@@ -1 +1 @@\n-a\n+b\n"; + let item = crate::corpus::Item::new("R-pr1".into(), Class::R, "t".into(), d); + let body = request_body(&r.model_id, &r.prompt, d); + let p = classify(&reply(200, &ok_body("VERDICT: FAIL\n- src/x.rs: b"))); + let rec = r.receipt( + &item, + &body, + &p, + 12.0, + "raw/R-pr1.txt", + "2026-09-25T10:00:00Z", + ); + let expect = Expect { + prereg_sha: PREREG, + corpus_version: CORPUS, + }; + let line = serde_json::to_string(&rec).expect("json"); + assert!( + admissible(&line, expect).is_ok(), + "{:?}", + admissible(&line, expect) + ); + let (rows, rej) = collect(&line, expect, &[&item], |_| true, |_| Some(p.text.clone())); + assert!(rej.is_empty(), "{rej:?}"); + assert_eq!((score(&rows).recall.k, rows[0].localized), (1, true)); + + // No usage block: the row cannot be admitted as a run, so it is ServeError. + let mut bare: Value = serde_json::from_str(&ok_body("VERDICT: PASS")).expect("v"); + bare.as_object_mut().expect("o").remove("usage"); + let p = classify(&reply(200, &bare.to_string())); + let rec = r.receipt(&item, &body, &p, 1.0, "raw/x", "2026-09-25T10:00:00Z"); + assert_eq!(rec.verdict, Verdict::NotRun(NotRun::ServeError)); + + let nr = r.not_run(&item, NotRun::NoDeclaredExecutor, "2026-09-25T10:00:00Z"); + assert!(admissible(&serde_json::to_string(&nr).expect("j"), expect).is_ok()); +} + +#[test] +fn post_talks_http_to_a_loopback_serve() { + let listener = std::net::TcpListener::bind("127.0.0.1:0").expect("bind"); + let addr = listener.local_addr().expect("addr"); + let canned = ok_body("VERDICT: PASS"); + let server = std::thread::spawn(move || { + let (mut s, _) = listener.accept().expect("accept"); + let mut buf = vec![0u8; 65_536]; + let mut got = Vec::new(); + while !String::from_utf8_lossy(&got).contains("\"stream\"") { + let n = s.read(&mut buf).expect("read"); + if n == 0 { + break; + } + got.extend_from_slice(&buf[..n]); + } + let resp = format!( + "HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{canned}", + canned.len() + ); + s.write_all(resp.as_bytes()).expect("write"); + String::from_utf8_lossy(&got).into_owned() + }); + let body = request_body("m", "P", "d\n"); + let rep = post(&format!("http://{addr}/v1/chat/completions"), &body).expect("post"); + let seen = server.join().expect("join"); + assert!(seen.starts_with("POST /v1/chat/completions"), "{seen}"); + assert!( + seen.contains("\"temperature\":0.0") || seen.contains("\"temperature\":0"), + "{seen}" + ); + assert_eq!(rep.status, 200); + assert_eq!(classify(&rep).verdict, Verdict::Pass); +} + +#[test] +fn rfc3339_by_hand() { + assert_eq!(rfc3339(0), "1970-01-01T00:00:00Z"); + assert_eq!(rfc3339(951_782_400), "2000-02-29T00:00:00Z"); + assert_eq!(rfc3339(1_790_337_600), "2026-09-25T12:00:00Z"); + assert_eq!(rfc3339(4_107_542_399), "2100-02-28T23:59:59Z"); +} + +/// A bounded read refuses a body past its cap instead of truncating it (#4459 +/// quorum r2, 3/3 lanes): a cut-off reply returned as `Ok` would be scored as +/// if it were the whole one. +#[test] +fn read_bounded_refuses_past_its_cap_and_never_truncates() { + let table: &[(&[u8], u64, Result<&str, ()>)] = &[ + (b"", 4, Ok("")), + (b"abc", 4, Ok("abc")), + (b"abcd", 4, Ok("abcd")), + (b"abcde", 4, Err(())), + (b"abcdefgh", 4, Err(())), + ]; + for &(input, cap, ref want) in table { + let got = read_bounded(input, cap); + match want { + Ok(s) => assert_eq!(got.as_deref(), Ok(*s), "input={input:?} cap={cap}"), + Err(()) => assert!( + got.as_ref().is_err_and(|e| e.contains("cap")), + "input={input:?} cap={cap} got={got:?}" + ), + } + } +} + +/// Invalid UTF-8 inside the cap is refused, never returned lossily; a body that +/// ends exactly at the cap is whole, whatever the cap (#4459 quorum r3). +#[test] +fn read_bounded_refuses_invalid_utf8_and_takes_a_body_ending_at_any_cap() { + assert!(read_bounded(&[0x61, 0xff][..], 4).is_err()); + assert_eq!(read_bounded(&b"abcd"[..], u64::MAX).as_deref(), Ok("abcd")); + assert_eq!(read_bounded(&b""[..], 0).as_deref(), Ok("")); + assert!(read_bounded(&b"a"[..], 0).is_err_and(|e| e.contains("cap"))); +} diff --git a/crates/aprender-review-experiment/src/kappa_probe.rs b/crates/aprender-review-experiment/src/kappa_probe.rs new file mode 100644 index 0000000000..1dd1c46fde --- /dev/null +++ b/crates/aprender-review-experiment/src/kappa_probe.rs @@ -0,0 +1,414 @@ +//! PRM-C13 (was PRA-001 T13): the lane κ_err probe and its independence gate +//! (PRM-001 v3 §3 H7 and S-14; contract +//! `lane-independence-v1`). +//! +//! κ_err is Cohen's κ on binary per-item error indicators against GOLD, for +//! every (shadow lane, counted lane) pair, over matured gold rows of one split. +//! It reuses the prereg-locked [`crate::stats::error_kappa`]; this module only +//! decides which items count and what an error is. +//! +//! - An item is one `(quorum_id, round)` whose rows are `label_tier = gold`, +//! matured (`outcome_matured_at` non-null), on the requested split, with +//! `trace_status = ok` and a settled outcome: `merged` is clean, +//! `reverted_le14d` / `regression_escape` is a defect. `pending` and +//! `closed_unmerged` carry no gold label and are excluded. +//! - A lane flags a defect with `request_changes` and clears it with `approve`. +//! Any other verdict, a `failed` parse or a non-`Verdict` state is UNPARSED: +//! the item is excluded for that pair, never counted as an error. +//! - A pair with fewer than `min_n` shared items, or with κ undefined, is +//! `insufficient`: it never prints a number (S-14). +//! +//! The gate is H7 as PRM-001 v3 §3 states it, δ-free: on the items the shadow +//! lane and EVERY counted lane scored, κ_err(shadow, V) ≤ the largest +//! voter–voter κ_err for each voter V, decided by the prereg-locked +//! [`crate::stats::h7_holds`]. It refuses (S-14) below `min_n` common items, +//! on a voter with no rows, and when `h7_holds` is undecided. The voter set and +//! `min_n` come only from the run manifest, registered strictly before +//! training started. + +use serde::Serialize; +use serde_json::Value; +use std::collections::{BTreeMap, BTreeSet}; + +/// The fields of one agent-trace-v1 row the probe reads. +#[derive(Debug, Clone)] +pub struct Row { + /// `quorum_id` + pub quorum_id: String, + /// `round` + pub round: u64, + /// `lane` + pub lane: String, + /// `counted` + pub counted: bool, + /// `true` = flagged a defect, `false` = cleared it, `None` = unparsed. + pub flag: Option, + /// `true` = gold defect, `false` = clean, `None` = no gold label. + pub defect: Option, + /// Gold, matured, `trace_status = ok`. + pub gold_matured: bool, + /// `split_guard.split` + pub split: String, +} + +fn text<'a>(v: &'a Value, k: &str) -> &'a str { + v.get(k).and_then(Value::as_str).unwrap_or("") +} + +fn row_of(v: &Value) -> Result { + let quorum_id = v + .get("quorum_id") + .and_then(Value::as_str) + .ok_or("no quorum_id")? + .to_string(); + let round = v.get("round").and_then(Value::as_u64).ok_or("no round")?; + let lane = v + .get("lane") + .and_then(Value::as_str) + .ok_or("no lane")? + .to_string(); + let parsed = + text(v, "lane_state") == "Verdict" && matches!(text(v, "parse_status"), "ok" | "repaired"); + let flag = match (parsed, text(v, "verdict")) { + (true, "request_changes") => Some(true), + (true, "approve") => Some(false), + _ => None, + }; + let defect = match text(v, "outcome") { + "merged" => Some(false), + "reverted_le14d" | "regression_escape" => Some(true), + _ => None, + }; + let matured = v.get("outcome_matured_at").is_some_and(|m| !m.is_null()); + Ok(Row { + quorum_id, + round, + lane, + counted: v.get("counted").and_then(Value::as_bool).unwrap_or(false), + flag, + defect, + gold_matured: text(v, "label_tier") == "gold" && matured && text(v, "trace_status") == "ok", + split: v + .get("split_guard") + .map(|s| text(s, "split")) + .unwrap_or("") + .to_string(), + }) +} + +/// Parse agent-trace-v1 JSONL. A malformed line is an error naming its line +/// number, never a silently dropped row. +pub fn parse_rows(jsonl: &str) -> Result, String> { + jsonl + .lines() + .enumerate() + .filter(|(_, l)| !l.trim().is_empty()) + .map(|(i, l)| { + serde_json::from_str::(l) + .map_err(|e| e.to_string()) + .and_then(|v| row_of(&v)) + .map_err(|e| format!("line {}: {e}", i + 1)) + }) + .collect() +} + +/// Whether a pair produced a κ. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum PairStatus { + /// κ_err is defined on at least `min_n` shared items. + Ok, + /// Too few shared items, or κ undefined (S-14). + Insufficient, +} + +/// κ_err of one (shadow lane, counted lane) pair. +#[derive(Debug, Clone, Serialize)] +pub struct PairKappa { + /// The counted lane. + pub counted_lane: String, + /// Shared scored items. + pub n: usize, + /// `None` unless `status` is `Ok`. + pub kappa_err: Option, + /// See [`PairStatus`]. + pub status: PairStatus, +} + +/// Every pair's κ_err for one shadow lane on one split. +#[derive(Debug, Clone, Serialize)] +pub struct Scoreboard { + /// Always `lane-independence-v1`. + pub schema: &'static str, + /// The split scored. + pub split: String, + /// The shadow lane (the candidate side of every pair). + pub shadow_lane: String, + /// The shared-item floor applied. + pub min_n: usize, + /// One entry per counted lane present in the rows, sorted by lane. + pub pairs: Vec, +} + +type Items<'a> = BTreeMap<(&'a str, u64), BTreeMap<&'a str, Vec<&'a Row>>>; + +/// Matured gold `split` rows grouped item → lane → rows. +fn items_of<'a>(rows: &'a [Row], split: &str) -> Items<'a> { + let mut items: Items<'a> = BTreeMap::new(); + for r in rows + .iter() + .filter(|r| r.gold_matured && r.split == split && r.defect.is_some()) + { + items + .entry((&r.quorum_id, r.round)) + .or_default() + .entry(&r.lane) + .or_default() + .push(r); + } + items +} + +/// A lane's `(flag, defect)` on one item; `None` if absent, unparsed, or +/// ambiguous (two rows for one item). +fn one(m: &BTreeMap<&str, Vec<&Row>>, l: &str) -> Option<(bool, bool)> { + match m.get(l).map(Vec::as_slice) { + Some([r]) => Some((r.flag?, r.defect?)), + _ => None, + } +} + +/// κ_err of `shadow` against every counted lane, on matured gold `split` rows. +#[must_use] +pub fn probe(rows: &[Row], split: &str, shadow: &str, min_n: usize) -> Scoreboard { + let items = items_of(rows, split); + let counted: BTreeSet<&str> = rows + .iter() + .filter(|r| r.gold_matured && r.split == split && r.defect.is_some()) + .filter(|r| r.counted && r.lane != shadow) + .map(|r| r.lane.as_str()) + .collect(); + let pairs = counted + .into_iter() + .map(|lane| { + let (mut a, mut b) = (Vec::new(), Vec::new()); + for m in items.values() { + if let (Some((fs, ds)), Some((fl, dl))) = (one(m, shadow), one(m, lane)) { + // Both rows describe one PR; disagreeing gold is not gold. + if ds == dl { + a.push(fs != ds); + b.push(fl != dl); + } + } + } + let kappa_err = (a.len() >= min_n) + .then(|| crate::stats::error_kappa(&a, &b)) + .flatten(); + PairKappa { + counted_lane: lane.to_string(), + n: a.len(), + kappa_err, + status: if kappa_err.is_some() { + PairStatus::Ok + } else { + PairStatus::Insufficient + }, + } + }) + .collect(); + Scoreboard { + schema: "lane-independence-v1", + split: split.to_string(), + shadow_lane: shadow.to_string(), + min_n, + pairs, + } +} + +/// The `lane_independence` block of a run manifest. H7 is δ-free (PRM-001 v3 +/// §3), so what is pre-registered is the voter set and the item floor. +#[derive(Debug, Clone, Serialize)] +pub struct Manifest { + /// Shared-item floor for the gate to decide. + pub min_n: usize, + /// The counted lanes (voters) the gate must see on every item. + pub counted_lanes: Vec, + /// When the block was registered (`YYYY-MM-DDTHH:MM:SSZ`). + pub registered_at: String, + /// When candidate training started (`YYYY-MM-DDTHH:MM:SSZ`). + pub training_started_at: String, +} + +/// Strict UTC `YYYY-MM-DDTHH:MM:SSZ`, so string order is time order. +fn utc_stamp(s: &str) -> bool { + let b = s.as_bytes(); + b.len() == 20 + && b.iter().enumerate().all(|(i, &c)| match i { + 4 | 7 => c == b'-', + 10 => c == b'T', + 13 | 16 => c == b':', + 19 => c == b'Z', + _ => c.is_ascii_digit(), + }) +} + +/// Parse and validate a run manifest's `lane_independence` block. A missing +/// or unusable field, a block registered after training started, or a `delta` +/// (v2's rule, gone in v3: nothing would read it) is an error. +pub fn parse_manifest(json: &str) -> Result { + let v: Value = serde_json::from_str(json).map_err(|e| format!("manifest: {e}"))?; + let li = v + .get("lane_independence") + .ok_or("manifest has no lane_independence block")?; + if li.get("delta").is_some() { + return Err( + "lane_independence.delta: H7 is δ-free in PRM-001 v3 (κ(qwen, V) ≤ max voter–voter κ); remove it" + .into(), + ); + } + let min_n = li + .get("min_n") + .and_then(Value::as_u64) + .filter(|&n| n > 0) + .ok_or("lane_independence.min_n must be a positive integer")?; + let counted_lanes: Vec = li + .get("counted_lanes") + .and_then(Value::as_array) + .map(|a| { + a.iter() + .filter_map(|l| l.as_str().map(str::to_string)) + .collect() + }) + .filter(|a: &Vec| !a.is_empty()) + .ok_or("lane_independence.counted_lanes must name at least one lane")?; + let stamp = |k: &str| -> Result { + li.get(k) + .and_then(Value::as_str) + .filter(|s| utc_stamp(s)) + .map(str::to_string) + .ok_or(format!( + "lane_independence.{k} must be YYYY-MM-DDTHH:MM:SSZ" + )) + }; + let registered_at = stamp("registered_at")?; + let training_started_at = stamp("training_started_at")?; + if registered_at >= training_started_at { + return Err(format!( + "lane_independence registered at {registered_at}, not before training started at {training_started_at}" + )); + } + Ok(Manifest { + min_n: usize::try_from(min_n).map_err(|e| e.to_string())?, + counted_lanes, + registered_at, + training_started_at, + }) +} + +/// One lane pair's κ_err on the gate's common items. +#[derive(Debug, Clone, Serialize)] +pub struct GatePair { + pub pair: [String; 2], + pub kappa_err: Option, +} + +/// The gate's decision and the κ it decided on. +#[derive(Debug, Clone, Serialize)] +pub struct GateVerdict { + /// `true` only when [`crate::stats::h7_holds`] returned `Some(true)` on at + /// least `min_n` common items. + pub pass: bool, + /// Every reason for refusal; empty when `pass`. + pub refusals: Vec, + /// Items every lane (shadow and each voter) scored, gold agreeing. + pub n: usize, + /// κ_err(shadow, V) per voter, manifest order. + pub qwen_vs_voter: Vec, + /// κ_err(V_i, V_j) per voter pair, i < j. + pub voter_vs_voter: Vec, + /// The largest voter–voter κ_err, when every pair is defined. + pub ceiling: Option, +} + +/// H7 (PRM-001 v3 §3, δ-free) on candidate rows: with I the matured gold +/// `split` items that `shadow` and EVERY counted lane scored (one parsed row +/// each, gold agreeing), `pass` ⇔ `stats::h7_holds` on I is `Some(true)`, i.e. +/// κ_err(shadow, V) ≤ max voter–voter κ_err for every voter V. Refuses (S-14) +/// on fewer than `min_n` common items, a voter with no row, and an undecided +/// `h7_holds` (fewer than two voters, or an undefined κ). +#[must_use] +pub fn gate(rows: &[Row], split: &str, shadow: &str, m: &Manifest) -> GateVerdict { + let items = items_of(rows, split); + let mut refusals = Vec::new(); + let voters: Vec<&str> = m.counted_lanes.iter().map(String::as_str).collect(); + for v in &voters { + if !items.values().any(|l| l.contains_key(v)) { + refusals.push(format!("S-14 {v}: no scored rows")); + } + } + // err[lane] over I, in item order. + let lanes: Vec<&str> = std::iter::once(shadow) + .chain(voters.iter().copied()) + .collect(); + let mut err: Vec> = vec![Vec::new(); lanes.len()]; + for m in items.values() { + let Some(got) = lanes.iter().map(|l| one(m, l)).collect::>>() else { + continue; + }; + // All rows describe one PR; disagreeing gold is not gold. + if got.iter().all(|&(_, d)| d == got[0].1) { + for (e, (f, d)) in err.iter_mut().zip(got) { + e.push(f != d); + } + } + } + let n = err[0].len(); + if n < m.min_n { + refusals.push(format!( + "S-14 insufficient: {n} items scored by the shadow lane and every voter, min_n={}", + m.min_n + )); + } + let k = |i: usize, j: usize| GatePair { + pair: [lanes[i].to_string(), lanes[j].to_string()], + kappa_err: crate::stats::error_kappa(&err[i], &err[j]), + }; + let qwen_vs_voter: Vec = (1..lanes.len()).map(|j| k(0, j)).collect(); + let voter_vs_voter: Vec = (1..lanes.len()) + .flat_map(|i| (i + 1..lanes.len()).map(move |j| (i, j))) + .map(|(i, j)| k(i, j)) + .collect(); + let kappas = |p: &[GatePair]| p.iter().map(|p| p.kappa_err).collect::>(); + let ceiling = kappas(&voter_vs_voter) + .into_iter() + .collect::>>() + .and_then(|v| v.into_iter().reduce(f64::max)); + match crate::stats::h7_holds(&kappas(&qwen_vs_voter), &kappas(&voter_vs_voter)) { + Some(true) => {} + Some(false) => { + let c = ceiling.unwrap_or(f64::NAN); + for p in &qwen_vs_voter { + if let Some(q) = p.kappa_err.filter(|&q| q > c) { + refusals.push(format!( + "S-14 {}: κ_err {q:.4} > voter–voter max {c:.4}", + p.pair[1] + )); + } + } + } + None => refusals + .push("S-14 H7 undecided: fewer than two voters, or a κ_err is undefined".into()), + } + GateVerdict { + pass: refusals.is_empty(), + refusals, + n, + qwen_vs_voter, + voter_vs_voter, + ceiling, + } +} + +#[cfg(test)] +#[path = "kappa_probe_tests.rs"] +mod tests; diff --git a/crates/aprender-review-experiment/src/kappa_probe_tests.rs b/crates/aprender-review-experiment/src/kappa_probe_tests.rs new file mode 100644 index 0000000000..469a0d50ba --- /dev/null +++ b/crates/aprender-review-experiment/src/kappa_probe_tests.rs @@ -0,0 +1,305 @@ +use super::*; + +/// One agent-trace-v1 row, as much of it as the probe reads. +fn row(lane: &str, key: u32, verdict: &str, outcome: &str) -> String { + row_with(lane, key, verdict, outcome, "gold", "val", true) +} + +fn row_with( + lane: &str, + key: u32, + verdict: &str, + outcome: &str, + tier: &str, + split: &str, + matured: bool, +) -> String { + let counted = lane != "qwen-shadow"; + let matured = if matured { + r#""2026-09-01T00:00:00Z""# + } else { + "null" + }; + format!( + r#"{{"schema":"agent-trace-v1","quorum_id":"q{key}","round":1,"lane":"{lane}","counted":{counted}, + "lane_state":"Verdict","verdict":"{verdict}","parse_status":"ok","label_tier":"{tier}", + "outcome":"{outcome}","outcome_matured_at":{matured},"split_guard":{{"split":"{split}"}},"trace_status":"ok"}}"# + ) + .replace('\n', "") +} + +/// `n` items; `defects` of them regressed. Each lane's verdict per item comes from `flag(i)`. +fn lane_rows( + lane: &str, + n: u32, + defect: impl Fn(u32) -> bool, + flag: impl Fn(u32) -> bool, +) -> Vec { + (0..n) + .map(|i| { + let v = if flag(i) { + "request_changes" + } else { + "approve" + }; + let o = if defect(i) { + "reverted_le14d" + } else { + "merged" + }; + row(lane, i, v, o) + }) + .collect() +} + +fn board(lines: &[String], min_n: usize) -> Scoreboard { + let rows = parse_rows(&lines.join("\n")).expect("fixture parses"); + probe(&rows, "val", "qwen-shadow", min_n) +} + +fn pair<'a>(b: &'a Scoreboard, lane: &str) -> &'a PairKappa { + b.pairs + .iter() + .find(|p| p.counted_lane == lane) + .expect("pair present") +} + +/// Qwen and sonnet share half their errors: a known, non-degenerate κ. +fn base_fixture() -> Vec { + let defect = |i: u32| i % 2 == 0; + let mut v = lane_rows("sonnet", 40, defect, |i| (i % 2 == 0) != (i % 5 == 0)); + v.extend(lane_rows("qwen-shadow", 40, defect, |i| { + (i % 2 == 0) != (i % 5 == 0 || i % 7 == 0) + })); + v +} + +#[test] +fn falsify_lind_001_only_matured_gold_val_rows_are_scored() { + let clean = board(&base_fixture(), 10); + let k = pair(&clean, "sonnet").kappa_err.expect("defined"); + // Planted noise: every lane gets rows that would be pure agreement-errors if counted. + let mut noisy = base_fixture(); + for (i, (tier, split, matured, outcome)) in [ + ("silver", "val", true, "reverted_le14d"), + ("pending", "val", true, "reverted_le14d"), + ("quarantined", "val", true, "reverted_le14d"), + ("gold", "train", true, "reverted_le14d"), + ("gold", "test", true, "reverted_le14d"), + ("gold", "val", false, "reverted_le14d"), + ("gold", "val", true, "pending"), + ("gold", "val", true, "closed_unmerged"), + ] + .into_iter() + .enumerate() + { + let key = 1000 + i as u32; + for lane in ["sonnet", "qwen-shadow"] { + noisy.push(row_with( + lane, key, "approve", outcome, tier, split, matured, + )); + } + } + // Unparsed / non-verdict rows are excluded, never counted as errors. + noisy.push(row("sonnet", 2000, "abstain", "reverted_le14d")); + noisy.push(row("qwen-shadow", 2000, "approve", "reverted_le14d")); + let n = board(&noisy, 10); + assert_eq!( + pair(&n, "sonnet").n, + pair(&clean, "sonnet").n, + "noise rows were scored" + ); + assert_eq!(pair(&n, "sonnet").kappa_err, Some(k)); + // And the κ is the frozen stats::error_kappa on the error vectors, not a second formula. + let (a, b): (Vec, Vec) = (0..40u32) + .map(|i| (i % 5 == 0 || i % 7 == 0, i % 5 == 0)) + .unzip(); + assert_eq!(Some(k), crate::stats::error_kappa(&a, &b)); +} + +#[test] +fn falsify_lind_002_thin_or_undefined_pairs_are_insufficient() { + let thin = board(&base_fixture(), 41); + let p = pair(&thin, "sonnet"); + assert_eq!(p.status, PairStatus::Insufficient); + assert_eq!(p.kappa_err, None, "a thin pair must not print a number"); + // Both lanes always right → constant error vector → κ undefined. + let defect = |i: u32| i % 2 == 0; + let mut v = lane_rows("sonnet", 40, defect, defect); + v.extend(lane_rows("qwen-shadow", 40, defect, defect)); + let u = board(&v, 10); + assert_eq!(pair(&u, "sonnet").status, PairStatus::Insufficient); + assert_eq!(pair(&u, "sonnet").kappa_err, None); + // The gate refuses an undefined κ (S-14), it never passes by default. + v.extend(lane_rows("haiku", 40, defect, defect)); + let g = run_gate(&[v], &manifest(&["sonnet", "haiku"], 10)); + assert!( + !g.pass && g.refusals.iter().any(|r| r.contains("S-14")), + "{:?}", + g.refusals + ); +} + +fn manifest(voters: &[&str], min_n: usize) -> Manifest { + Manifest { + min_n, + counted_lanes: voters.iter().map(|v| (*v).to_string()).collect(), + registered_at: "2026-09-20T00:00:00Z".into(), + training_started_at: "2026-09-21T00:00:00Z".into(), + } +} + +/// `n` items, half regressed; `lane` errs exactly on the items where `err(i)`. +fn errs(lane: &str, n: u32, err: impl Fn(u32) -> bool) -> Vec { + let defect = |i: u32| i % 2 == 0; + lane_rows(lane, n, defect, move |i| defect(i) != err(i)) +} + +fn run_gate(lines: &[Vec], m: &Manifest) -> GateVerdict { + let rows = parse_rows(&lines.concat().join("\n")).expect("fixture parses"); + gate(&rows, "val", "qwen-shadow", m) +} + +fn five(i: u32) -> bool { + i % 5 == 0 +} + +/// Sonnet and haiku share most errors; qwen errs elsewhere. +fn voters() -> Vec> { + vec![ + errs("sonnet", 40, five), + errs("haiku", 40, |i| five(i) || i % 11 == 0), + ] +} + +#[test] +fn falsify_lind_003_h7_is_qwen_kappa_at_most_max_voter_kappa() { + let m = manifest(&["sonnet", "haiku"], 10); + let mut v = voters(); + v.push(errs("qwen-shadow", 40, |i| i % 3 == 0)); + let g = run_gate(&v, &m); + assert!(g.pass, "{:?}", g.refusals); + assert_eq!(g.n, 40); + let c = g.ceiling.expect("defined"); + assert!(g + .qwen_vs_voter + .iter() + .all(|p| p.kappa_err.expect("defined") <= c)); + // A qwen that copies sonnet's blind spots: κ = 1 > the voter–voter max. + let mut v = voters(); + v.push(errs("qwen-shadow", 40, five)); + let g = run_gate(&v, &m); + assert!(c < 1.0 && !g.pass, "a voter-copying qwen passed"); + assert!( + g.refusals.iter().any(|r| r.contains("sonnet")), + "{:?}", + g.refusals + ); + // Inclusive: identical voters put the ceiling at 1, so the same copy passes. + let v = vec![ + errs("sonnet", 40, five), + errs("haiku", 40, five), + errs("qwen-shadow", 40, five), + ]; + let g = run_gate(&v, &m); + assert_eq!(g.ceiling, Some(1.0)); + assert!(g.pass, "κ = ceiling must pass (≤): {:?}", g.refusals); + // The ceiling is the MAX over every voter pair, not the first or the least. + let mut v = voters(); + v.push(errs("agy", 40, |i| i % 3 == 0)); + v.push(errs("qwen-shadow", 40, |i| i % 4 == 0)); + let g = run_gate(&v, &manifest(&["sonnet", "haiku", "agy"], 10)); + let vv: Vec = g + .voter_vs_voter + .iter() + .map(|p| p.kappa_err.expect("defined")) + .collect(); + assert_eq!(vv.len(), 3); + assert_eq!(g.ceiling, vv.iter().copied().reduce(f64::max)); + assert!( + vv.iter().any(|&k| Some(k) != g.ceiling), + "the fixture must separate max from the rest" + ); +} + +#[test] +fn falsify_lind_002_the_gate_refuses_what_it_cannot_decide() { + let mut v = voters(); + v.push(errs("qwen-shadow", 40, |i| i % 3 == 0)); + // One voter: no voter–voter κ, so H7 is undecided — a refusal, not a pass. + let g = run_gate(&v, &manifest(&["sonnet"], 10)); + assert!( + !g.pass && g.refusals.iter().any(|r| r.contains("undecided")), + "{:?}", + g.refusals + ); + // A voter the manifest names but the rows lack is a refusal, not a skip. + let g = run_gate(&v, &manifest(&["sonnet", "haiku", "agy"], 10)); + assert!( + !g.pass && g.refusals.iter().any(|r| r.contains("agy")), + "{:?}", + g.refusals + ); + // Below min_n common items. + let g = run_gate(&v, &manifest(&["sonnet", "haiku"], 41)); + assert!( + !g.pass && g.refusals.iter().any(|r| r.contains("insufficient")), + "{:?}", + g.refusals + ); + assert!(run_gate(&v, &manifest(&["sonnet", "haiku"], 40)).pass); +} + +/// Every κ, the ceiling included, is taken on I, the items qwen AND every +/// voter scored — not on each pair's own overlap. +#[test] +fn falsify_lind_002_kappas_are_on_the_common_items() { + let haiku = |i: u32| if i < 20 { i % 3 == 0 } else { five(i) }; + let v = vec![ + errs("sonnet", 40, five), + errs("haiku", 40, haiku), + errs("qwen-shadow", 20, |i| i % 4 == 0), + ]; + // An item whose gold the lanes disagree on is not gold: it is not in I. + let mut v = v; + v.push(vec![ + row("sonnet", 99, "approve", "merged"), + row("haiku", 99, "approve", "reverted_le14d"), + row("qwen-shadow", 99, "approve", "merged"), + ]); + let g = run_gate(&v, &manifest(&["sonnet", "haiku"], 10)); + assert_eq!(g.n, 20); + let e = |f: &dyn Fn(u32) -> bool, n: u32| (0..n).map(f).collect::>(); + let on_i = crate::stats::error_kappa(&e(&five, 20), &e(&haiku, 20)); + let on_all = crate::stats::error_kappa(&e(&five, 40), &e(&haiku, 40)); + assert_ne!( + on_i, on_all, + "the fixture must separate the two, or this proves nothing" + ); + assert_eq!(g.ceiling, on_i); +} + +#[test] +fn falsify_lind_004_voters_and_min_n_are_pre_registered_and_delta_is_gone() { + let ok = r#"{"lane_independence":{"min_n":30,"counted_lanes":["sonnet","haiku","agy"], + "registered_at":"2026-09-20T00:00:00Z","training_started_at":"2026-09-21T00:00:00Z"}}"#; + let m = parse_manifest(ok).expect("valid manifest"); + assert_eq!((m.min_n, m.counted_lanes.len()), (30, 3)); + for bad in [ + "{}", + r#"{"lane_independence":{}}"#, + // v2's δ: H7 is δ-free in v3, and a δ nothing reads is a false assurance + r#"{"lane_independence":{"delta":0.05,"min_n":30,"counted_lanes":["sonnet"],"registered_at":"2026-09-20T00:00:00Z","training_started_at":"2026-09-21T00:00:00Z"}}"#, + r#"{"lane_independence":{"min_n":0,"counted_lanes":["sonnet"],"registered_at":"2026-09-20T00:00:00Z","training_started_at":"2026-09-21T00:00:00Z"}}"#, + r#"{"lane_independence":{"min_n":30,"counted_lanes":[],"registered_at":"2026-09-20T00:00:00Z","training_started_at":"2026-09-21T00:00:00Z"}}"#, + // registered AFTER training started: the voters could be picked after seeing κ + r#"{"lane_independence":{"min_n":30,"counted_lanes":["sonnet"],"registered_at":"2026-09-22T00:00:00Z","training_started_at":"2026-09-21T00:00:00Z"}}"#, + // same instant is not "before" + r#"{"lane_independence":{"min_n":30,"counted_lanes":["sonnet"],"registered_at":"2026-09-21T00:00:00Z","training_started_at":"2026-09-21T00:00:00Z"}}"#, + // a timestamp that does not compare lexicographically (offset, not Z) + r#"{"lane_independence":{"min_n":30,"counted_lanes":["sonnet"],"registered_at":"2026-09-20T00:00:00+02:00","training_started_at":"2026-09-21T00:00:00Z"}}"#, + "not json", + ] { + assert!(parse_manifest(bad).is_err(), "accepted: {bad}"); + } +} diff --git a/crates/aprender-review-experiment/src/ladder.rs b/crates/aprender-review-experiment/src/ladder.rs new file mode 100644 index 0000000000..d757920c9a --- /dev/null +++ b/crates/aprender-review-experiment/src/ladder.rs @@ -0,0 +1,296 @@ +//! PRM-09 promotion ladder (PRM-001 v3 §5.4; contract `rex-promotion-ladder-v2`, +//! which supersedes the REX-09 shadow → tripwire → vote ladder of v1). +//! +//! The ladder reads only a `prm-001-report-v1` report (the §9 template: +//! `spec: PRM-001, version: 3`) and fails closed: +//! - **shadow** is the default, and anything the ladder cannot read keeps it; +//! - **tripwire** needs H4 and the replay p95 within the queue budget; +//! - **tie-breaker** (decides only a 1–1 split) also needs H5, H7, H1, H2 and +//! 7-day availability ≥ 95%; +//! - **vote** also needs ≥ 20 decided tie-breaks whose 14-day outcome agreement +//! is at least the voters'. +//! +//! H4/H5 are decided by the Holm-adjusted p; the CI is report-only. H1/H2 (count +//! tests) and H7 (δ-free, outside the Holm family) are decided by their verdict. +//! `hardware_ruling.rung_eligible` caps the rung. Lambda is rung-2 shadow only +//! (R-9): a lambda or unnamed primary cell caps the lane at tripwire (S-4). +//! +//! [`evidence`] is the rung one report grants; [`decide`] steps the lane from +//! its current rung by at most one rung either way, except that S-4 caps at once. + +use serde::{Deserialize, Serialize}; + +pub const SPEC: &str = "PRM-001"; +pub const SPEC_VERSION: u64 = 3; +/// §5.4: 7-day availability for tie-breaker. +pub const MIN_AVAILABILITY: f64 = 0.95; +/// §5.4: decided tie-breaks for vote. +pub const MIN_TIE_BREAKS: u64 = 20; + +/// A rung of the ladder, lowest first. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub enum Mode { + Shadow, + Tripwire, + TieBreaker, + Vote, +} + +impl Mode { + const ALL: [Mode; 4] = [Mode::Shadow, Mode::Tripwire, Mode::TieBreaker, Mode::Vote]; + + fn up(self) -> Mode { + Self::ALL[(self as usize + 1).min(3)] + } + + fn down(self) -> Mode { + Self::ALL[(self as usize).saturating_sub(1)] + } +} + +#[derive(Debug, Deserialize)] +struct Hypothesis { + id: String, + /// Holm-adjusted one-sided p. `ci` may be present and is ignored. + #[serde(default)] + p_holm: Option, + #[serde(default)] + verdict: Option, +} + +#[derive(Debug, Default, Deserialize)] +struct Results { + #[serde(default)] + hypotheses: Vec, +} + +#[derive(Debug, Default, Deserialize)] +struct Lane { + #[serde(default)] + availability_7d: Option, +} + +#[derive(Debug, Default, Deserialize)] +struct Speed { + #[serde(default)] + within_budget: Option, +} + +#[derive(Debug, Deserialize)] +struct Ruling { + #[serde(default)] + primary: Option, + #[serde(default)] + rung_eligible: Option, +} + +#[derive(Debug, Default, Deserialize)] +struct Promotion { + #[serde(default)] + tie_breaks_decided: Option, + #[serde(default)] + tie_break_outcome_agreement: Option, + #[serde(default)] + voter_outcome_agreement: Option, +} + +#[derive(Debug, Deserialize)] +struct Report { + #[serde(default)] + spec: Option, + #[serde(default)] + version: Option, + prereg_sha: String, + #[serde(default)] + exploratory: bool, + #[serde(default)] + lane: Lane, + #[serde(default)] + speed: Speed, + #[serde(default)] + results: Results, + #[serde(default)] + hardware_ruling: Option, + #[serde(default)] + promotion: Promotion, +} + +/// The ladder's decision and every reason it stopped below `vote`. +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +pub struct Decision { + pub mode: Mode, + pub reasons: Vec, +} + +fn verdict_holds<'a>(r: &'a Report, id: &str) -> Result<&'a Hypothesis, String> { + let h = r + .results + .hypotheses + .iter() + .find(|h| h.id == id) + .ok_or_else(|| format!("{id}: not in the report"))?; + if h.verdict.as_deref() == Some("holds") { + Ok(h) + } else { + Err(format!("{id}: verdict {:?}", h.verdict)) + } +} + +/// H4/H5: an explicit `holds` verdict whose Holm-adjusted p is finite and ≤ α. +fn holm_holds(r: &Report, id: &str) -> Result<(), String> { + match verdict_holds(r, id)?.p_holm { + Some(p) if p.is_finite() && (0.0..=crate::stats::ALPHA).contains(&p) => Ok(()), + p => Err(format!( + "{id}: Holm p {p:?} is not ≤ {}", + crate::stats::ALPHA + )), + } +} + +fn shadow(reason: String) -> Decision { + Decision { + mode: Mode::Shadow, + reasons: vec![reason], + } +} + +/// The rung the evidence alone grants, and every unmet gate. +fn gates(r: &Report, reasons: &mut Vec) -> Mode { + let mut trip = Vec::new(); + trip.extend(holm_holds(r, "H4").err()); + if r.speed.within_budget != Some(true) { + trip.push(format!( + "speed: replay p95 within the queue budget is {:?}", + r.speed.within_budget + )); + } + if !trip.is_empty() { + reasons.extend(trip); + return Mode::Shadow; + } + let mut tb: Vec = [ + holm_holds(r, "H5"), + verdict_holds(r, "H7").map(|_| ()), + verdict_holds(r, "H1").map(|_| ()), + verdict_holds(r, "H2").map(|_| ()), + ] + .into_iter() + .filter_map(Result::err) + .collect(); + match r.lane.availability_7d { + Some(a) if a >= MIN_AVAILABILITY => {} + a => tb.push(format!("availability_7d {a:?} is not ≥ {MIN_AVAILABILITY}")), + } + if !tb.is_empty() { + reasons.extend(tb); + return Mode::Tripwire; + } + let p = &r.promotion; + let decided = p.tie_breaks_decided.unwrap_or(0); + if decided < MIN_TIE_BREAKS { + reasons.push(format!( + "tie_breaks_decided {decided} is not ≥ {MIN_TIE_BREAKS}" + )); + return Mode::TieBreaker; + } + match (p.tie_break_outcome_agreement, p.voter_outcome_agreement) { + (Some(t), Some(v)) if t >= v => Mode::Vote, + (t, v) => { + reasons.push(format!( + "tie-break outcome agreement {t:?} is not ≥ the voters' {v:?}" + )); + Mode::TieBreaker + } + } +} + +/// `true` when the primary cell is lambda or unnamed (R-9: lambda may not sit +/// at tie-breaker or above). +fn s4(r: &Report) -> bool { + r.hardware_ruling + .as_ref() + .and_then(|h| h.primary.as_deref()) + .is_none_or(|p| p.starts_with("lambda")) +} + +fn read(report_json: &str, prereg_sha: &str) -> Result { + let r: Report = serde_json::from_str(report_json) + .map_err(|e| shadow(format!("report does not parse: {e}")))?; + if r.spec.as_deref() != Some(SPEC) || r.version != Some(SPEC_VERSION) { + return Err(shadow(format!( + "spec {:?} version {:?} is not {SPEC} v{SPEC_VERSION} (prm-001-report-v1)", + r.spec, r.version + ))); + } + if r.prereg_sha != prereg_sha { + return Err(shadow("prereg_sha differs from the lock".into())); + } + if r.exploratory { + return Err(shadow("exploratory report (R-1)".into())); + } + Ok(r) +} + +/// The rung one report grants, capped by its hardware ruling. Never errors: +/// an unreadable, foreign, exploratory or stale report is shadow. +#[must_use] +pub fn evidence(report_json: &str, prereg_sha: &str) -> Decision { + match read(report_json, prereg_sha) { + Ok(r) => capped(&r), + Err(d) => d, + } +} + +fn capped(r: &Report) -> Decision { + let mut reasons = Vec::new(); + let mut mode = gates(r, &mut reasons); + let Some(h) = &r.hardware_ruling else { + reasons.push("no hardware ruling".into()); + return Decision { + mode: Mode::Shadow, + reasons, + }; + }; + let cap = h.rung_eligible.unwrap_or(Mode::Shadow); + if cap < mode { + reasons.push(format!("hardware ruling caps the lane at {cap:?}")); + mode = cap; + } + if s4(r) && mode > Mode::Tripwire { + reasons.push(format!( + "S-4: primary cell {} is lambda or unnamed; lambda is shadow-only (R-9)", + h.primary.as_deref().unwrap_or("(unnamed)") + )); + mode = Mode::Tripwire; + } + Decision { mode, reasons } +} + +/// Step the lane from `current` on one report: at most one rung up, and a +/// demotion drops exactly one rung. A lambda or unnamed primary cell above +/// tripwire is capped at once (S-4), the one exception to the one-rung rule. +#[must_use] +pub fn decide(report_json: &str, prereg_sha: &str, current: Mode) -> Decision { + let (mut d, lambda) = match read(report_json, prereg_sha) { + Ok(r) => (capped(&r), s4(&r)), + Err(d) => (d, false), + }; + if lambda && current > Mode::Tripwire { + d.mode = Mode::Tripwire; + return d; + } + let stepped = d.mode.clamp(current.down(), current.up()); + if stepped != d.mode { + d.reasons.push(format!( + "one rung per report: {current:?} → {stepped:?}, not {:?}", + d.mode + )); + d.mode = stepped; + } + d +} + +#[cfg(test)] +#[path = "ladder_tests.rs"] +mod tests; diff --git a/crates/aprender-review-experiment/src/ladder_tests.rs b/crates/aprender-review-experiment/src/ladder_tests.rs new file mode 100644 index 0000000000..250f71d454 --- /dev/null +++ b/crates/aprender-review-experiment/src/ladder_tests.rs @@ -0,0 +1,316 @@ +// `json!` expands to an `unwrap` of an infallible `to_value`. +#![allow(clippy::disallowed_methods)] + +use super::*; +use serde_json::{json, Value}; + +const PREREG: &str = "ef51087dc79bab0ad160e8a14f5b13e2ea43986b30c05dafc63c84c2dc21cdc0"; + +fn h(id: &str, p: f64) -> Value { + json!({"id": id, "ci": [0.01, 0.2], "p_holm": p, "verdict": "holds"}) +} + +/// A `prm-001-report-v1` report on which every gate up to vote passes. +fn full() -> Value { + json!({ + "spec": "PRM-001", + "version": 3, + "prereg_sha": PREREG, + "lane": {"rung": "tie-breaker", "availability_7d": 0.97}, + "speed": {"within_budget": true}, + "results": {"hypotheses": [ + {"id": "H1", "verdict": "holds"}, + {"id": "H2", "verdict": "holds"}, + h("H4", 0.01), + h("H5", 0.04), + {"id": "H7", "verdict": "holds"}, + ]}, + "hardware_ruling": {"primary": "gx10-cpu", "rung_eligible": "vote"}, + "promotion": { + "tie_breaks_decided": 20, + "tie_break_outcome_agreement": 0.9, + "voter_outcome_agreement": 0.9 + } + }) +} + +fn set(mut v: Value, ptr: &str, x: Value) -> Value { + *v.pointer_mut(ptr).unwrap_or_else(|| panic!("{ptr}")) = x; + v +} + +fn drop_h(mut v: Value, id: &str) -> Value { + let hs = v["results"]["hypotheses"] + .as_array_mut() + .expect("hypotheses"); + hs.retain(|h| h["id"] != id); + v +} + +fn replace_h(v: Value, x: Value) -> Value { + let id = x["id"].as_str().expect("id").to_string(); + let mut v = drop_h(v, &id); + v["results"]["hypotheses"] + .as_array_mut() + .expect("hypotheses") + .push(x); + v +} + +/// The rung the evidence grants, before the one-rung step. +fn rung(v: &Value) -> Mode { + evidence(&v.to_string(), PREREG).mode +} + +#[test] +fn every_gate_passed_is_a_full_vote() { + let d = evidence(&full().to_string(), PREREG); + assert_eq!(d.mode, Mode::Vote, "{:?}", d.reasons); + assert!(d.reasons.is_empty()); + assert_eq!( + serde_json::to_value(Mode::TieBreaker).expect("mode"), + json!("tie-breaker") + ); +} + +/// FALSIFY-RXG-001: a failing H4 (the planted report) keeps the lane in shadow. +#[test] +fn falsify_rxg_001_a_failing_h4_keeps_the_lane_in_shadow() { + let fail = json!({"id": "H4", "ci": [-0.05, 0.1], "p_holm": 0.3, "verdict": "fails"}); + assert_eq!(rung(&replace_h(full(), fail)), Mode::Shadow); + for bad in [ + json!({"id": "H4", "p_holm": 0.051, "verdict": "holds"}), + json!({"id": "H4", "p_holm": -0.1, "verdict": "holds"}), + json!({"id": "H4", "verdict": "holds"}), + ] { + assert_eq!(rung(&replace_h(full(), bad.clone())), Mode::Shadow, "{bad}"); + } + assert_eq!(rung(&drop_h(full(), "H4")), Mode::Shadow); + let nan = full() + .to_string() + .replace("\"p_holm\":0.01", "\"p_holm\":NaN"); + assert_eq!(evidence(&nan, PREREG).mode, Mode::Shadow); +} + +/// FALSIFY-RXG-002: H4 alone is tripwire, never above; the speed gate and the +/// hardware ruling cap the rung. +#[test] +fn falsify_rxg_002_h4_alone_is_tripwire_never_vote() { + let h5_fail = json!({"id": "H5", "p_holm": 0.6, "verdict": "fails"}); + let d = evidence(&replace_h(full(), h5_fail).to_string(), PREREG); + assert_eq!(d.mode, Mode::Tripwire, "{:?}", d.reasons); + assert!( + d.reasons.iter().any(|r| r.starts_with("H5")), + "{:?}", + d.reasons + ); + // the planted report: H4 only + let h4_only = drop_h(drop_h(drop_h(drop_h(full(), "H5"), "H7"), "H1"), "H2"); + assert_eq!(rung(&h4_only), Mode::Tripwire); + // speed: replay p95 over the queue budget, or unmeasured, is shadow + assert_eq!( + rung(&set(full(), "/speed/within_budget", json!(false))), + Mode::Shadow + ); + let mut no_speed = full(); + no_speed.as_object_mut().expect("obj").remove("speed"); + assert_eq!(rung(&no_speed), Mode::Shadow); + // the hardware ruling caps; none is shadow + for (cap, want) in [ + ("shadow", Mode::Shadow), + ("tripwire", Mode::Tripwire), + ("tie-breaker", Mode::TieBreaker), + ] { + let v = set(full(), "/hardware_ruling/rung_eligible", json!(cap)); + assert_eq!(rung(&v), want, "{cap}"); + } + // a ruling that names no eligible rung grants none + let mut no_rung = full(); + no_rung["hardware_ruling"] + .as_object_mut() + .expect("ruling") + .remove("rung_eligible"); + assert_eq!(rung(&no_rung), Mode::Shadow); + let mut no_ruling = full(); + no_ruling + .as_object_mut() + .expect("obj") + .remove("hardware_ruling"); + assert_eq!(rung(&no_ruling), Mode::Shadow); +} + +/// FALSIFY-RXG-003: only a locked, confirmatory prm-001-report-v1 is read. +#[test] +fn falsify_rxg_003_only_a_locked_confirmatory_report_is_read() { + let old = full() + .to_string() + .replace("\"spec\":\"PRM-001\"", "\"schema\":\"rex-001-report-v2\""); + let cases = [ + ("rex-001-report-v2", old), + ("v2 spec", set(full(), "/version", json!(2)).to_string()), + ( + "stale prereg", + set(full(), "/prereg_sha", json!("0".repeat(64))).to_string(), + ), + ("exploratory", { + let mut v = full(); + v["exploratory"] = json!(true); + v.to_string() + }), + ("not json", "{".to_string()), + ]; + for (name, text) in cases { + assert_eq!(evidence(&text, PREREG).mode, Mode::Shadow, "{name}"); + } +} + +/// FALSIFY-RXG-004: the CI is report-only; the Holm p decides H4/H5. +#[test] +fn falsify_rxg_004_the_ci_is_report_only() { + let crossing = json!({"id": "H4", "ci": [-0.01, 0.2], "p_holm": 0.02, "verdict": "holds"}); + assert_eq!(rung(&replace_h(full(), crossing)), Mode::Vote); + let clean = json!({"id": "H4", "ci": [0.05, 0.2], "p_holm": 0.2, "verdict": "holds"}); + assert_eq!(rung(&replace_h(full(), clean)), Mode::Shadow); +} + +/// FALSIFY-RXG-005: tie-breaker needs H4 ∧ H5 ∧ H7 ∧ H1 ∧ H2 and 7-day +/// availability ≥ 95%; without any one of them the lane stays tripwire. +#[test] +fn falsify_rxg_005_tie_breaker_needs_h4_h5_h7_h1_h2_and_availability() { + let tb = set(full(), "/promotion/tie_breaks_decided", json!(0)); + assert_eq!(rung(&tb), Mode::TieBreaker); + for id in ["H5", "H7", "H1", "H2"] { + let d = evidence(&drop_h(tb.clone(), id).to_string(), PREREG); + assert_eq!(d.mode, Mode::Tripwire, "without {id}"); + assert!( + d.reasons.iter().any(|r| r.starts_with(id)), + "{id}: {:?}", + d.reasons + ); + let fails = json!({"id": id, "p_holm": 0.01, "verdict": "fails"}); + assert_eq!( + rung(&replace_h(tb.clone(), fails)), + Mode::Tripwire, + "{id} fails" + ); + } + // H5 is in the Holm family: a holds verdict over α does not hold + let h5_p = json!({"id": "H5", "p_holm": 0.6, "verdict": "holds"}); + assert_eq!(rung(&replace_h(tb.clone(), h5_p)), Mode::Tripwire); + // H7 is outside the Holm family: its verdict decides, a p is not needed + assert!(full()["results"]["hypotheses"][4].get("p_holm").is_none()); + // availability: inclusive at 0.95, below or absent is tripwire + assert_eq!( + rung(&set(tb.clone(), "/lane/availability_7d", json!(0.95))), + Mode::TieBreaker + ); + assert_eq!( + rung(&set(tb.clone(), "/lane/availability_7d", json!(0.9499))), + Mode::Tripwire + ); + let mut no_avail = tb.clone(); + no_avail["lane"] + .as_object_mut() + .expect("lane") + .remove("availability_7d"); + assert_eq!(rung(&no_avail), Mode::Tripwire); +} + +/// FALSIFY-RXG-006: a lambda primary cell is refused at tie-breaker and above +/// (R-9, S-4), whatever the evidence. +#[test] +fn falsify_rxg_006_lambda_primary_is_refused_above_tripwire() { + let v = set(full(), "/hardware_ruling/primary", json!("lambda-cuda")); + let d = evidence(&v.to_string(), PREREG); + assert_eq!(d.mode, Mode::Tripwire, "{:?}", d.reasons); + assert!( + d.reasons.iter().any(|r| r.contains("S-4")), + "{:?}", + d.reasons + ); + // the reason names the cell as written, never a Rust Debug rendering + assert!( + d.reasons + .iter() + .any(|r| r.contains("primary cell lambda-cuda is") && !r.contains("Some(")), + "{:?}", + d.reasons + ); + // a missing primary is not a non-lambda cell + let mut none = full(); + none["hardware_ruling"] + .as_object_mut() + .expect("ruling") + .remove("primary"); + assert_eq!(rung(&none), Mode::Tripwire); + let n = evidence(&none.to_string(), PREREG); + assert!( + n.reasons + .iter() + .any(|r| r.contains("primary cell (unnamed) is")), + "{:?}", + n.reasons + ); +} + +/// FALSIFY-RXG-007: vote needs ≥ 20 decided tie-breaks whose 14-day outcome +/// agreement is at least the voters' on the same rounds. +#[test] +fn falsify_rxg_007_vote_needs_twenty_agreeing_tie_breaks() { + assert_eq!( + rung(&set(full(), "/promotion/tie_breaks_decided", json!(19))), + Mode::TieBreaker + ); + assert_eq!( + rung(&set( + full(), + "/promotion/tie_break_outcome_agreement", + json!(0.89) + )), + Mode::TieBreaker + ); + let mut none = full(); + none.as_object_mut().expect("obj").remove("promotion"); + assert_eq!(rung(&none), Mode::TieBreaker); + let mut no_voters = full(); + no_voters["promotion"] + .as_object_mut() + .expect("promotion") + .remove("voter_outcome_agreement"); + assert_eq!(rung(&no_voters), Mode::TieBreaker); +} + +/// FALSIFY-RXG-008: the ladder never skips a rung, and a demotion drops +/// exactly one rung; S-4 is the exception and caps at once. +#[test] +fn falsify_rxg_008_never_skips_a_rung_and_demotes_one() { + let all = full().to_string(); + let step = |cur| decide(&all, PREREG, cur).mode; + assert_eq!(step(Mode::Shadow), Mode::Tripwire); + assert_eq!(step(Mode::Tripwire), Mode::TieBreaker); + assert_eq!(step(Mode::TieBreaker), Mode::Vote); + let d = decide("{", PREREG, Mode::Vote); + assert_eq!(d.mode, Mode::TieBreaker, "{:?}", d.reasons); + assert_eq!(decide("{", PREREG, Mode::Tripwire).mode, Mode::Shadow); + assert_eq!(decide("{", PREREG, Mode::Shadow).mode, Mode::Shadow); + let lambda = set(full(), "/hardware_ruling/primary", json!("lambda-cuda")).to_string(); + // S-4 caps a vote lane at tripwire at once, but never promotes past one rung + assert_eq!(decide(&lambda, PREREG, Mode::Vote).mode, Mode::Tripwire); + assert_eq!(decide(&lambda, PREREG, Mode::Shadow).mode, Mode::Tripwire); + assert_eq!( + decide(&lambda, PREREG, Mode::TieBreaker).mode, + Mode::Tripwire + ); + // S-4 never promotes: a lambda lane with no evidence stays in shadow + let lambda_h4_fail = drop_h( + set(full(), "/hardware_ruling/primary", json!("lambda-cuda")), + "H4", + ); + assert_eq!( + decide(&lambda_h4_fail.to_string(), PREREG, Mode::Shadow).mode, + Mode::Shadow + ); + // a tripwire-only report demotes a vote lane one rung, not two + let h4_only = drop_h(drop_h(full(), "H5"), "H7").to_string(); + assert_eq!(decide(&h4_only, PREREG, Mode::Vote).mode, Mode::TieBreaker); +} diff --git a/crates/aprender-review-experiment/src/ledger.rs b/crates/aprender-review-experiment/src/ledger.rs new file mode 100644 index 0000000000..9f63fffc13 --- /dev/null +++ b/crates/aprender-review-experiment/src/ledger.rs @@ -0,0 +1,343 @@ +//! REX-07 day-0 shadow lane: `review-ledger-v2` rows built from quorum +//! receipts (`docs/audits/quorum-*.json`), and the shadow-coverage check. +//! v2 (PRA-001 T2) keeps each counted lane's findings text and its +//! agent-trace-v1 shas; v1 rows stay readable through [`read_row`] and +//! migrate with [`v1::Row::migrate`]. Outcomes are appended as +//! [`OutcomeJoin`]s; a written row is never rewritten. +//! +//! The 4B lane runs inside the quorum rail as a non-voting advisory lane +//! (paiml-implement `advisory-lane.sh`, which records `.advisory_lane` in the +//! receipt). This module does not run the lane; it reads what the rail wrote +//! and holds it to §7 REX-07: +//! - every quorum after activation carries a shadow row (a receipt with no +//! `advisory_lane` is **missing**, never silently skipped); +//! - the apr row is one typed value, `Verdict | NotRun | Refused` (the +//! paiml-implement#436 wire form, one spelling with receipt-lint); an +//! unknown key or value fails to parse, so free text is never a reason; +//! - the shadow is never counted, and the quorum width is the counted lanes +//! alone, so a gx10-down round keeps its width. + +use serde::{Deserialize, Serialize}; + +pub const SCHEME: &str = "review-ledger-v2"; + +/// Rows written before v2 stay readable: see [`v1`] and [`read_row`]. +pub const SCHEME_V1: &str = "review-ledger-v1"; + +#[path = "ledger_v1.rs"] +pub mod v1; + +#[derive(Debug, Deserialize)] +struct Lane { + #[serde(default)] + family: Option, + #[serde(default)] + model: Option, + #[serde(default)] + verdict: Option, + #[serde(default)] + findings: Option>, + /// agent-trace-v1 blob shas (exact bytes sent / full raw reply). + #[serde(default)] + input_sha256: Option, + #[serde(default)] + output_sha256: Option, +} + +#[derive(Debug, Deserialize)] +struct Advisory { + #[serde(default)] + state: Option, + /// The typed apr row. Absent on an `off` lane; absent otherwise is a violation. + #[serde(default)] + row: Option, + #[serde(default)] + apr: Option, + #[serde(default)] + weights_sha256: Option, + /// The rail's uncounted flag (`counted` there is the counted round's verdict). + #[serde(default)] + counts: Option, +} + +#[derive(Debug, Deserialize)] +struct QuorumReceipt { + ticket: String, + head: String, + #[serde(default)] + pr: Option, + #[serde(default)] + diff_sha256: Option, + #[serde(default)] + agreed: Option, + #[serde(default)] + width: Option, + #[serde(default)] + lanes: Vec, + #[serde(default)] + advisory_lane: Option, +} + +/// One counted lane, as the ledger keeps it. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct LaneRow { + pub family: Option, + pub model: Option, + pub verdict: Option, + /// Carried by both versions; equals the text's length when text exists. + pub findings_count: usize, + /// The findings text, verbatim: a count cannot train anything. `None` is + /// text never captured (a v1 row, or a receipt without the key): a trace + /// gap, never an empty list. + pub findings: Option>, + /// agent-trace-v1 blob shas; a lane without both is a trace gap. + pub input_sha256: Option, + pub output_sha256: Option, +} + +/// The apr lane's verdict. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)] +pub enum Verdict { + #[serde(rename = "PASS")] + Pass, + #[serde(rename = "FAIL")] + Fail, +} + +/// Why the apr lane did not run. Closed: a new reason is a schema change. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +pub enum NotRun { + NoExecutor, + Busy, + Timeout, + ContextOverflow, + TrainActive, +} + +/// What removed a cell from the ladder. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub enum RemovedBy { + Ladder, + GpuProof, + Parse, +} + +/// The apr lane's result on one quorum — the paiml-implement#436 wire form: +/// `{"Verdict":{"verdict":"PASS","cell":"gx10-cuda","backend":"cuda"}}`, +/// `{"NotRun":"Busy"}`, `{"Refused":{"cell":"gx10-cuda","removed_by":"gpu-proof"}}`. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub enum Shadow { + Verdict { + verdict: Verdict, + cell: String, + backend: String, + }, + NotRun(NotRun), + Refused { + cell: String, + removed_by: RemovedBy, + }, +} + +/// One `review-ledger-v2` row (§5.2; PRA-001 T2). +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Row { + pub schema: String, + pub repo: String, + pub ticket: String, + pub head: String, + pub pr: Option, + pub diff_sha256: Option, + pub agreed: Option, + pub width: usize, + pub lanes: Vec, + pub shadow: Shadow, + pub apr_tag: Option, + /// From the receipt; a row without it is an identity gap. + pub weights_sha256: Option, + /// Joined later (merged / reverted ≤ 14 d / escape); `pending` until then. + pub outcome: String, +} + +/// What a set of receipts resolves to. +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize)] +pub struct Coverage { + pub quorums: usize, + pub carried: usize, + /// Receipts with no shadow row: REX-07 requires 0. + pub missing: Vec, + /// Receipts where the shadow was counted or changed the width. + pub violations: Vec, + /// Rows without an apr tag or weights sha (the §5.2 identity fields). + pub identity_gaps: usize, + /// Counted lanes without findings text or both agent-trace-v1 shas: the + /// Jidoka goal is 0. + pub trace_gaps: usize, +} + +impl Coverage { + /// §7 REX-07 acceptance: every quorum carries a shadow row, none counted, + /// and every row names its apr tag and weights sha (PRM v2 item 5). + #[must_use] + pub fn holds(&self) -> bool { + self.quorums > 0 + && self.missing.is_empty() + && self.violations.is_empty() + && self.identity_gaps == 0 + && self.trace_gaps == 0 + } +} + +/// A finding as text: a string stays verbatim, anything else is its JSON. +fn finding_text(v: &serde_json::Value) -> String { + v.as_str().map_or_else(|| v.to_string(), str::to_owned) +} + +/// An outcome join, appended to its own JSONL. Rows are never rewritten. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct OutcomeJoin { + pub repo: String, + pub head: String, + pub outcome: String, + /// RFC 3339 UTC; the latest join for a (repo, head) wins. + pub at: String, +} + +/// A row's outcome read through the append-only joins, latest first; +/// the row's own value (`pending`) when nothing joined. +#[must_use] +pub fn outcome<'a>(row: &'a Row, joins: &'a [OutcomeJoin]) -> &'a str { + joins + .iter() + .filter(|j| j.repo == row.repo && j.head == row.head) + .max_by(|a, b| a.at.cmp(&b.at)) + .map_or(row.outcome.as_str(), |j| j.outcome.as_str()) +} + +/// A counted lane whose trace a fine-tune could not use: no findings text, +/// or not both agent-trace-v1 shas. +fn trace_gap(l: &LaneRow) -> bool { + l.findings.is_none() || l.input_sha256.is_none() || l.output_sha256.is_none() +} + +/// A ledger line of either version, dispatched on its `schema`. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum AnyRow { + V1(v1::Row), + V2(Row), +} + +/// Read one ledger line. An unknown `schema` is an error, never a guess. +pub fn read_row(line: &str) -> Result { + #[derive(Deserialize)] + struct Head { + schema: String, + } + let h: Head = serde_json::from_str(line).map_err(|e| e.to_string())?; + match h.schema.as_str() { + SCHEME_V1 => serde_json::from_str(line) + .map(AnyRow::V1) + .map_err(|e| format!("{SCHEME_V1}: {e}")), + SCHEME => serde_json::from_str(line) + .map(AnyRow::V2) + .map_err(|e| format!("{SCHEME}: {e}")), + s => Err(format!("unknown ledger schema {s:?}")), + } +} + +/// Build ledger rows from `(name, json)` receipts and measure shadow coverage. +/// A receipt that does not parse is a violation, never dropped. +#[must_use] +pub fn build(receipts: &[(String, String)], repo: &str) -> (Vec, Coverage) { + let mut rows = Vec::new(); + let mut c = Coverage { + quorums: receipts.len(), + ..Coverage::default() + }; + for (name, text) in receipts { + let q: QuorumReceipt = match serde_json::from_str(text) { + Ok(q) => q, + Err(e) => { + c.violations.push(format!("{name}: {e}")); + continue; + } + }; + // `off` is the rail saying the lane was not run: no shadow row. + let Some(a) = q + .advisory_lane + .as_ref() + .filter(|a| a.state.as_deref() != Some("off")) + else { + c.missing.push(name.clone()); + continue; + }; + let shadow = match a.row.clone().map(serde_json::from_value::) { + Some(Ok(s)) => s, + Some(Err(e)) => { + c.violations.push(format!("{name}: apr row: {e}")); + continue; + } + None => { + c.violations.push(format!("{name}: no typed apr row")); + continue; + } + }; + c.carried += 1; + if a.counts != Some(false) { + c.violations + .push(format!("{name}: shadow lane not recorded as uncounted")); + } + let width = q.lanes.len(); + if q.width.is_some_and(|w| w != width as u64) { + c.violations.push(format!( + "{name}: width {:?} is not the {width} counted lanes", + q.width + )); + } + let row = Row { + schema: SCHEME.into(), + repo: repo.into(), + ticket: q.ticket, + head: q.head, + pr: q.pr, + diff_sha256: q.diff_sha256, + agreed: q.agreed, + width, + lanes: q + .lanes + .iter() + .map(|l| LaneRow { + family: l.family.clone(), + model: l.model.clone(), + verdict: l.verdict.clone(), + findings_count: l.findings.as_ref().map_or(0, Vec::len), + findings: l + .findings + .as_ref() + .map(|f| f.iter().map(finding_text).collect()), + input_sha256: l.input_sha256.clone(), + output_sha256: l.output_sha256.clone(), + }) + .collect(), + shadow, + apr_tag: a.apr.clone(), + weights_sha256: a.weights_sha256.clone(), + outcome: "pending".into(), + }; + if row.apr_tag.is_none() || row.weights_sha256.is_none() { + c.identity_gaps += 1; + } + c.trace_gaps += row.lanes.iter().filter(|l| trace_gap(l)).count(); + rows.push(row); + } + (rows, c) +} + +#[cfg(test)] +#[path = "ledger_tests.rs"] +mod tests; diff --git a/crates/aprender-review-experiment/src/ledger_tests.rs b/crates/aprender-review-experiment/src/ledger_tests.rs new file mode 100644 index 0000000000..5239ed13b2 --- /dev/null +++ b/crates/aprender-review-experiment/src/ledger_tests.rs @@ -0,0 +1,212 @@ +use super::*; + +fn receipt(advisory: Option<&str>, width: u64) -> String { + let lanes = r#"[{"family":"claude","model":"claude-sonnet-5","verdict":"PASS","findings":[],"input_sha256":"i1","output_sha256":"o1"}, + {"family":"gemini","model":"gemini-3.1-pro-high","verdict":"PASS","findings":["ledger.rs:205 drops the weights sha",{"f":1}],"input_sha256":"i2","output_sha256":"o2"}, + {"family":"claude","model":"claude-haiku-4-5","verdict":"PASS","findings":[],"input_sha256":"i3","output_sha256":"o3"}]"#; + let adv = advisory.map_or(String::new(), |a| format!(r#","advisory_lane":{a}"#)); + format!( + r#"{{"ticket":"PMAT-1","head":"abc","diff_sha256":"d","agreed":true,"width":{width},"lanes":{lanes}{adv}}}"# + ) +} + +const ANSWERED: &str = r#"{"state":"answered","row":{"Verdict":{"verdict":"PASS","cell":"gx10-cuda","backend":"cuda"}},"apr":"0.69.3","weights_sha256":"w4b","counted":"PASS","counts":false}"#; +const DOWN: &str = r#"{"state":"unavailable","row":{"NotRun":"NoExecutor"},"apr":"0.69.3","weights_sha256":"w4b","counted":"PASS","counts":false}"#; + +fn one(text: String) -> (Vec, Coverage) { + build(&[("quorum-PMAT-1.json".into(), text)], "paiml/aprender") +} + +#[test] +fn an_answered_shadow_becomes_a_ledger_row() { + let (rows, c) = one(receipt(Some(ANSWERED), 3)); + assert!(c.holds(), "{c:?}"); + let r = &rows[0]; + assert_eq!(r.schema, SCHEME); + assert_eq!(r.width, 3); + assert_eq!( + r.lanes[1].findings, + Some(vec![ + "ledger.rs:205 drops the weights sha".to_string(), + r#"{"f":1}"#.to_string() + ]), + "findings are kept as text, verbatim" + ); + assert_eq!(r.lanes[1].output_sha256.as_deref(), Some("o2")); + assert_eq!(c.trace_gaps, 0); + assert_eq!( + r.shadow, + Shadow::Verdict { + verdict: Verdict::Pass, + cell: "gx10-cuda".into(), + backend: "cuda".into() + } + ); + assert_eq!(r.weights_sha256.as_deref(), Some("w4b")); + assert_eq!(r.outcome, "pending"); + assert_eq!(c.identity_gaps, 0); +} + +#[test] +fn falsify_rxl_001_a_quorum_without_a_shadow_row_is_missing() { + let recs = vec![ + ("a.json".to_string(), receipt(Some(ANSWERED), 3)), + ("b.json".to_string(), receipt(None, 3)), + ]; + let (rows, c) = build(&recs, "paiml/aprender"); + assert_eq!(rows.len(), 1); + assert_eq!((c.quorums, c.carried), (2, 1)); + assert_eq!(c.missing, ["b.json"]); + assert!(!c.holds(), "one missing shadow row fails REX-07"); + assert!(!build(&[], "r").1.holds(), "no quorum is not 100 %"); + let off = r#"{"state":"off","counts":false,"why":"advisory_lane disabled"}"#; + assert!( + !one(receipt(Some(off), 3)).1.holds(), + "a lane that was off is not a shadow row" + ); + let (_, bad) = build(&[("x".into(), "{".into())], "r"); + assert!(!bad.holds(), "an unparsable receipt is not dropped"); +} + +#[test] +fn falsify_rxl_002_gx10_down_is_unknown_and_keeps_the_width() { + let (rows, c) = one(receipt(Some(DOWN), 3)); + assert!(c.holds(), "{c:?}"); + assert_eq!(rows[0].width, 3); + assert_eq!(rows[0].shadow, Shadow::NotRun(NotRun::NoExecutor)); + // A verdict without its cell is not a verdict. + let half = ANSWERED.replace(r#","cell":"gx10-cuda""#, ""); + assert!(!one(receipt(Some(&half), 3)).1.holds(), "no cell"); +} + +#[test] +fn falsify_rxl_003_a_counted_or_width_changing_shadow_is_a_violation() { + let counted = ANSWERED.replace(r#""counts":false"#, r#""counts":true"#); + assert!(!one(receipt(Some(&counted), 3)).1.holds(), "counted"); + let silent = ANSWERED.replace(r#","counts":false"#, ""); + assert!( + !one(receipt(Some(&silent), 3)).1.holds(), + "uncounted unrecorded" + ); + assert!(!one(receipt(Some(ANSWERED), 4)).1.holds(), "width 4"); + assert!(!one(receipt(Some(DOWN), 2)).1.holds(), "width 2"); +} + +#[test] +fn every_wire_form_round_trips() { + for row in [ + r#"{"Verdict":{"verdict":"FAIL","cell":"intel-wgpu","backend":"wgpu"}}"#, + r#"{"NotRun":"NoExecutor"}"#, + r#"{"NotRun":"Busy"}"#, + r#"{"NotRun":"Timeout"}"#, + r#"{"NotRun":"ContextOverflow"}"#, + r#"{"NotRun":"TrainActive"}"#, + r#"{"Refused":{"cell":"gx10-cuda","removed_by":"ladder"}}"#, + r#"{"Refused":{"cell":"gx10-cuda","removed_by":"gpu-proof"}}"#, + r#"{"Refused":{"cell":"gx10-cuda","removed_by":"parse"}}"#, + ] { + let s: Shadow = serde_json::from_str(row).expect(row); + assert_eq!(serde_json::to_string(&s).expect("ser"), row); + } +} + +#[test] +fn falsify_rxl_004_a_receipt_without_a_weights_sha_does_not_hold() { + let bare = ANSWERED.replace(r#","weights_sha256":"w4b""#, ""); + let (rows, c) = one(receipt(Some(&bare), 3)); + assert_eq!(rows.len(), 1, "the row is kept"); + assert_eq!(c.identity_gaps, 1); + assert!(!c.holds(), "an identity gap fails holds()"); + let untagged = ANSWERED.replace(r#""apr":"0.69.3""#, r#""apr":null"#); + assert!(!one(receipt(Some(&untagged), 3)).1.holds(), "no apr tag"); +} + +#[test] +fn falsify_rxl_005_a_free_text_reason_fails_to_parse() { + for row in [ + r#"{"NotRun":"gx10 connect refused"}"#, + r#"{"LaneUnavailable":{"why":"gx10 connect refused"}}"#, + r#"{"Refused":{"cell":"gx10-cuda","removed_by":"aprender#9999: refuses qwen35"}}"#, + r#"{"Verdict":{"verdict":"LGTM","cell":"gx10-cuda","backend":"cuda"}}"#, + r#"{"Verdict":{"verdict":"PASS","cell":"gx10-cuda","backend":"cuda","why":"x"}}"#, + ] { + assert!(serde_json::from_str::(row).is_err(), "{row}"); + let adv = ANSWERED.replace( + r#"{"Verdict":{"verdict":"PASS","cell":"gx10-cuda","backend":"cuda"}}"#, + row, + ); + let (rows, c) = one(receipt(Some(&adv), 3)); + assert!(rows.is_empty() && !c.holds(), "{row} is a violation"); + } + let untyped = r#"{"state":"answered","apr":"0.69.3","weights_sha256":"w","counts":false}"#; + assert!(!one(receipt(Some(untyped), 3)).1.holds(), "no typed row"); +} + +#[test] +fn falsify_rxl_006_a_counted_lane_without_trace_shas_does_not_hold() { + for gap in [ + r#","output_sha256":"o2""#, + r#","input_sha256":"i2""#, + r#""findings":["ledger.rs:205 drops the weights sha",{"f":1}],"#, + ] { + let (rows, c) = one(receipt(Some(ANSWERED), 3).replace(gap, "")); + assert_eq!(rows.len(), 1, "the row is kept"); + assert_eq!(c.trace_gaps, 1, "{gap}"); + assert!(!c.holds(), "a lane without its trace is RED: {gap}"); + } +} + +#[test] +fn falsify_rxl_007_an_outcome_join_appends_and_never_rewrites() { + let (rows, _) = one(receipt(Some(ANSWERED), 3)); + let before = rows.clone(); + let j = |head: &str, outcome: &str, at: &str| OutcomeJoin { + repo: "paiml/aprender".into(), + head: head.into(), + outcome: outcome.into(), + at: at.into(), + }; + assert_eq!(outcome(&rows[0], &[]), "pending"); + let joins = [ + j("abc", "reverted", "2026-09-30T00:00:00Z"), + j("abc", "merged", "2026-09-25T00:00:00Z"), + j("other", "escape", "2026-10-01T00:00:00Z"), + ]; + assert_eq!(outcome(&rows[0], &joins), "reverted", "latest join wins"); + assert_eq!(rows, before, "the row itself is never rewritten"); +} + +// v1 rows exactly as the two v1 writers serialized them. +const V1_LEGACY: &str = r#"{"schema":"review-ledger-v1","repo":"paiml/aprender","ticket":"PMAT-1","head":"abc","pr":null,"diff_sha256":"d","agreed":true,"width":3,"lanes":[{"family":"claude","model":"claude-sonnet-5","verdict":"PASS","findings":2}],"shadow":{"state":"Unknown","reason":{"LaneUnavailable":{"why":"gx10 down"}}},"apr_tag":null,"weights_sha256":null,"outcome":"pending"}"#; +const V1_TYPED: &str = r#"{"schema":"review-ledger-v1","repo":"paiml/aprender","ticket":"PMAT-1","head":"abc","pr":4354,"diff_sha256":"d","agreed":true,"width":3,"lanes":[{"family":"gemini","model":"gemini-3.1-pro-high","verdict":"FAIL","findings":2}],"shadow":{"Verdict":{"verdict":"PASS","cell":"gx10-cuda","backend":"cuda"}},"apr_tag":"0.69.3","weights_sha256":"w4b","outcome":"merged"}"#; + +#[test] +fn falsify_rxl_008_v1_rows_parse_and_migrate_without_invented_text() { + for line in [V1_LEGACY, V1_TYPED] { + assert!( + matches!(read_row(line), Ok(AnyRow::V1(_))), + "{:?}", + read_row(line) + ); + } + let Ok(AnyRow::V1(typed)) = read_row(V1_TYPED) else { + panic!("v1 typed row") + }; + let m = typed.migrate().expect("a typed v1 row migrates"); + assert_eq!(m.schema, SCHEME); + assert_eq!(m.lanes[0].findings_count, 2, "the count carries over"); + assert_eq!(m.lanes[0].findings, None, "text never captured stays None"); + assert_eq!((m.pr, m.outcome.as_str()), (Some(4354), "merged")); + // A migrated row round-trips as v2 and still reads as a trace gap. + let line = serde_json::to_string(&m).expect("ser"); + assert_eq!(read_row(&line), Ok(AnyRow::V2(m.clone()))); + assert!(trace_gap(&m.lanes[0])); + // A legacy shadow has no cell/backend: refused, not guessed. + let Ok(AnyRow::V1(legacy)) = read_row(V1_LEGACY) else { + panic!("v1 legacy row") + }; + assert!(legacy.migrate().is_err()); + // A v1 body under the v2 name, and an unknown schema, are errors. + assert!(read_row(&V1_TYPED.replace(SCHEME_V1, SCHEME)).is_err()); + assert!(read_row(&V1_TYPED.replace(SCHEME_V1, "review-ledger-v9")).is_err()); +} diff --git a/crates/aprender-review-experiment/src/ledger_v1.rs b/crates/aprender-review-experiment/src/ledger_v1.rs new file mode 100644 index 0000000000..a4fa330cd0 --- /dev/null +++ b/crates/aprender-review-experiment/src/ledger_v1.rs @@ -0,0 +1,105 @@ +//! The frozen `review-ledger-v1` row, kept so rows written before v2 stay +//! readable. Two shapes were written under that one schema string: the +//! REX-07 original (402aff870, a `state`-tagged shadow) and PRM v2 (a) +//! (89d994bc1, the typed paiml-implement#436 shadow). Both parse here. +//! Nothing writes v1 any more. + +use serde::Deserialize; + +use super::Shadow; + +/// A v1 counted lane: findings were stored as a count only. +#[derive(Debug, Clone, PartialEq, Eq, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct LaneRow { + pub family: Option, + pub model: Option, + pub verdict: Option, + pub findings: usize, +} + +/// 402aff870's reason for a missing shadow verdict. +#[derive(Debug, Clone, PartialEq, Eq, Deserialize)] +pub enum Unknown { + LaneUnavailable { why: Option }, +} + +/// 402aff870's shadow, before the #436 wire form existed. +#[derive(Debug, Clone, PartialEq, Eq, Deserialize)] +#[serde(tag = "state")] +pub enum LegacyShadow { + Answered { verdict: String, served_by: String }, + Unknown { reason: Unknown }, +} + +/// Either v1 shadow; the typed form is tried first. +#[derive(Debug, Clone, PartialEq, Eq, Deserialize)] +#[serde(untagged)] +pub enum V1Shadow { + Typed(Shadow), + Legacy(LegacyShadow), +} + +/// One `review-ledger-v1` row. +#[derive(Debug, Clone, PartialEq, Eq, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Row { + pub schema: String, + pub repo: String, + pub ticket: String, + pub head: String, + pub pr: Option, + pub diff_sha256: Option, + pub agreed: Option, + pub width: usize, + pub lanes: Vec, + pub shadow: V1Shadow, + pub apr_tag: Option, + pub weights_sha256: Option, + pub outcome: String, +} + +impl Row { + /// Migrate to v2. The finding count carries over. Findings text and + /// trace shas were never captured, so they stay `None` (a trace gap), + /// never an empty list. A legacy shadow has no cell or backend, so it + /// has no typed #436 form: it is refused and stays readable as v1. + pub fn migrate(self) -> Result { + let shadow = match self.shadow { + V1Shadow::Typed(s) => s, + V1Shadow::Legacy(l) => { + return Err(format!( + "{}@{}: legacy shadow {l:?} has no typed #436 form", + self.ticket, self.head + )) + } + }; + Ok(super::Row { + schema: super::SCHEME.into(), + repo: self.repo, + ticket: self.ticket, + head: self.head, + pr: self.pr, + diff_sha256: self.diff_sha256, + agreed: self.agreed, + width: self.width, + lanes: self + .lanes + .into_iter() + .map(|l| super::LaneRow { + family: l.family, + model: l.model, + verdict: l.verdict, + findings_count: l.findings, + findings: None, + input_sha256: None, + output_sha256: None, + }) + .collect(), + shadow, + apr_tag: self.apr_tag, + weights_sha256: self.weights_sha256, + outcome: self.outcome, + }) + } +} diff --git a/crates/aprender-review-experiment/src/lib.rs b/crates/aprender-review-experiment/src/lib.rs new file mode 100644 index 0000000000..7993fd967a --- /dev/null +++ b/crates/aprender-review-experiment/src/lib.rs @@ -0,0 +1,50 @@ +//! PRM-001 PROMETHEUS review-lane experiment (spec `docs/specifications/PRM-001-prometheus.md`; was REX-001). +//! +//! - [`prereg`]: the REX-00 pre-registration lock (`rex-prereg-v2`). +//! - [`stats`]: the frozen analysis code — Wilson, McNemar exact, seeded +//! bootstrap, Holm, H6 interference. +//! - [`corpus`]: the REX-02 review corpus (`review-corpus-v1`). +//! - [`contamination`]: the sealed-test leak check (`review-corpus-contamination-v1`). +//! - [`receipt`]: the REX-03 receipt schema, verdict parser and admissibility +//! (`review-experiment-receipt-v1`). +//! - [`score`]: the REX-03 scorer (§2.3 metrics, §3 hypothesis inputs). +//! - [`harness`]: the REX-03 client that drives a resident `apr serve`. +//! - [`admission`]: the REX-04 per-cell admission file (`rex-cell-admission-v1`). +//! - [`pilot`]: the REX-05 pilot projection and the §2.2 sample-size rule. +//! - [`ledger`]: the REX-07 shadow-lane `review-ledger-v1` rows and coverage. +//! - [`ladder`]: the REX-09 shadow → tripwire → vote promotion gates (H4/H5). +//! - [`ratchet`]: the REX-10 `review-lane-perf-ratchet-v1` p95 ratchet (§5.1). +//! - [`champion`]: the REX-11 §5.4 champion/challenger promotion gate. +//! - [`fewshot`]: the REX-11 B1 challengers: prompt versions and leak-proof retrieval few-shot. +//! - [`b2`]: the REX-12 B2 loop: verb-gated row status and the teacher-logit dataset receipt. +//! - [`datacard`]: PRM-C14 (was PRA T14): the Croissant + RAI card, Datasheet and G16 weekly +//! yield/bytes receipt over the agent-trace index (`trace-datacard-v1`). + +pub mod admission; +pub mod b2; +pub mod build_corpus; +pub mod champion; +pub mod cluster; +pub mod contamination; +pub mod corpus; +pub mod datacard; +pub mod dedup; +pub mod fewshot; +pub mod harness; +pub mod kappa_probe; +pub mod ladder; +pub mod ledger; +pub mod outcome_join; +pub mod pilot; +pub mod pool; +pub mod prereg; +pub mod ratchet; +pub mod receipt; +pub mod replay; +pub mod score; +pub mod secret; +pub mod sparse_logits; +pub mod split_guard; +pub mod stats; +pub mod terms; +pub mod workload; diff --git a/crates/aprender-review-experiment/src/outcome_join.rs b/crates/aprender-review-experiment/src/outcome_join.rs new file mode 100644 index 0000000000..0e82292d6c --- /dev/null +++ b/crates/aprender-review-experiment/src/outcome_join.rs @@ -0,0 +1,202 @@ +//! PRM-C10 `trace-outcome-join-v1`: PR outcomes and HRQ rulings as gold +//! (gate G-MAT; contract `trace-outcome-join-v1`, FALSIFY-TOJ-001..005). +//! +//! - **Outcome**: from a PR's merge / close / revert / escape events. A revert +//! at most [`WINDOW_DAYS`] after the merge is `reverted_le14d`; a later +//! revert or any escape is `regression_escape`. +//! - **Maturity**: an outcome is gold only once its window has closed, +//! [`WINDOW_DAYS`] UTC dates after the merge (or close). Until then it reads +//! `pending`, and pending is never a negative. +//! - **HRQ rulings** are gold when made; the latest ruling on a round wins. +//! - [`gold`] emits rows whose `label_source` is a [`crate::pool::GOLD_SOURCES`] +//! value; [`matured_between`] is the weekly matured count. + +use std::collections::BTreeMap; + +use serde::{Deserialize, Serialize}; + +use crate::ledger::Verdict; +use crate::split_guard::{date, day}; + +pub const SCHEME: &str = "trace-outcome-join-v1"; + +/// The outcome window, from the merge. +pub const WINDOW_DAYS: i64 = 14; + +/// The v3 lifecycle `outcome`. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum Outcome { + Pending, + Merged, + RevertedLe14d, + RegressionEscape, + ClosedUnmerged, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum EventKind { + Merged, + Closed, + Reverted, + /// A regression traced to the PR. + Escape, +} + +/// One PR event, as the join collects it. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Event { + pub repo: String, + pub pr: u64, + pub kind: EventKind, + /// RFC 3339 UTC. + pub at: String, +} + +/// A PR's outcome as of `now`. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Resolved { + /// `repo#PR`. + pub group_key: String, + pub outcome: Outcome, + /// `YYYY-MM-DD` the window closed; `None` while pending. + pub outcome_matured_at: Option, +} + +/// A human ruling on one quorum round. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Ruling { + pub repo: String, + pub pr: u64, + pub head: String, + pub verdict: Verdict, + pub ruled_by: String, + /// RFC 3339 UTC. + pub at: String, +} + +/// One gold label. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "label_source", rename_all = "snake_case", deny_unknown_fields)] +pub enum Label { + Outcome { + group_key: String, + outcome: Outcome, + outcome_matured_at: String, + }, + HrqRuling { + group_key: String, + head: String, + verdict: Verdict, + ruled_by: String, + at: String, + }, +} + +fn day_of(what: &str, at: &str) -> Result { + day(at).ok_or_else(|| format!("{what}: at {at:?} is not a date")) +} + +/// Each PR's outcome as of `now`, one row per `repo#PR`, sorted by key. +/// A revert or escape with no merge, or an unreadable date, is an error. +pub fn resolve(events: &[Event], now: &str) -> Result, String> { + let today = day_of("now", now)?; + let mut by_pr: BTreeMap> = BTreeMap::new(); + for e in events { + let key = format!("{}#{}", e.repo, e.pr); + let d = day_of(&key, &e.at)?; + by_pr.entry(key).or_default().push((e.kind, d)); + } + by_pr + .into_iter() + .map(|(key, evs)| { + let first = |k: EventKind| evs.iter().filter(|e| e.0 == k).map(|e| e.1).min(); + let (outcome, from) = if let Some(m) = first(EventKind::Merged) { + let late = |k| evs.iter().any(|&(ek, d)| ek == k && d >= m); + let outcome = match first(EventKind::Reverted).filter(|&r| r >= m) { + Some(r) if r - m <= WINDOW_DAYS => Outcome::RevertedLe14d, + Some(_) => Outcome::RegressionEscape, + None if late(EventKind::Escape) => Outcome::RegressionEscape, + None => Outcome::Merged, + }; + (outcome, m) + } else if let Some(c) = first(EventKind::Closed) { + if evs + .iter() + .any(|e| matches!(e.0, EventKind::Reverted | EventKind::Escape)) + { + return Err(format!("{key}: a revert or escape of an unmerged PR")); + } + (Outcome::ClosedUnmerged, c) + } else { + return Err(format!("{key}: a revert or escape with no merge")); + }; + let matured = today - from >= WINDOW_DAYS; + Ok(Resolved { + group_key: key, + outcome: if matured { outcome } else { Outcome::Pending }, + outcome_matured_at: matured.then(|| date(from + WINDOW_DAYS)), + }) + }) + .collect() +} + +/// Gold labels: matured outcomes, then the latest HRQ ruling per round. An +/// outcome that is neither pending nor matured, or a ruling with no ruler or +/// date, is an error. +pub fn gold(resolved: &[Resolved], rulings: &[Ruling]) -> Result, String> { + let mut out = Vec::new(); + for r in resolved { + match (&r.outcome, &r.outcome_matured_at) { + (Outcome::Pending, _) => {} + (&outcome, Some(m)) => out.push(Label::Outcome { + group_key: r.group_key.clone(), + outcome, + outcome_matured_at: m.clone(), + }), + (o, None) => return Err(format!("{}: {o:?} before its window closed", r.group_key)), + } + } + let mut latest: BTreeMap<(String, &str), &Ruling> = BTreeMap::new(); + for r in rulings { + let key = format!("{}#{}", r.repo, r.pr); + if r.ruled_by.trim().is_empty() { + return Err(format!("{key}@{}: ruling has no ruler", r.head)); + } + day_of(&key, &r.at)?; + let e = latest.entry((key, &r.head)).or_insert(r); + if r.at > e.at { + *e = r; + } + } + out.extend( + latest + .into_iter() + .map(|((group_key, head), r)| Label::HrqRuling { + group_key, + head: head.to_owned(), + verdict: r.verdict, + ruled_by: r.ruled_by.clone(), + at: r.at.clone(), + }), + ); + Ok(out) +} + +/// Outcomes whose window closed in `[from, to)` (`YYYY-MM-DD`). +#[must_use] +pub fn matured_between(resolved: &[Resolved], from: &str, to: &str) -> usize { + resolved + .iter() + .filter_map(|r| r.outcome_matured_at.as_deref()) + .filter(|m| *m >= from && *m < to) + .count() +} + +#[cfg(test)] +#[path = "outcome_join_tests.rs"] +mod tests; diff --git a/crates/aprender-review-experiment/src/outcome_join_tests.rs b/crates/aprender-review-experiment/src/outcome_join_tests.rs new file mode 100644 index 0000000000..fd448a74bc --- /dev/null +++ b/crates/aprender-review-experiment/src/outcome_join_tests.rs @@ -0,0 +1,236 @@ +use super::*; + +const NOW: &str = "2026-09-25T12:00:00Z"; + +fn ev(pr: u64, kind: EventKind, at: &str) -> Event { + Event { + repo: "paiml/aprender".into(), + pr, + kind, + at: at.into(), + } +} + +fn ruling(head: &str, verdict: Verdict, ruled_by: &str, at: &str) -> Ruling { + Ruling { + repo: "paiml/aprender".into(), + pr: 7, + head: head.into(), + verdict, + ruled_by: ruled_by.into(), + at: at.into(), + } +} + +fn one(events: &[Event], now: &str) -> Resolved { + let r = resolve(events, now).expect("resolves"); + assert_eq!(r.len(), 1, "{r:?}"); + r.into_iter().next().expect("one") +} + +/// FALSIFY-TOJ-001: a 3-day-old outcome is not gold. Until the 14 d window +/// has closed the PR reads `pending`, and only day 14 matures it. +#[test] +fn falsify_toj_001_immature_outcome_is_refused_as_gold() { + let three = one(&[ev(1, EventKind::Merged, "2026-09-22T09:00:00Z")], NOW); + assert_eq!(three.outcome, Outcome::Pending); + assert_eq!(three.outcome_matured_at, None); + assert!(gold(&[three], &[]).expect("gold").is_empty()); + // A 3-day-old revert is known but still inside the window. + let rev = one( + &[ + ev(1, EventKind::Merged, "2026-09-21T00:00:00Z"), + ev(1, EventKind::Reverted, "2026-09-22T00:00:00Z"), + ], + NOW, + ); + assert_eq!(rev.outcome, Outcome::Pending); + // Days count by UTC date: 09-12 to 09-25 is 13, whatever the hour. + let d13 = one( + &[ev(1, EventKind::Merged, "2026-09-12T00:00:00Z")], + "2026-09-25T23:59:59Z", + ); + assert_eq!(d13.outcome, Outcome::Pending, "13 days is immature"); + let d14 = one(&[ev(1, EventKind::Merged, "2026-09-11T08:00:00Z")], NOW); + assert_eq!(d14.outcome, Outcome::Merged); + assert_eq!(d14.outcome_matured_at.as_deref(), Some("2026-09-25")); + let g = gold(&[d14], &[]).expect("gold"); + assert_eq!( + g, + vec![Label::Outcome { + group_key: "paiml/aprender#1".into(), + outcome: Outcome::Merged, + outcome_matured_at: "2026-09-25".into(), + }] + ); +} + +/// FALSIFY-TOJ-002: the terminal outcome. A revert on day ≤ 14 after the +/// merge is `reverted_le14d`; a later revert or any escape is +/// `regression_escape`; a PR closed without a merge is `closed_unmerged`. +#[test] +fn falsify_toj_002_outcome_follows_the_revert_window() { + let old = "2026-08-01T00:00:00Z"; + let at = |kind, at| one(&[ev(2, EventKind::Merged, old), ev(2, kind, at)], NOW).outcome; + assert_eq!( + at(EventKind::Reverted, "2026-08-15T23:00:00Z"), + Outcome::RevertedLe14d + ); + assert_eq!( + at(EventKind::Reverted, "2026-08-16T00:00:00Z"), + Outcome::RegressionEscape + ); + assert_eq!( + at(EventKind::Escape, "2026-08-03T00:00:00Z"), + Outcome::RegressionEscape + ); + assert_eq!( + one(&[ev(2, EventKind::Closed, old)], NOW).outcome, + Outcome::ClosedUnmerged + ); + assert_eq!( + one(&[ev(2, EventKind::Merged, old)], NOW).outcome, + Outcome::Merged + ); + // A merged PR's close event does not make it unmerged. + assert_eq!( + one( + &[ev(2, EventKind::Closed, old), ev(2, EventKind::Merged, old)], + NOW + ) + .outcome, + Outcome::Merged + ); + // The window runs from the merge, not from an earlier event. + let r = one( + &[ + ev(2, EventKind::Reverted, "2026-08-20T00:00:00Z"), + ev(2, EventKind::Merged, "2026-08-10T00:00:00Z"), + ], + NOW, + ); + assert_eq!(r.outcome, Outcome::RevertedLe14d); + assert_eq!(r.outcome_matured_at.as_deref(), Some("2026-08-24")); + // A revert or escape dated before the merge is not of this merge. + for kind in [EventKind::Reverted, EventKind::Escape] { + let early = one( + &[ + ev(2, kind, "2026-08-05T00:00:00Z"), + ev(2, EventKind::Merged, "2026-08-10T00:00:00Z"), + ], + NOW, + ); + assert_eq!(early.outcome, Outcome::Merged, "{kind:?}"); + } + assert_eq!( + serde_json::to_string(&Outcome::RevertedLe14d).expect("ser"), + "\"reverted_le14d\"" + ); +} + +/// FALSIFY-TOJ-003: pending is never a negative, and a join that cannot be +/// read is an error, never a guess. +#[test] +fn falsify_toj_003_pending_is_never_a_label_and_bad_joins_refuse() { + let open = one(&[ev(4, EventKind::Merged, "2026-09-24T00:00:00Z")], NOW); + assert_eq!(open.outcome, Outcome::Pending); + assert!(gold(&[open], &[]).expect("gold").is_empty()); + for bad in [ + vec![ev(3, EventKind::Reverted, "2026-08-01T00:00:00Z")], + vec![ev(3, EventKind::Escape, "2026-08-01T00:00:00Z")], + vec![ + ev(3, EventKind::Closed, "2026-08-01T00:00:00Z"), + ev(3, EventKind::Reverted, "2026-08-02T00:00:00Z"), + ], + vec![ev(3, EventKind::Merged, "yesterday")], + ] { + assert!(resolve(&bad, NOW).is_err(), "{bad:?}"); + } + assert!(resolve(&[], "now").is_err()); + // A hand-built immature row is refused, not silently dropped. + let forged = Resolved { + group_key: "paiml/aprender#9".into(), + outcome: Outcome::Merged, + outcome_matured_at: None, + }; + assert!(gold(&[forged], &[]).is_err()); +} + +/// FALSIFY-TOJ-004: an HRQ ruling is gold as soon as it is made; the latest +/// ruling on a round wins; a ruling with no ruler is refused. +#[test] +fn falsify_toj_004_hrq_rulings_are_gold() { + let g = gold( + &[], + &[ + ruling("abc", Verdict::Pass, "noah", "2026-09-25T10:00:00Z"), + ruling("abc", Verdict::Fail, "noah", "2026-09-25T11:00:00Z"), + ruling("def", Verdict::Pass, "noah", "2026-09-25T09:00:00Z"), + ], + ) + .expect("gold"); + assert_eq!( + g, + vec![ + Label::HrqRuling { + group_key: "paiml/aprender#7".into(), + head: "abc".into(), + verdict: Verdict::Fail, + ruled_by: "noah".into(), + at: "2026-09-25T11:00:00Z".into(), + }, + Label::HrqRuling { + group_key: "paiml/aprender#7".into(), + head: "def".into(), + verdict: Verdict::Pass, + ruled_by: "noah".into(), + at: "2026-09-25T09:00:00Z".into(), + }, + ] + ); + let v = serde_json::to_value(&g[0]).expect("ser"); + assert_eq!(v["label_source"], "hrq_ruling"); + assert!(crate::pool::GOLD_SOURCES.contains(&v["label_source"].as_str().expect("str"))); + for bad in [ + ruling("abc", Verdict::Pass, " ", "2026-09-25T10:00:00Z"), + ruling("abc", Verdict::Pass, "noah", "today"), + ] { + assert!(gold(&[], &[bad]).is_err()); + } +} + +/// FALSIFY-TOJ-005: the weekly count is the outcomes whose window closed in +/// `[from, to)`, and nothing still pending. +#[test] +fn falsify_toj_005_weekly_matured_count() { + let rs = resolve( + &[ + ev(10, EventKind::Merged, "2026-09-04T00:00:00Z"), // matures 09-18: in + ev(11, EventKind::Closed, "2026-09-10T00:00:00Z"), // matures 09-24: in + ev(12, EventKind::Merged, "2026-09-11T00:00:00Z"), // matures 09-25: out (to) + ev(13, EventKind::Merged, "2026-09-03T00:00:00Z"), // matures 09-17: out (from-1) + ev(14, EventKind::Merged, "2026-09-20T00:00:00Z"), // pending + ], + NOW, + ) + .expect("resolves"); + assert_eq!(rs.len(), 5); + assert_eq!(matured_between(&rs, "2026-09-18", "2026-09-25"), 2); + assert_eq!(matured_between(&rs, "2026-09-17", "2026-09-26"), 4); + let v = serde_json::to_value(gold(&rs, &[]).expect("gold").first().expect("one")).expect("ser"); + assert_eq!(v["label_source"], "outcome"); +} + +#[test] +fn date_inverts_day() { + for s in [ + "1970-01-01", + "2000-02-29", + "2024-12-31", + "2026-09-25", + "2100-03-01", + ] { + let d = crate::split_guard::day(s).expect("day"); + assert_eq!(crate::split_guard::date(d), s); + } +} diff --git a/crates/aprender-review-experiment/src/pilot.rs b/crates/aprender-review-experiment/src/pilot.rs new file mode 100644 index 0000000000..058c94ea7b --- /dev/null +++ b/crates/aprender-review-experiment/src/pilot.rs @@ -0,0 +1,107 @@ +//! REX-05 pilot projection: the variance and runtime of the dev-item pilot on +//! one admitted cell, and the §2.2 sample-size rule applied to it. +//! +//! The rule (analysis plan §Sample-size rule): project the test-split Wilson +//! half-width on recall at the pilot's point estimate; above +//! [`MAX_HALF_WIDTH`] the corpus grows, and the hypotheses do not change. +//! A pilot with no defect item in its denominator has no point estimate, so +//! the rule falls back to the §2.2 design basis p = 0.5 and says so. + +use crate::score::Ratio; +use crate::stats::{percentile, sample_sd, wilson_half_width}; +use serde::Serialize; + +/// §2.2: grow the corpus when the projected half-width exceeds this. +pub const MAX_HALF_WIDTH: f64 = 0.12; + +/// Where the projected recall came from. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)] +pub enum Basis { + /// The pilot's own recall point estimate. + Pilot, + /// No defect item in the pilot denominator: the §2.2 design p = 0.5. + DesignWorstCase, +} + +/// Spread of one timing sample, in milliseconds. +#[derive(Debug, Clone, Copy, PartialEq, Serialize)] +pub struct Spread { + pub n: usize, + pub mean: f64, + /// `None` below two values. + pub sd: Option, + pub p50: f64, + pub p95: f64, +} + +impl Spread { + /// `None` for an empty sample (nothing ran, so there is no spread). + #[must_use] + pub fn of(x: &[f64]) -> Option { + if x.is_empty() || x.iter().any(|v| !v.is_finite()) { + return None; + } + Some(Self { + n: x.len(), + mean: x.iter().sum::() / x.len() as f64, + sd: sample_sd(x), + p50: percentile(x, 0.50), + p95: percentile(x, 0.95), + }) + } +} + +/// What the pilot projects for the full run on this cell. +#[derive(Debug, Clone, PartialEq, Serialize)] +pub struct Projection { + pub recall: Ratio, + pub basis: Basis, + pub p: f64, + pub n_defect_test: u64, + /// Wilson 95 % half-width at `p` on `n_defect_test` (`None` when n = 0). + pub half_width: Option, + /// §2.2: true when `half_width` exceeds [`MAX_HALF_WIDTH`] or cannot be + /// computed. An unknown width is never a reason to keep the corpus. + pub grow_corpus: bool, + pub warm: Option, + pub cold: Option, + /// Test items plus the 10 % determinism re-run. + pub projected_requests: u64, + /// `projected_requests` × warm mean; `None` when nothing warm ran. + pub projected_runtime_s: Option, +} + +/// Apply the §2.2 rule and the runtime projection to one pilot. +#[must_use] +pub fn project( + recall: Ratio, + warm_wall_ms: &[f64], + cold_wall_ms: &[f64], + n_defect_test: u64, + n_test: u64, +) -> Projection { + let (basis, p) = match recall.value() { + Some(p) => (Basis::Pilot, p), + None => (Basis::DesignWorstCase, 0.5), + }; + let k = (p * n_defect_test as f64).round() as u64; + let half_width = wilson_half_width(k, n_defect_test); + let warm = Spread::of(warm_wall_ms); + let projected_requests = n_test + n_test.div_ceil(10); + Projection { + recall, + basis, + p, + n_defect_test, + half_width, + grow_corpus: half_width.is_none_or(|h| h > MAX_HALF_WIDTH), + warm, + cold: Spread::of(cold_wall_ms), + projected_requests, + projected_runtime_s: warm.map(|w| w.mean * projected_requests as f64 / 1e3), + } +} + +#[cfg(test)] +#[path = "pilot_tests.rs"] +mod tests; diff --git a/crates/aprender-review-experiment/src/pilot_tests.rs b/crates/aprender-review-experiment/src/pilot_tests.rs new file mode 100644 index 0000000000..5ebf9a50e4 --- /dev/null +++ b/crates/aprender-review-experiment/src/pilot_tests.rs @@ -0,0 +1,49 @@ +use super::*; + +#[test] +fn the_design_point_sits_just_under_the_rule() { + // §2.2 [C] quotes about 0.117 for 70 defect items at p = 0.5; that is the + // Wald width. The frozen Wilson width the rule reads is about 0.114. + let pr = project(Ratio::new(0, 0), &[], &[], 70, 140); + assert_eq!(pr.basis, Basis::DesignWorstCase); + let h = pr.half_width.expect("n > 0"); + assert!((h - 0.114).abs() < 0.001, "{h}"); + assert!(!pr.grow_corpus); +} + +#[test] +fn falsify_rxp_001_a_wide_projection_grows_the_corpus() { + // 40 defect items at the pilot's p = 0.5: half-width about 0.15 > 0.12. + let pr = project(Ratio::new(3, 6), &[1.0], &[], 40, 80); + assert_eq!(pr.basis, Basis::Pilot); + assert!(pr.half_width.expect("n > 0") > MAX_HALF_WIDTH); + assert!(pr.grow_corpus, "a > 0.12 projection must grow the corpus"); + // An unknown width (no test defects) is never a reason to keep the corpus. + assert!(project(Ratio::new(3, 6), &[1.0], &[], 0, 80).grow_corpus); +} + +#[test] +fn falsify_rxp_002_the_projection_uses_the_pilot_estimate() { + // A high pilot recall narrows the interval below the rule at n = 50. + let pr = project(Ratio::new(9, 10), &[1.0], &[], 50, 100); + assert_eq!(pr.p, 0.9); + assert!(!pr.grow_corpus, "{:?}", pr.half_width); + // The same n at the design p = 0.5 would grow it: the estimate matters. + assert!(project(Ratio::new(5, 10), &[1.0], &[], 50, 100).grow_corpus); +} + +#[test] +fn falsify_rxp_003_runtime_counts_the_rerun_and_nothing_warm_is_unknown() { + let pr = project(Ratio::new(1, 2), &[1000.0, 3000.0], &[9000.0], 70, 145); + assert_eq!(pr.projected_requests, 145 + 15); + assert_eq!(pr.projected_runtime_s, Some(2.0 * 160.0)); + let w = pr.warm.expect("warm ran"); + assert_eq!((w.n, w.p50, w.p95), (2, 1000.0, 3000.0)); + assert_eq!(pr.cold.map(|c| c.n), Some(1)); + let none = project(Ratio::new(1, 2), &[], &[], 70, 145); + assert_eq!( + none.projected_runtime_s, None, + "no warm sample is not zero seconds" + ); + assert!(Spread::of(&[1.0, f64::NAN]).is_none()); +} diff --git a/crates/aprender-review-experiment/src/pool.rs b/crates/aprender-review-experiment/src/pool.rs new file mode 100644 index 0000000000..1576b211a3 --- /dev/null +++ b/crates/aprender-review-experiment/src/pool.rs @@ -0,0 +1,171 @@ +//! Training-pool admission gate (ruling agent-trace item 6; PRA-001 T6, +//! contract `trace-admission-secret-v1`). +//! +//! A captured row enters the fine-tune/distill pool only when it is secret-scan +//! clean under BOTH scanners of [`crate::secret`] AND carries no sealed-test +//! hash (`review-corpus-contamination-v1`). +//! A secret hit QUARANTINES the row as-is: it is never redacted in place, since a +//! redacted row is a row the scanner already failed on once. A sealed-test hit +//! REFUSES the row. +//! +//! G-PROV (PRM-001 v3 R-18, contract `trace-admission-prov-v1`): a row is +//! train-eligible only when it is a local row (qwen, the 27B teacher: provider +//! `local`, fully terms-tagged per [`crate::terms::tags`]) or a gold label +//! (`label_source` one of [`GOLD_SOURCES`]). Hosted outputs are never +//! eligible, whatever else the row claims, and an untagged row cannot be shown +//! not to be hosted, so it is ineligible too. + +use serde_json::Value; + +use crate::contamination::Index; +use crate::secret::{scan, Hit}; +use crate::terms::{tags, Gap, Provider}; + +/// The `label_source` values that make a row a gold label (v3 G-PROV): a +/// matured outcome, an HRQ ruling, a sealed-corpus label. +pub const GOLD_SOURCES: [&str; 3] = ["outcome", "hrq_ruling", "sealed_corpus"]; + +/// Why a row fails G-PROV. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum Prov { + /// A hosted provider's output: never a training target (R-18). + Hosted(Provider), + /// Terms tags missing or inconsistent: provenance unproven. + Untagged(Vec), + /// A `label_source` that is not a gold source. + NotGold(String), +} + +/// Where one input row went. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum Verdict { + /// Clean: the row is in the pool. + Admitted, + /// A secret was found; the row is held byte-for-byte outside the pool. + Quarantined { secrets: Vec }, + /// The row carries a sealed test item; it never enters the pool. + Refused { items: Vec }, + /// Not a local row nor a gold label (G-PROV); it never enters the pool. + Ineligible(Prov), + /// The sealed index is empty, so contamination is unproven either way. + Unchecked, +} + +/// The outcome for a JSONL batch: one verdict per non-empty line, and the pool. +#[derive(Debug, Default)] +pub struct Admission { + pub verdicts: Vec<(String, Verdict)>, +} + +impl Admission { + /// The rows admitted to the training pool, in input order. + #[must_use] + pub fn pool(&self) -> Vec<&str> { + self.with(|v| matches!(v, Verdict::Admitted)) + } + + /// The quarantined rows, byte-identical to their input. + #[must_use] + pub fn quarantine(&self) -> Vec<&str> { + self.with(|v| matches!(v, Verdict::Quarantined { .. })) + } + + fn with(&self, keep: impl Fn(&Verdict) -> bool) -> Vec<&str> { + self.verdicts + .iter() + .filter(|(_, v)| keep(v)) + .map(|(r, _)| r.as_str()) + .collect() + } +} + +/// Admit a JSONL batch of captured rows against the sealed-test index. +/// +/// Sealed-test hits are checked first: such a row is refused outright, even if +/// it also carries a secret, so no test item is ever stored beside training data. +#[must_use] +pub fn admit(rows: &str, sealed: &Index) -> Admission { + let verdicts = rows + .lines() + .filter(|l| !l.trim().is_empty()) + .map(|l| (l.to_string(), verdict(l, sealed))) + .collect(); + Admission { verdicts } +} + +fn verdict(row: &str, sealed: &Index) -> Verdict { + if sealed.is_empty() { + return Verdict::Unchecked; + } + let mut items: Vec = sealed + .scan(row, "row") + .into_iter() + .map(|h| h.item) + .collect(); + items.dedup(); + if !items.is_empty() { + return Verdict::Refused { items }; + } + // Before the secret scan, so a hosted row never reaches the quarantine, + // whose rows may be released after review. + if let Err(p) = provenance(row) { + return Verdict::Ineligible(p); + } + let found = scan(row); + if found.is_empty() { + Verdict::Admitted + } else { + Verdict::Quarantined { secrets: found } + } +} + +/// G-PROV for one row. +/// +/// # Errors +/// The row is hosted, untagged, or labelled by a non-gold source. +pub fn provenance(row: &str) -> Result<(), Prov> { + let Ok(v) = serde_json::from_str::(row) else { + return Err(Prov::Untagged(vec![Gap::NotJson])); + }; + // A hosted provider refuses first: a gold label never carries a hosted output. + if let Some(p) = v + .get("provider") + .and_then(Value::as_str) + .and_then(Provider::parse) + { + if p != Provider::Local { + return Err(Prov::Hosted(p)); + } + } + if let Some(s) = v.get("label_source") { + return match s.as_str() { + Some(g) if GOLD_SOURCES.contains(&g) => Ok(()), + _ => Err(Prov::NotGold(s.to_string())), + }; + } + // Hosted providers returned above, so tagged here means local. + tags(&v).map(|_| ()).map_err(Prov::Untagged) +} + +/// Test fixture: `v` with the terms tags of a local (apr-serve) row added. +#[cfg(test)] +pub(crate) fn local_tagged(mut v: Value) -> String { + let terms = r#"{"url":"https://www.anthropic.com/legal/commercial-terms","effective_date":"2025-06-17","fetched_at":"2026-09-25T17:00:00Z"}"#; + if let Value::Object(m) = &mut v { + m.insert("provider".into(), "local".into()); + m.insert( + "model_id".into(), + "3f1a9c0b5e2d4f6a8b7c9d0e1f2a3b4c5d6e7f8091a2b3c4d5e6f708192a3b4c".into(), + ); + m.insert("access_channel".into(), "apr-serve".into()); + m.insert( + "terms_ref".into(), + serde_json::from_str(terms).expect("fixture terms_ref is JSON"), + ); + } + v.to_string() +} + +#[cfg(test)] +#[path = "pool_tests.rs"] +mod tests; diff --git a/crates/aprender-review-experiment/src/pool_tests.rs b/crates/aprender-review-experiment/src/pool_tests.rs new file mode 100644 index 0000000000..22f1fa883f --- /dev/null +++ b/crates/aprender-review-experiment/src/pool_tests.rs @@ -0,0 +1,178 @@ +// `json!` expands to an `unwrap` of an infallible `to_value`. +#![allow(clippy::disallowed_methods)] + +use super::*; +use crate::corpus::{hunk_fingerprints, sha256_hex, Sealed}; +use crate::secret::{Hit, Scanner}; +use crate::terms::Provider; + +const TEST_DIFF: &str = "--- a/crates/x/src/a.rs\n+++ b/crates/x/src/a.rs\n@@ -10,5 +10,5 @@\n fn f(x: u32) -> bool {\n- x > 1\n+ x >= 1\n }\n \n"; + +fn index() -> Index { + Index::new(&[Sealed { + id: "R007".into(), + diff_sha256: sha256_hex(TEST_DIFF.as_bytes()), + hunks: hunk_fingerprints(TEST_DIFF), + }]) +} + +/// AWS's documented example access key id, assembled at run time so the source +/// file itself carries no key literal for repo secret scanners to flag. +fn planted_aws_key() -> String { + ["AKIA", "IOSFODNN7", "EXAMPLE"].concat() +} + +/// A local (qwen) lane row: G-PROV eligible, so only the check under test decides. +fn row(output: &str) -> String { + local_tagged(serde_json::json!({"lane": "qwen", "output": output})) +} + +/// FALSIFY-TAS-004: a planted AWS key literal in a lane output is quarantined, +/// verbatim, and is not in the pool; a clean neighbour is admitted. +#[test] +fn falsify_tas_004_planted_aws_key_is_quarantined() { + let dirty = row(&format!("export AWS_ACCESS_KEY_ID={}", planted_aws_key())); + let clean = row("LGTM: the bound check is inclusive now."); + let a = admit(&format!("{clean}\n{dirty}\n"), &index()); + assert_eq!(a.verdicts[0], (clean.clone(), Verdict::Admitted)); + assert_eq!( + a.verdicts[1], + ( + dirty.clone(), + Verdict::Quarantined { + secrets: vec![Hit { + scanner: Scanner::Builtin, + rule: "aws-access-key-id" + }] + } + ), + "the row is held byte-for-byte, never redacted in place" + ); + assert_eq!(a.pool(), vec![clean.as_str()]); + assert_eq!(a.quarantine(), vec![dirty.as_str()]); +} + +/// FALSIFY-TAS-005: a row carrying a sealed test-split diff is refused and not +/// in the pool, as is a row carrying only the item's diff sha. +#[test] +fn falsify_tas_005_planted_test_split_diff_is_refused() { + let leak = serde_json::json!({"lane": "haiku-4-5", "input": TEST_DIFF}).to_string(); + let sha = row(&format!("reviewed {}", sha256_hex(TEST_DIFF.as_bytes()))); + let clean = row("no findings"); + let a = admit(&format!("{leak}\n{sha}\n{clean}\n"), &index()); + for (i, r) in [&leak, &sha].into_iter().enumerate() { + assert_eq!( + a.verdicts[i], + ( + r.clone(), + Verdict::Refused { + items: vec!["R007".into()] + } + ) + ); + } + assert_eq!(a.pool(), vec![clean.as_str()]); +} + +/// An empty sealed index proves nothing about contamination: admission refuses +/// to run rather than admit everything vacuously. +#[test] +fn empty_sealed_index_refuses_everything() { + let a = admit(&format!("{}\n", row("no findings")), &Index::new(&[])); + assert!(a.pool().is_empty()); +} + +/// A correctly tagged hosted lane row: clean of secrets and sealed items, so +/// only G-PROV can keep it out. +fn hosted(provider: &str, model: &str, channel: &str) -> String { + let mut v: serde_json::Value = serde_json::from_str(&row("LGTM")).expect("row is JSON"); + v["provider"] = provider.into(); + v["model_id"] = model.into(); + v["access_channel"] = channel.into(); + v.to_string() +} + +/// FALSIFY-TAP-001 (G-PROV, R-18): a planted hosted row in a pool is +/// ineligible and not in the pool, for each hosted provider; its local +/// neighbour is admitted. +#[test] +fn falsify_tap_001_planted_hosted_row_is_ineligible() { + let claude = hosted("anthropic", "claude-sonnet-5", "claude-code-cli"); + let gemini = hosted("google", "gemini-3.1-pro-high", "antigravity"); + let local = row("LGTM"); + assert!(crate::terms::tags(&serde_json::from_str(&claude).expect("json")).is_ok()); + let a = admit(&format!("{claude}\n{gemini}\n{local}\n"), &index()); + assert_eq!( + a.verdicts[0].1, + Verdict::Ineligible(Prov::Hosted(Provider::Anthropic)) + ); + assert_eq!( + a.verdicts[1].1, + Verdict::Ineligible(Prov::Hosted(Provider::Google)) + ); + assert_eq!( + a.pool(), + vec![local.as_str()], + "hosted rows in the pool = 0" + ); +} + +/// FALSIFY-TAP-002: a hosted row cannot launder itself: claiming a gold +/// `label_source`, or carrying a secret, it is still ineligible (never +/// quarantined, whose rows may be released). +#[test] +fn falsify_tap_002_hosted_row_is_ineligible_whatever_it_claims() { + let mut v: serde_json::Value = + serde_json::from_str(&hosted("anthropic", "claude-sonnet-5", "anthropic-api")) + .expect("json"); + v["label_source"] = "outcome".into(); + let gold_claim = v.to_string(); + v["output"] = format!("AWS_ACCESS_KEY_ID={}", planted_aws_key()).into(); + let with_secret = v.to_string(); + let a = admit(&format!("{gold_claim}\n{with_secret}\n"), &index()); + for (_, verdict) in &a.verdicts { + assert_eq!( + *verdict, + Verdict::Ineligible(Prov::Hosted(Provider::Anthropic)) + ); + } + assert!(a.pool().is_empty() && a.quarantine().is_empty()); +} + +/// FALSIFY-TAP-003: an untagged or mis-tagged row is ineligible: provenance +/// that cannot be shown local is not local. +#[test] +fn falsify_tap_003_untagged_row_is_ineligible() { + let bare = serde_json::json!({"lane": "sonnet-5", "output": "LGTM"}).to_string(); + let mut v: serde_json::Value = serde_json::from_str(&row("LGTM")).expect("json"); + v["provider"] = "Anthropic".into(); + let misspelt = v.to_string(); + let a = admit(&format!("{bare}\n{misspelt}\nnot json\n"), &index()); + for (r, verdict) in &a.verdicts { + assert!( + matches!(verdict, Verdict::Ineligible(Prov::Untagged(g)) if !g.is_empty()), + "{r}: {verdict:?}" + ); + } + assert!(a.pool().is_empty()); +} + +/// FALSIFY-TAP-004: a gold label (each source) is eligible without provider +/// tags; any other `label_source` is not. +#[test] +fn falsify_tap_004_gold_labels_are_eligible_and_only_gold() { + let gold: Vec = GOLD_SOURCES + .iter() + .map(|s| serde_json::json!({"pr": 4354, "label_source": s, "defect": true}).to_string()) + .collect(); + let silver = serde_json::json!({"pr": 4354, "label_source": "quorum_majority"}).to_string(); + let a = admit(&format!("{}\n{silver}\n", gold.join("\n")), &index()); + assert_eq!( + a.pool(), + gold.iter().map(String::as_str).collect::>() + ); + assert_eq!( + a.verdicts[3].1, + Verdict::Ineligible(Prov::NotGold("\"quorum_majority\"".into())) + ); +} diff --git a/crates/aprender-review-experiment/src/prereg.rs b/crates/aprender-review-experiment/src/prereg.rs new file mode 100644 index 0000000000..3638851f14 --- /dev/null +++ b/crates/aprender-review-experiment/src/prereg.rs @@ -0,0 +1,248 @@ +//! PRM-00b pre-registration lock (contract `rex-prereg-v1`, scheme `rex-prereg-v2`, rule R-1). +//! +//! `rex-prereg-v2` locks PRM-001 v3 (`docs/specifications/PRM-001-prometheus.md`). The v1 +//! lock (`docs/audits/rex-001/prereg.lock`, prereg_sha `ef51087d…`) is SUPERSEDED, never +//! deleted or edited (operator, 2026-09-25); its spec and plan stay byte-frozen in-tree and +//! `falsify_rex_prereg_004` keeps them verifying. +//! +//! The prereg sha is a sha256 over the sha256 of four frozen components, in a +//! fixed order: +//! +//! 1. `spec_s2_s5` — spec bytes from the line `## §2 ` up to (not including) +//! the line `## §6 ` (design, hypotheses, decision rule, improvement loop); +//! 2. `stats_rs` — the analysis code; +//! 3. `analysis_plan` — `docs/audits/prm-001/analysis-plan.md` (plan v2); +//! 4. `prompt_v1` — the fixed review prompt. +//! +//! The sealed test-item manifest is NOT a prereg component: it is filled once by +//! REX-02 (placeholder → sealed) and carried separately as the corpus version. +//! Everything outside §2–§5 (header, §7 tickets, §9 schema) may be edited +//! without a new spec version. + +use sha2::{Digest, Sha256}; + +/// The spec as committed in-tree. +pub const SPEC: &str = include_str!("../../../docs/specifications/PRM-001-prometheus.md"); +/// The analysis code. +pub const STATS_RS: &str = include_str!("stats.rs"); +/// The analysis plan. +pub const ANALYSIS_PLAN: &str = include_str!("../../../docs/audits/prm-001/analysis-plan.md"); +/// Prompt v1 (the G1 prompt header; the diff is appended after it). +pub const PROMPT_V1: &str = include_str!("../../../docs/audits/review-corpus/prompts/v1.txt"); +/// The committed lock. +pub const LOCK: &str = include_str!("../../../docs/audits/prm-001/prereg.lock"); + +/// The superseded v1 lock and the v1 spec and plan it froze (never edited). +pub const LOCK_V1: &str = include_str!("../../../docs/audits/rex-001/prereg.lock"); +/// The v1 spec (`rex-prereg-v1`), byte-frozen in §2–§5. +pub const SPEC_V1: &str = + include_str!("../../../docs/specifications/review-experiment-protocol.md"); +/// The v1 analysis plan, byte-frozen. +pub const ANALYSIS_PLAN_V1: &str = include_str!("../../../docs/audits/rex-001/analysis-plan.md"); + +/// Contract id stamped into the digest so a lock can never match another scheme. +pub const SCHEME: &str = "rex-prereg-v2"; + +/// Lowercase hex sha256. +#[must_use] +pub fn sha256_hex(bytes: &[u8]) -> String { + Sha256::digest(bytes) + .iter() + .map(|b| format!("{b:02x}")) + .collect() +} + +/// Spec §2–§5 exactly. `None` if either heading is missing or out of order — +/// a spec that lost its §2 or §6 heading cannot be pre-registered. +#[must_use] +pub fn spec_sections_2_to_5(spec: &str) -> Option<&str> { + let start = line_start(spec, "## §2 ")?; + let end = line_start(&spec[start..], "## §6 ")? + start; + Some(&spec[start..end]) +} + +fn line_start(text: &str, prefix: &str) -> Option { + if text.starts_with(prefix) { + return Some(0); + } + text.find(&format!("\n{prefix}")).map(|i| i + 1) +} + +/// Component digests in lock order. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Components { + pub spec_s2_s5: String, + pub stats_rs: String, + pub analysis_plan: String, + pub prompt_v1: String, +} + +impl Components { + /// Digest the given sources. `None` when the spec has no §2..§6 span. + #[must_use] + pub fn of(spec: &str, stats: &str, plan: &str, prompt: &str) -> Option { + Some(Self { + spec_s2_s5: sha256_hex(spec_sections_2_to_5(spec)?.as_bytes()), + stats_rs: sha256_hex(stats.as_bytes()), + analysis_plan: sha256_hex(plan.as_bytes()), + prompt_v1: sha256_hex(prompt.as_bytes()), + }) + } + + /// The components as compiled into this crate. + #[must_use] + pub fn in_tree() -> Option { + Self::of(SPEC, STATS_RS, ANALYSIS_PLAN, PROMPT_V1) + } + + /// The prereg sha. + #[must_use] + pub fn prereg_sha(&self) -> String { + sha256_hex(self.canonical().as_bytes()) + } + + fn canonical(&self) -> String { + format!( + "{SCHEME}\nspec_s2_s5 {}\nstats_rs {}\nanalysis_plan {}\nprompt_v1 {}\n", + self.spec_s2_s5, self.stats_rs, self.analysis_plan, self.prompt_v1 + ) + } + + /// Render a lock file (used once, by REX-00, to write `prereg.lock`). + #[must_use] + pub fn render_lock(&self) -> String { + format!( + "# PRM-001 pre-registration lock (rex-prereg-v2) over spec PRM-001 v3. Written by PRM-00b; v1 had 0 data rows.\n\ + # A mismatch against the tree is a new spec version (R-1), never an edit here.\n\ + spec_s2_s5={}\nstats_rs={}\nanalysis_plan={}\nprompt_v1={}\nprereg_sha={}\n", + self.spec_s2_s5, + self.stats_rs, + self.analysis_plan, + self.prompt_v1, + self.prereg_sha() + ) + } +} + +/// Look up `key=value` in a lock body. +#[must_use] +pub fn lock_value<'a>(lock: &'a str, key: &str) -> Option<&'a str> { + lock.lines() + .filter(|l| !l.starts_with('#')) + .find_map(|l| l.strip_prefix(key)?.strip_prefix('=')) + .map(str::trim) +} + +/// Every mismatch between a lock and the given components (empty = verified). +#[must_use] +pub fn verify(lock: &str, c: &Components) -> Vec { + let want = [ + ("spec_s2_s5", c.spec_s2_s5.clone()), + ("stats_rs", c.stats_rs.clone()), + ("analysis_plan", c.analysis_plan.clone()), + ("prompt_v1", c.prompt_v1.clone()), + ("prereg_sha", c.prereg_sha()), + ]; + want.iter() + .filter_map(|(k, v)| match lock_value(lock, k) { + Some(got) if got == v => None, + Some(got) => Some(format!("{k}: lock {got} != tree {v}")), + None => Some(format!("{k}: missing from lock")), + }) + .collect() +} + +/// The prereg sha recorded in the committed lock (what every receipt carries). +#[must_use] +pub fn locked_prereg_sha() -> Option<&'static str> { + lock_value(LOCK, "prereg_sha") +} + +#[cfg(test)] +mod tests { + use super::*; + + /// FALSIFY-REX-PREREG-001: the committed lock matches the tree. Any edit to + /// §2–§5, the analysis code, the plan or prompt v1 turns this RED. + #[test] + fn falsify_rex_prereg_001_lock_matches_tree() { + let c = Components::in_tree().expect("spec has §2 and §6 headings"); + let bad = verify(LOCK, &c); + assert!( + bad.is_empty(), + "REX-001 prereg drift (R-1: this is a new spec version, not a lock edit):\n{}\n\ + expected lock:\n{}", + bad.join("\n"), + c.render_lock() + ); + } + + /// FALSIFY-REX-PREREG-002: a planted edit to §3 changes the prereg sha. + #[test] + fn falsify_rex_prereg_002_planted_section3_edit_is_caught() { + let planted = SPEC.replacen("McNemar exact, one-sided", "t-test", 1); + assert_ne!(planted, SPEC, "the §3 anchor text must exist"); + let c = Components::of(&planted, STATS_RS, ANALYSIS_PLAN, PROMPT_V1).expect("span"); + assert!(!verify(LOCK, &c).is_empty()); + } + + /// FALSIFY-REX-PREREG-003: edits OUTSIDE §2–§5 do not move the prereg sha. + #[test] + fn falsify_rex_prereg_003_status_line_edit_is_not_a_new_version() { + let edited = SPEC.replacen( + "**Runner:** the aprender traffic cop", + "**Runner:** the fleet", + 1, + ); + assert_ne!(edited, SPEC); + let c = Components::of(&edited, STATS_RS, ANALYSIS_PLAN, PROMPT_V1).expect("span"); + assert!(verify(LOCK, &c).is_empty()); + } + + /// FALSIFY-REX-PREREG-004: the superseded v1 lock is untouched and the v1 + /// spec §2–§5, plan and prompt it froze still verify against it. (v1 + /// `stats_rs` is verified against git `5f0ae10ac`; the code moved on.) + #[test] + fn falsify_rex_prereg_004_v1_lock_is_frozen_and_verifies() { + assert_eq!( + lock_value(LOCK_V1, "prereg_sha"), + Some("ef51087dc79bab0ad160e8a14f5b13e2ea43986b30c05dafc63c84c2dc21cdc0") + ); + assert!(LOCK_V1.starts_with("# REX-001 pre-registration lock (rex-prereg-v1).")); + let span = spec_sections_2_to_5(SPEC_V1).expect("v1 §2..§6"); + assert_eq!( + lock_value(LOCK_V1, "spec_s2_s5"), + Some(sha256_hex(span.as_bytes()).as_str()) + ); + assert_eq!( + lock_value(LOCK_V1, "analysis_plan"), + Some(sha256_hex(ANALYSIS_PLAN_V1.as_bytes()).as_str()) + ); + assert_eq!( + lock_value(LOCK_V1, "prompt_v1"), + Some(sha256_hex(PROMPT_V1.as_bytes()).as_str()) + ); + assert_ne!( + lock_value(LOCK_V1, "prereg_sha"), + locked_prereg_sha(), + "v2 is a new lock" + ); + } + #[test] + fn span_requires_both_headings() { + assert!(spec_sections_2_to_5("## §2 a\nb\n## §6 c\n").is_some()); + assert_eq!(spec_sections_2_to_5("## §2 a\nb\n"), None); + assert_eq!(spec_sections_2_to_5("x\n## §6 c\n## §2 a\n"), None); + assert_eq!( + spec_sections_2_to_5("pre\n## §2 a\nb\n## §6 c\n"), + Some("## §2 a\nb\n") + ); + } + + #[test] + fn prompt_v1_is_the_g1_prompt() { + assert!(PROMPT_V1.starts_with("You are a code reviewer. Review this diff.")); + assert!(PROMPT_V1.ends_with("Be concrete.\n\n")); + assert_eq!(locked_prereg_sha().map(str::len), Some(64)); + } +} diff --git a/crates/aprender-review-experiment/src/ratchet.rs b/crates/aprender-review-experiment/src/ratchet.rs new file mode 100644 index 0000000000..3d2847c301 --- /dev/null +++ b/crates/aprender-review-experiment/src/ratchet.rs @@ -0,0 +1,382 @@ +//! REX-10 perf ratchet (§5.1): review p95 on the primary cell may not rise. +//! +//! Every tag reruns the perf arm on the primary cell over one fixed item set +//! (full dev split + 30 fixed test items, timing only). `record` turns that +//! tag's warm, non-rerun receipts into one `review-lane-perf-ratchet-v1` entry +//! holding each item's wall time. `check` reads the entries of one cell in +//! record order: +//! - fewer than [`ARM_AFTER`] tags: **arming**, nothing to compare yet; +//! - each later tag is paired by item with the best (lowest-p95) earlier tag, +//! and a paired bootstrap gives a CI for `p95(new) / p95(best)`, each p95 +//! the Harrell–Davis estimate. A CI whose lower bound is above 1 is a +//! regression and the andon is RED. +//! +//! The paired CI, not "the two p95 CIs overlap", is the rule: the same items +//! are timed every tag, so pairing removes item-to-item spread. The unpaired +//! overlap rule could not see a +10 % p95 regression at the pilot's n +//! (FALSIFY-RXR-002 measures this). Nor could a paired nearest-rank p95: it is +//! one order statistic, so one item's run-to-run noise decides it; the +//! Harrell–Davis estimate of the same quantile weights the whole tail. +//! The only level used is the prereg's [`ALPHA`]; no noise threshold is +//! invented (R-7). + +use std::collections::BTreeMap; + +use serde::{Deserialize, Serialize}; + +use crate::stats::{percentile, percentile_sorted, SplitMix64, ALPHA, BOOTSTRAP_RESAMPLES}; + +pub const SCHEME: &str = "review-lane-perf-ratchet-v1"; +/// The ratchet holds after this many recorded tags (§5.1). +pub const ARM_AFTER: usize = 3; +const P95: f64 = 0.95; +/// The harness seed (`harness::DECODING.seed`), so every CI reproduces. +const SEED: u64 = 4354; + +/// One recorded tag on one cell. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Entry { + pub schema: String, + pub tag: String, + pub cell: String, + pub apr_sha256: String, + /// Warm wall time per item, ms (the median when an item ran twice). + pub items: BTreeMap, + pub p95_ms: f64, + /// Unpaired percentile-bootstrap CI of this tag's p95, for display. + pub p95_ci_ms: [f64; 2], + /// The same p95 from llama.cpp on the same cell and items, if it ran. + pub llama_cpp_p95_ms: Option, + pub receipts_sha256: String, +} + +/// A warm, non-rerun timing already filtered to one cell and one tag. +#[derive(Debug, Clone)] +pub struct Sample<'a> { + pub item_id: &'a str, + pub apr_sha256: &'a str, + pub wall_ms: f64, +} + +fn median(v: &mut [f64]) -> f64 { + v.sort_by(f64::total_cmp); + let n = v.len(); + if n % 2 == 1 { + v[n / 2] + } else { + (v[n / 2 - 1] + v[n / 2]) / 2.0 + } +} + +/// Per-item wall time: the median of each item's finite, positive timings. +fn per_item(samples: &[Sample<'_>]) -> Result, String> { + let mut by: BTreeMap> = BTreeMap::new(); + for s in samples { + if !(s.wall_ms.is_finite() && s.wall_ms > 0.0) { + return Err(format!( + "{}: wall_ms {} is not a timing", + s.item_id, s.wall_ms + )); + } + by.entry(s.item_id.to_string()).or_default().push(s.wall_ms); + } + Ok(by + .into_iter() + .map(|(k, mut v)| (k, median(&mut v))) + .collect()) +} + +fn ci_of(mut stat: Vec) -> [f64; 2] { + stat.sort_by(f64::total_cmp); + [ + percentile_sorted(&stat, ALPHA / 2.0), + percentile_sorted(&stat, 1.0 - ALPHA / 2.0), + ] +} + +/// Unpaired percentile-bootstrap CI of the p95 of `x`. +fn p95_ci(x: &[f64]) -> [f64; 2] { + let mut rng = SplitMix64::new(SEED); + let mut buf = vec![0.0; x.len()]; + let stat = (0..BOOTSTRAP_RESAMPLES) + .map(|_| { + for b in &mut buf { + *b = x[rng.below(x.len())]; + } + percentile(&buf, P95) + }) + .collect(); + ci_of(stat) +} + +/// Build one tag's entry. A tag with no timings did not run and is not a +/// recorded tag (R-6); mixed apr binaries under one tag are refused. +pub fn record( + tag: &str, + cell: &str, + samples: &[Sample<'_>], + llama_cpp: &[Sample<'_>], + receipts_sha256: &str, +) -> Result { + let shas: std::collections::BTreeSet<&str> = samples.iter().map(|s| s.apr_sha256).collect(); + let apr_sha256 = match shas.len() { + 0 => { + return Err(format!( + "{tag} on {cell}: no warm timings; the tag did not run" + )) + } + 1 => shas.into_iter().next().unwrap_or_default().to_string(), + n => { + return Err(format!( + "{tag} on {cell}: {n} different apr binaries under one tag" + )) + } + }; + let items = per_item(samples)?; + let v: Vec = items.values().copied().collect(); + let llama_cpp_p95_ms = if llama_cpp.is_empty() { + None + } else { + let l: Vec = per_item(llama_cpp)?.into_values().collect(); + Some(percentile(&l, P95)) + }; + Ok(Entry { + schema: SCHEME.into(), + tag: tag.into(), + cell: cell.into(), + apr_sha256, + p95_ms: percentile(&v, P95), + p95_ci_ms: p95_ci(&v), + items, + llama_cpp_p95_ms, + receipts_sha256: receipts_sha256.into(), + }) +} + +/// ln Γ(x), Lanczos (g = 7, n = 9); x > 0. +fn ln_gamma(x: f64) -> f64 { + const C: [f64; 9] = [ + 0.999_999_999_999_809_9, + 676.520_368_121_885_1, + -1_259.139_216_722_402_8, + 771.323_428_777_653_1, + -176.615_029_162_140_6, + 12.507_343_278_686_905, + -0.138_571_095_265_720_12, + 9.984_369_578_019_572e-6, + 1.505_632_735_149_311_6e-7, + ]; + let x = x - 1.0; + let t = x + 7.5; + let s = C[1..] + .iter() + .enumerate() + .fold(C[0], |s, (i, c)| s + c / (x + i as f64 + 1.0)); + 0.5 * (2.0 * std::f64::consts::PI).ln() + (x + 0.5) * t.ln() - t + s.ln() +} + +fn not_tiny(v: f64) -> f64 { + if v.abs() < 1e-300 { + 1e-300 + } else { + v + } +} + +/// Continued fraction of the incomplete beta (modified Lentz). +fn beta_cf(a: f64, b: f64, x: f64) -> f64 { + let (qab, qap, qam) = (a + b, a + 1.0, a - 1.0); + let mut c = 1.0; + let mut d = not_tiny(1.0 - qab * x / qap).recip(); + let mut h = d; + for m in 1..=300 { + let m = f64::from(m); + let m2 = 2.0 * m; + for aa in [ + m * (b - m) * x / ((qam + m2) * (a + m2)), + -(a + m) * (qab + m) * x / ((a + m2) * (qap + m2)), + ] { + d = not_tiny(1.0 + aa * d).recip(); + c = not_tiny(1.0 + aa / c); + h *= d * c; + } + if (d * c - 1.0).abs() < 1e-15 { + break; + } + } + h +} + +/// Regularized incomplete beta I_x(a, b). +fn inc_beta(a: f64, b: f64, x: f64) -> f64 { + if x <= 0.0 { + return 0.0; + } + if x >= 1.0 { + return 1.0; + } + let front = + (ln_gamma(a + b) - ln_gamma(a) - ln_gamma(b) + a * x.ln() + b * (1.0 - x).ln()).exp(); + if x < (a + 1.0) / (a + b + 2.0) { + front * beta_cf(a, b, x) / a + } else { + 1.0 - front * beta_cf(b, a, 1.0 - x) / b + } +} + +/// Harrell–Davis weights for quantile `q` of `n` sorted values: a weighted +/// sum of every order statistic, so one noisy tail item cannot swing it the +/// way it swings the nearest-rank p95. +#[must_use] +pub fn harrell_davis_weights(n: usize, q: f64) -> Vec { + let (a, b) = ((n as f64 + 1.0) * q, (n as f64 + 1.0) * (1.0 - q)); + (1..=n) + .map(|i| inc_beta(a, b, i as f64 / n as f64) - inc_beta(a, b, (i - 1) as f64 / n as f64)) + .collect() +} + +fn hd(buf: &mut [f64], w: &[f64]) -> f64 { + buf.sort_by(f64::total_cmp); + buf.iter().zip(w).map(|(x, w)| x * w).sum() +} + +/// Harrell–Davis p95 ratio `new / reference` over paired items, and its +/// paired-bootstrap CI (items resampled together, so both sides see the +/// same items). +#[must_use] +pub fn paired_p95_ratio_ci(pairs: &[(f64, f64)]) -> Option<(f64, [f64; 2])> { + if pairs.is_empty() { + return None; + } + let w = harrell_davis_weights(pairs.len(), P95); + let (mut a, mut b): (Vec, Vec) = pairs.iter().copied().unzip(); + let point = hd(&mut a, &w) / hd(&mut b, &w); + let mut rng = SplitMix64::new(SEED); + let stat = (0..BOOTSTRAP_RESAMPLES) + .map(|_| { + for i in 0..pairs.len() { + let (n, r) = pairs[rng.below(pairs.len())]; + a[i] = n; + b[i] = r; + } + hd(&mut a, &w) / hd(&mut b, &w) + }) + .collect(); + Some((point, ci_of(stat))) +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)] +#[serde(rename_all = "lowercase")] +pub enum Andon { + Arming, + Green, + Red, +} + +/// One armed comparison: `tag` against the best earlier `reference`. +#[derive(Debug, Clone, PartialEq, Serialize)] +pub struct Comparison { + pub tag: String, + pub reference: String, + pub paired_items: usize, + pub p95_ratio: f64, + pub p95_ratio_ci: [f64; 2], + pub regression: bool, +} + +#[derive(Debug, Clone, PartialEq, Serialize)] +pub struct Verdict { + pub andon: Andon, + pub cell: Option, + pub tags: usize, + pub comparisons: Vec, + pub violations: Vec, + pub notes: Vec, +} + +/// Read the ratchet file of one cell and decide the andon. Violations (a +/// foreign schema or cell, a duplicate tag, no common items) are RED too: +/// the ratchet fails closed. +#[must_use] +pub fn check(entries: &[Entry]) -> Verdict { + let mut v = Verdict { + andon: Andon::Arming, + cell: entries.first().map(|e| e.cell.clone()), + tags: entries.len(), + comparisons: Vec::new(), + violations: Vec::new(), + notes: Vec::new(), + }; + let mut seen = std::collections::BTreeSet::new(); + for e in entries { + if e.schema != SCHEME { + v.violations + .push(format!("{}: schema {} is not {SCHEME}", e.tag, e.schema)); + } + if Some(&e.cell) != v.cell.as_ref() { + v.violations + .push(format!("{}: cell {} is not the file's cell", e.tag, e.cell)); + } + if !seen.insert(e.tag.as_str()) { + v.violations.push(format!("{}: tag recorded twice", e.tag)); + } + } + for (i, e) in entries.iter().enumerate().skip(ARM_AFTER) { + let best = entries[..i] + .iter() + .min_by(|a, b| a.p95_ms.total_cmp(&b.p95_ms)) + .unwrap_or(&entries[0]); + let pairs: Vec<(f64, f64)> = e + .items + .iter() + .filter_map(|(k, n)| best.items.get(k).map(|r| (*n, *r))) + .collect(); + if pairs.len() != e.items.len() || pairs.len() != best.items.len() { + v.notes.push(format!( + "{} vs {}: {} common items of {} / {}", + e.tag, + best.tag, + pairs.len(), + e.items.len(), + best.items.len() + )); + } + let Some((ratio, ci)) = paired_p95_ratio_ci(&pairs) else { + v.violations + .push(format!("{} vs {}: no common items", e.tag, best.tag)); + continue; + }; + v.comparisons.push(Comparison { + tag: e.tag.clone(), + reference: best.tag.clone(), + paired_items: pairs.len(), + p95_ratio: ratio, + p95_ratio_ci: ci, + regression: ci[0] > 1.0, + }); + } + for w in entries.windows(2) { + let gap = |e: &Entry| e.llama_cpp_p95_ms.map(|l| e.p95_ms / l); + if let (Some(g0), Some(g1)) = (gap(&w[0]), gap(&w[1])) { + if g1 > g0 { + v.notes.push(format!( + "{}: gap to llama.cpp grew {g0:.3}× → {g1:.3}× (§9.1: file it)", + w[1].tag + )); + } + } + } + v.andon = if !v.violations.is_empty() || v.comparisons.iter().any(|c| c.regression) { + Andon::Red + } else if entries.len() > ARM_AFTER { + Andon::Green + } else { + Andon::Arming + }; + v +} + +#[cfg(test)] +#[path = "ratchet_tests.rs"] +mod tests; diff --git a/crates/aprender-review-experiment/src/ratchet_tests.rs b/crates/aprender-review-experiment/src/ratchet_tests.rs new file mode 100644 index 0000000000..4f9b661819 --- /dev/null +++ b/crates/aprender-review-experiment/src/ratchet_tests.rs @@ -0,0 +1,200 @@ +use super::*; + +/// The pilot's item count: the dev split (~100) plus 30 fixed test items. +const N: usize = 130; + +/// Approximate N(0,1) from twelve uniforms, on the frozen generator. +fn gauss(rng: &mut SplitMix64) -> f64 { + (0..12) + .map(|_| (rng.next_u64() >> 11) as f64 / (1u64 << 53) as f64) + .sum::() + - 6.0 +} + +/// Item difficulty: log-normal wall time around 8 s with σ = 0.6, which is +/// wide (diff sizes vary by an order of magnitude); fixed across tags. +fn base() -> Vec { + let mut rng = SplitMix64::new(1); + (0..N) + .map(|_| (8000f64.ln() + 0.6 * gauss(&mut rng)).exp()) + .collect() +} + +/// One tag's run: every item × `scale`, with 3 % run-to-run noise. +fn tag(name: &str, seed: u64, scale: f64) -> Entry { + let mut rng = SplitMix64::new(seed); + let ids: Vec = (0..N).map(|i| format!("item-{i:03}")).collect(); + let ms: Vec = base() + .iter() + .map(|b| b * scale * (1.0 + 0.03 * gauss(&mut rng))) + .collect(); + let s: Vec> = ids + .iter() + .zip(&ms) + .map(|(id, &wall_ms)| Sample { + item_id: id, + apr_sha256: "aa", + wall_ms, + }) + .collect(); + record(name, "C4", &s, &[], "rr").expect("records") +} + +fn baseline() -> Vec { + vec![ + tag("v0.70.0", 10, 1.0), + tag("v0.70.1", 11, 1.0), + tag("v0.70.2", 12, 1.0), + ] +} + +#[test] +fn falsify_rxr_001_a_planted_ten_percent_p95_regression_is_red() { + let mut e = baseline(); + assert_eq!( + check(&e).andon, + Andon::Arming, + "3 tags only arm the ratchet" + ); + e.push(tag("v0.70.3", 13, 1.0)); + let same = check(&e); + assert_eq!(same.andon, Andon::Green, "{same:?}"); + e[3] = tag("v0.70.3", 13, 1.10); + let v = check(&e); + assert_eq!(v.andon, Andon::Red, "{v:?}"); + let c = &v.comparisons[0]; + assert!(c.regression && c.p95_ratio_ci[0] > 1.0 && c.paired_items == N); +} + +#[test] +fn falsify_rxr_001_a_gain_is_kept_the_reference_is_the_best_tag() { + let mut e = baseline(); + e[2] = tag("v0.70.2", 12, 0.80); + e.push(tag("v0.70.3", 13, 1.0)); + let v = check(&e); + assert_eq!(v.andon, Andon::Red, "giving back a 20 % gain is a rise"); + assert_eq!(v.comparisons[0].reference, "v0.70.2"); +} + +/// The finding behind the paired rule: at the pilot's n, the two unpaired +/// p95 CIs of a +10 % tag still overlap, so the §4 overlap rule calls it a +/// tie. The ratchet could not meet its own acceptance on that rule. +#[test] +fn falsify_rxr_002_the_unpaired_overlap_rule_misses_ten_percent() { + let a = tag("v0.70.2", 12, 1.0); + let b = tag("v0.70.3", 13, 1.10); + assert!(b.p95_ms > a.p95_ms); + assert!( + b.p95_ci_ms[0] <= a.p95_ci_ms[1], + "unpaired CIs {:?} vs {:?} unexpectedly separate", + a.p95_ci_ms, + b.p95_ci_ms + ); +} + +#[test] +fn falsify_rxr_003_the_ratchet_fails_closed() { + let mut dup = baseline(); + dup.push(tag("v0.70.1", 13, 1.0)); + assert_eq!(check(&dup).andon, Andon::Red, "duplicate tag"); + + let mut foreign = baseline(); + foreign[1].cell = "C1".into(); + assert_eq!(check(&foreign).andon, Andon::Red, "foreign cell"); + + let mut disjoint = baseline(); + let mut t = tag("v0.70.3", 13, 1.0); + t.items = t + .items + .into_iter() + .map(|(k, v)| (format!("x{k}"), v)) + .collect(); + disjoint.push(t); + let v = check(&disjoint); + assert_eq!(v.andon, Andon::Red, "no common items"); + assert!(v.violations.iter().any(|m| m.contains("no common items"))); + + let ok = Sample { + item_id: "a", + apr_sha256: "aa", + wall_ms: 1.0, + }; + assert!( + record("t", "C4", &[], &[], "r").is_err(), + "a tag that did not run" + ); + let other = Sample { + apr_sha256: "bb", + ..ok.clone() + }; + assert!( + record("t", "C4", &[ok.clone(), other], &[], "r").is_err(), + "two binaries" + ); + let nan = Sample { + wall_ms: f64::NAN, + ..ok + }; + assert!( + record("t", "C4", &[nan], &[], "r").is_err(), + "a non-finite timing" + ); +} + +#[test] +fn falsify_rxr_003_a_growing_llama_cpp_gap_is_noted() { + let mut e = baseline(); + e[0].llama_cpp_p95_ms = Some(e[0].p95_ms); + e[1].llama_cpp_p95_ms = Some(e[1].p95_ms / 1.2); + let v = check(&e); + assert!( + v.notes.iter().any(|n| n.contains("gap to llama.cpp grew")), + "{v:?}" + ); +} + +#[test] +fn an_item_timed_twice_counts_its_median() { + let s = |ms| Sample { + item_id: "a", + apr_sha256: "aa", + wall_ms: ms, + }; + let e = record("t", "C4", &[s(1.0), s(9.0), s(2.0)], &[], "r").expect("records"); + assert_eq!(e.items["a"], 2.0); +} + +#[test] +fn harrell_davis_weights_are_a_distribution_centred_on_the_tail() { + let w = harrell_davis_weights(N, 0.95); + assert!((w.iter().sum::() - 1.0).abs() < 1e-9); + assert!(w.iter().all(|x| *x >= 0.0)); + let x: Vec = (1..=N).map(|i| i as f64).collect(); + let q: f64 = x.iter().zip(&w).map(|(x, w)| x * w).sum(); + assert!( + (q - 0.95 * (N as f64 + 1.0)).abs() < 1.0, + "HD p95 of 1..N = {q}" + ); +} + +/// Rule 6: one seed is an anecdote. Over 10 independent runs of the planted +/// tag the ratchet catches +10 % every time and seldom alarms on an unchanged +/// tag (40/40 caught and 0/40 alarms when first measured over 40 runs, at the +/// modelled 3 % run-to-run noise per item). +#[test] +fn falsify_rxr_001_power_and_false_alarms_over_ten_runs() { + let e = baseline(); + let red = |scale: f64| { + (100..110) + .filter(|&s| { + let mut t = e.clone(); + t.push(tag("v0.70.3", s, scale)); + check(&t).andon == Andon::Red + }) + .count() + }; + let (caught, false_alarms) = (red(1.10), red(1.0)); + eprintln!("RXR power: +10% caught {caught}/10, unchanged red {false_alarms}/10"); + assert_eq!(caught, 10); + assert!(false_alarms <= 1, "{false_alarms}/10 false alarms"); +} diff --git a/crates/aprender-review-experiment/src/receipt.rs b/crates/aprender-review-experiment/src/receipt.rs new file mode 100644 index 0000000000..d139c7c651 --- /dev/null +++ b/crates/aprender-review-experiment/src/receipt.rs @@ -0,0 +1,335 @@ +//! REX-03 receipt (contract `review-experiment-receipt-v1`, spec §2.3/§2.4). +//! +//! One JSONL row per (item, cell, arm). A row is admissible only if it +//! deserializes with every required field, carries no empty or `unknown` +//! identity field, names the locked prereg sha and the corpus version under +//! analysis, and (when it ran) carries token counts, timings, the raw-output +//! path+sha and a load snapshot. An inadmissible row is never a pass: the +//! scorer counts its item as not run. + +use crate::corpus::{Class, Loc, Split, Stratum}; +use serde::{Deserialize, Serialize}; + +/// Schema name every row carries. +pub const SCHEME: &str = "review-experiment-receipt-v1"; + +/// The lane under test. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)] +pub enum Arm { + /// Qwen3.5-4B-Q4_K_M through `apr serve` (primary). + #[serde(rename = "apr-4b")] + Apr4b, + /// Qwen3.5-9B-Q4_K_M through `apr serve` (control, H3). + #[serde(rename = "apr-9b")] + Apr9b, + /// Haiku 4.5 via `claude -p` (baseline). + #[serde(rename = "haiku")] + Haiku, + /// The agy lane (baseline). + #[serde(rename = "agy")] + Agy, +} + +impl Arm { + /// Hosted arms have no apr binary or local weights; they record `hosted`. + #[must_use] + pub fn is_hosted(self) -> bool { + matches!(self, Self::Haiku | Self::Agy) + } +} + +/// Why a (item, cell, arm) produced no verdict. Never correct, never a FAIL. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)] +pub enum NotRun { + /// No forjar-declared executor for the cell (R-4). + NoDeclaredExecutor, + /// The diff does not fit the context; it is never truncated. + ContextOverflow, + /// The backend refused to load or serve the model on this cell. + Refused, + /// The serve answered with an error that is not a context overflow. + ServeError, + /// Assigned by the scorer to an item whose receipt is inadmissible. + Inadmissible, +} + +/// Parsed verdict (§2.3). +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "state", content = "reason")] +pub enum Verdict { + Pass, + Fail, + /// The output had no well-formed `VERDICT:` line. Counted as incorrect. + Unparsed, + NotRun(NotRun), +} + +impl Verdict { + /// The row executed a request (parsed or not). + #[must_use] + pub fn executed(self) -> bool { + !matches!(self, Self::NotRun(_)) + } + + /// Correct per the analysis plan: a parsed FAIL on a defect, a parsed + /// PASS on a good item. `Unparsed` and `NotRun` are never correct (R-6). + #[must_use] + pub fn correct(self, defect: bool) -> bool { + match self { + Self::Fail => defect, + Self::Pass => !defect, + Self::Unparsed | Self::NotRun(_) => false, + } + } +} + +/// Decoding settings (fixed across every run, §2.1). +#[derive(Debug, Clone, Copy, PartialEq, Serialize, Deserialize)] +pub struct Decoding { + pub temperature: f64, + pub seed: u64, + pub max_tokens: u32, +} + +/// Token counts from the serve's `usage`. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +pub struct Tokens { + pub prompt: u64, + pub completion: u64, +} + +/// Server-measured phase timings (the serve's `timings`, PP-LLAMA-001 §3). +#[derive(Debug, Clone, Copy, PartialEq, Serialize, Deserialize)] +pub struct ServerTimings { + pub prompt_ms: f64, + pub predicted_ms: f64, + pub prompt_per_second: Option, + pub predicted_per_second: Option, +} + +/// Client wall-clock (request sent → verdict parsed) plus the server's own +/// timings. `server: null` means the serving path did not measure them; the +/// TTFT/tok/s columns are then `NotRun{NoSensor}`, never zero. +#[derive(Debug, Clone, Copy, PartialEq, Serialize, Deserialize)] +pub struct Timings { + pub wall_ms: f64, + pub server: Option, +} + +/// Host load at request time (§2.3). `co_running` and `peak_anon_bytes` +/// come from the executor unit (infra#1088); `null` where it does not report. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct Load { + pub loadavg1: f64, + pub cpus: u32, + pub co_running: Option>, + pub peak_anon_bytes: Option, +} + +/// Where the raw model output was written, and its sha. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct Output { + pub path: String, + pub sha256: String, +} + +/// One `review-experiment-receipt-v1` row. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Receipt { + pub schema: String, + pub item_id: String, + pub item_sha256: String, + pub class: Class, + pub stratum: Stratum, + pub split: Split, + pub cell: String, + pub host: String, + pub backend: String, + pub arm: Arm, + pub apr_tag: String, + pub apr_sha256: String, + pub model_id: String, + pub weights_sha256: String, + pub prompt_sha256: String, + /// sha of the exact request body sent (prompt composed with the diff). + pub request_sha256: String, + pub decoding: Decoding, + pub corpus_version: String, + pub prereg_sha: String, + /// RFC 3339 UTC. + pub started_at: String, + /// First request after the unit started (§2.4); reported separately. + pub cold: bool, + /// Member of the 10 % determinism re-run. + pub rerun: bool, + pub verdict: Verdict, + pub tokens: Option, + pub timings: Option, + pub output: Option, + pub load: Option, +} + +/// What the analysis requires every row to name. +#[derive(Debug, Clone, Copy)] +pub struct Expect<'a> { + pub prereg_sha: &'a str, + pub corpus_version: &'a str, +} + +pub(crate) fn is_hex64(s: &str) -> bool { + s.len() == 64 + && s.bytes() + .all(|b| b.is_ascii_hexdigit() && !b.is_ascii_uppercase()) +} + +/// Parse and check one JSONL row. `Err` lists every reason it is inadmissible. +/// +/// # Errors +/// The row does not deserialize, or fails any admissibility rule. +pub fn admissible(line: &str, expect: Expect<'_>) -> Result> { + let r: Receipt = serde_json::from_str(line).map_err(|e| vec![format!("schema: {e}")])?; + let bad = problems(&r, expect); + if bad.is_empty() { + Ok(r) + } else { + Err(bad) + } +} + +fn problems(r: &Receipt, expect: Expect<'_>) -> Vec { + let mut bad = Vec::new(); + if r.schema != SCHEME { + bad.push(format!("schema {:?} != {SCHEME}", r.schema)); + } + let named = [ + ("item_id", &r.item_id), + ("cell", &r.cell), + ("host", &r.host), + ("backend", &r.backend), + ("apr_tag", &r.apr_tag), + ("model_id", &r.model_id), + ("started_at", &r.started_at), + ]; + for (k, v) in named { + if v.trim().is_empty() || v.eq_ignore_ascii_case("unknown") { + bad.push(format!("{k} is empty or unknown")); + } + } + let mut shas = vec![ + ("item_sha256", &r.item_sha256), + ("prompt_sha256", &r.prompt_sha256), + ("request_sha256", &r.request_sha256), + ("prereg_sha", &r.prereg_sha), + ]; + if r.arm.is_hosted() { + for (k, v) in [ + ("apr_sha256", &r.apr_sha256), + ("weights_sha256", &r.weights_sha256), + ] { + if v != "hosted" { + bad.push(format!("{k} must be `hosted` for a hosted arm")); + } + } + } else { + shas.push(("apr_sha256", &r.apr_sha256)); + shas.push(("weights_sha256", &r.weights_sha256)); + } + for (k, v) in shas { + if !is_hex64(v) { + bad.push(format!("{k} is not a sha256")); + } + } + if r.prereg_sha != expect.prereg_sha { + bad.push("prereg_sha is not the locked one (exploratory data)".into()); + } + if r.corpus_version != expect.corpus_version { + bad.push(format!( + "corpus_version {:?} is not {:?}", + r.corpus_version, expect.corpus_version + )); + } + if r.decoding.temperature != 0.0 { + bad.push("decoding is not greedy (temperature != 0)".into()); + } + if r.verdict.executed() { + for (k, missing) in [ + ("tokens", r.tokens.is_none()), + ("timings", r.timings.is_none()), + ("output", r.output.is_none()), + ("load", r.load.is_none()), + ] { + if missing { + bad.push(format!("{k} missing on an executed row")); + } + } + } + if let Some(o) = &r.output { + if o.path.is_empty() || !is_hex64(&o.sha256) { + bad.push("output path/sha malformed".into()); + } + } + bad +} + +/// The model's answer with any `…` reasoning removed: the +/// verdict is read after the last `` when one is present. +#[must_use] +pub fn answer(output: &str) -> &str { + output.rsplit_once("").map_or(output, |(_, a)| a) +} + +/// §2.3: the verdict of the first `VERDICT:` line of the answer. Markdown +/// decoration (`**`, `#`, `>`, `-`) is ignored. The first word must be +/// exactly `PASS` or `FAIL`, and a line naming both (an echo of the prompt, +/// "PASS or FAIL") is `Unparsed`. No `VERDICT:` line → `Unparsed`. +#[must_use] +pub fn parse_verdict(output: &str) -> Verdict { + for line in answer(output).lines() { + let t = line + .trim() + .trim_start_matches(['*', '#', '>', '-', ' ', '`']); + let Some(rest) = t.strip_prefix("VERDICT:") else { + continue; + }; + let rest = rest.trim_start().trim_start_matches(['*', '`', ' ']); + let word: String = rest.chars().take_while(char::is_ascii_alphabetic).collect(); + let tail = &rest[word.len()..]; + return match word.as_str() { + "PASS" if !tail.contains("FAIL") => Verdict::Pass, + "FAIL" if !tail.contains("PASS") => Verdict::Fail, + _ => Verdict::Unparsed, + }; + } + Verdict::Unparsed +} + +/// Accepted spellings of a defect file `[A]`: the full repo path, or its +/// last two components (`src/foo.rs`). A bare file name is not enough — +/// `mod.rs`/`lib.rs` would match anything. +fn spellings(file: &str) -> Vec<&str> { + let mut v = vec![file]; + let cut: Vec = file.match_indices('/').map(|(i, _)| i).collect(); + if cut.len() >= 2 { + v.push(&file[cut[cut.len() - 2] + 1..]); + } + v +} + +/// §2.3 localization: some finding after the verdict line names a defect +/// file. Only meaningful on a FAIL of a defect item. +#[must_use] +pub fn localized(output: &str, defect: &[Loc]) -> bool { + let a = answer(output); + let findings = a + .find("VERDICT:") + .and_then(|i| a[i..].find('\n').map(|j| &a[i + j..])) + .unwrap_or(""); + defect + .iter() + .any(|l| spellings(&l.file).iter().any(|s| findings.contains(s))) +} + +#[cfg(test)] +#[path = "receipt_tests.rs"] +pub(crate) mod tests; diff --git a/crates/aprender-review-experiment/src/receipt_tests.rs b/crates/aprender-review-experiment/src/receipt_tests.rs new file mode 100644 index 0000000000..f54fd8d75e --- /dev/null +++ b/crates/aprender-review-experiment/src/receipt_tests.rs @@ -0,0 +1,200 @@ +// `json!` expands to an `unwrap` of an infallible `to_value`. +#![allow(clippy::disallowed_methods)] + +use super::*; + +pub(crate) const PREREG: &str = "ef51087dc79bab0ad160e8a14f5b13e2ea43986b30c05dafc63c84c2dc21cdc0"; +pub(crate) const CORPUS: &str = "review-corpus-v1@787d2026256cc08b"; +pub(crate) const EXPECT: Expect<'static> = Expect { + prereg_sha: PREREG, + corpus_version: CORPUS, +}; + +fn h(c: char) -> String { + std::iter::repeat_n(c, 64).collect() +} + +/// A complete, admissible executed row. +pub(crate) fn row(item: &str, class: Class, verdict: Verdict) -> Receipt { + Receipt { + schema: SCHEME.into(), + item_id: item.into(), + item_sha256: h('1'), + class, + stratum: Stratum::S, + split: Split::Test, + cell: "C2".into(), + host: "intel".into(), + backend: "cpu".into(), + arm: Arm::Apr4b, + apr_tag: "v0.69.3".into(), + apr_sha256: h('2'), + model_id: "Qwen3.5-4B-Q4_K_M".into(), + weights_sha256: h('3'), + prompt_sha256: h('4'), + request_sha256: h('5'), + decoding: Decoding { + temperature: 0.0, + seed: 4354, + max_tokens: 512, + }, + corpus_version: CORPUS.into(), + prereg_sha: PREREG.into(), + started_at: "2026-09-25T10:00:00Z".into(), + cold: false, + rerun: false, + verdict, + tokens: Some(Tokens { + prompt: 900, + completion: 80, + }), + timings: Some(Timings { + wall_ms: 4200.0, + server: None, + }), + output: Some(Output { + path: format!("raw/C2/apr-4b/{item}.txt"), + sha256: h('6'), + }), + load: Some(Load { + loadavg1: 1.5, + cpus: 32, + co_running: None, + peak_anon_bytes: None, + }), + } +} + +fn line(r: &Receipt) -> String { + serde_json::to_string(r).expect("serialize") +} + +#[test] +fn complete_row_is_admissible_and_round_trips() { + let r = row("R-pr1", Class::R, Verdict::Fail); + assert_eq!(admissible(&line(&r), EXPECT), Ok(r)); +} + +#[test] +fn falsify_rxr_001_missing_model_sha_is_inadmissible() { + let mut v: serde_json::Value = + serde_json::to_value(row("R-pr1", Class::R, Verdict::Fail)).expect("v"); + v.as_object_mut().expect("obj").remove("weights_sha256"); + let e = + admissible(&v.to_string(), EXPECT).expect_err("missing weights sha must be inadmissible"); + assert!(e[0].contains("weights_sha256"), "{e:?}"); + + let mut r = row("R-pr1", Class::R, Verdict::Fail); + r.weights_sha256 = "unknown".into(); + assert!( + admissible(&line(&r), EXPECT).is_err(), + "`unknown` weights sha" + ); + r.weights_sha256 = h('3')[..12].into(); + assert!( + admissible(&line(&r), EXPECT).is_err(), + "a sha prefix is not a sha" + ); +} + +#[test] +fn identity_prereg_corpus_and_decoding_are_enforced() { + let ok = row("G-pr1", Class::G, Verdict::Pass); + let cases: Vec<(&str, Box)> = vec![ + ("host", Box::new(|r| r.host = "unknown".into())), + ("cell", Box::new(|r| r.cell = String::new())), + ("prereg", Box::new(|r| r.prereg_sha = h('9'))), + ( + "corpus", + Box::new(|r| r.corpus_version = "review-corpus-v0@x".into()), + ), + ("greedy", Box::new(|r| r.decoding.temperature = 0.7)), + ("tokens", Box::new(|r| r.tokens = None)), + ("output", Box::new(|r| r.output = None)), + ("load", Box::new(|r| r.load = None)), + ("hosted", Box::new(|r| r.arm = Arm::Haiku)), + ("schema", Box::new(|r| r.schema = "v0".into())), + ]; + for (name, mutate) in cases { + let mut r = ok.clone(); + mutate(&mut r); + assert!( + admissible(&line(&r), EXPECT).is_err(), + "{name} must be inadmissible" + ); + } + let mut v: serde_json::Value = serde_json::to_value(&ok).expect("v"); + v["extra"] = serde_json::json!(1); + assert!(admissible(&v.to_string(), EXPECT).is_err(), "unknown field"); +} + +#[test] +fn not_run_rows_need_identity_but_not_timings_and_hosted_arms_say_hosted() { + let mut r = row("P-1", Class::P, Verdict::NotRun(NotRun::NoDeclaredExecutor)); + (r.tokens, r.timings, r.output, r.load) = (None, None, None, None); + assert!(admissible(&line(&r), EXPECT).is_ok()); + let mut hosted = row("P-1", Class::P, Verdict::Fail); + hosted.arm = Arm::Haiku; + hosted.apr_sha256 = "hosted".into(); + hosted.weights_sha256 = "hosted".into(); + hosted.model_id = "claude-haiku-4-5-20251001".into(); + assert!(admissible(&line(&hosted), EXPECT).is_ok()); +} + +#[test] +fn verdict_parser_table() { + let cases = [ + ("VERDICT: FAIL\n- off by one", Verdict::Fail), + ("VERDICT: PASS", Verdict::Pass), + ("**VERDICT: FAIL** — index", Verdict::Fail), + ("## VERDICT: **PASS**\n", Verdict::Pass), + ("Sure.\nVERDICT: FAIL\nVERDICT: PASS", Verdict::Fail), + ( + "VERDICT: PASS maybe\nVERDICT: FAIL", + Verdict::Fail, + ), + ("VERDICT: PASS or FAIL", Verdict::Unparsed), + ("VERDICT: LGTM", Verdict::Unparsed), + ("VERDICT: pass", Verdict::Unparsed), + ("VERDICT: PASSED", Verdict::Unparsed), + ("The verdict is FAIL", Verdict::Unparsed), + ("", Verdict::Unparsed), + ]; + for (out, want) in cases { + assert_eq!(parse_verdict(out), want, "{out:?}"); + } +} + +#[test] +fn unparsed_and_not_run_are_never_correct() { + for v in [Verdict::Unparsed, Verdict::NotRun(NotRun::ContextOverflow)] { + assert!(!v.correct(true) && !v.correct(false), "{v:?}"); + } + assert!(Verdict::Fail.correct(true) && !Verdict::Fail.correct(false)); + assert!(Verdict::Pass.correct(false) && !Verdict::Pass.correct(true)); +} + +#[test] +fn localization_matches_path_or_last_two_components_after_the_verdict() { + let loc = [Loc { + file: "crates/aprender-serve/src/api/router.rs".into(), + line: 10, + }]; + assert!(localized( + "VERDICT: FAIL\n- api/router.rs: wrong status", + &loc + )); + assert!(localized( + "VERDICT: FAIL\n- crates/aprender-serve/src/api/router.rs:10", + &loc + )); + assert!( + !localized("VERDICT: FAIL\n- router.rs looks off", &loc), + "bare file name" + ); + assert!( + !localized("api/router.rs\nVERDICT: FAIL\n- fine", &loc), + "before the verdict" + ); + assert!(!localized("no verdict api/router.rs", &loc)); +} diff --git a/crates/aprender-review-experiment/src/replay.rs b/crates/aprender-review-experiment/src/replay.rs new file mode 100644 index 0000000000..12a5375d99 --- /dev/null +++ b/crates/aprender-review-experiment/src/replay.rs @@ -0,0 +1,516 @@ +//! PRM-S1: the `review-replay-v1` speed benchmark (PRM-001 v3 §2.4 and §9; +//! contract `review-replay-v1`). +//! +//! A frozen set of real quorum diffs (inputs only, never a lane's output) is +//! replayed through `apr serve` and through llama.cpp `d1d3c3396` on the same +//! GGUF and the same cell. This module builds the set, defines the receipt row +//! and folds rows into the §9 speed block. It does no I/O. +//! +//! - **Set.** Four strata by composed input tokens: `2k` (0, 2048], `8k` +//! (2048, 8192], `16k` (8192, 16384], `32k` (16384, 32768]. [`PER_STRATUM`] +//! items each. One item per `repo#PR`, one per diff sha. Every candidate that +//! hits the sealed test set (diff sha, hunk fingerprint or near-dup cluster) +//! is dropped. Selection inside a stratum is `sha256(seed ‖ diff_sha)` order, +//! so it is frozen by `(version, seed, candidates)`. A short stratum is an +//! error: the set is never padded. +//! - **Rows** (`review-replay-receipt-v1`), one per `(engine, item)`. +//! - **Summary.** Refuses a mixed set, a GGUF or cell mismatch, unequal item +//! coverage between engines, duplicate rows, any not-run row, or a missing +//! voter latency sample. Every ratio is `apr / llama_cpp`: TTFT wants ≤ 2.0, +//! prefill and decode want ≥ 1.0. A server that reports no timing yields +//! `null`, never a number. + +use std::collections::{BTreeMap, BTreeSet}; + +use serde::{Deserialize, Serialize}; + +use crate::contamination::Index; +use crate::corpus::sha256_hex; +use crate::receipt::Verdict; +use crate::stats::{percentile, percentile_sorted, SplitMix64}; + +pub const SCHEMA: &str = "review-replay-v1"; +pub const ROW_SCHEMA: &str = "review-replay-receipt-v1"; +pub const PER_STRATUM: usize = 50; +/// Bootstrap resamples for the p95 interval. +pub const BOOTSTRAP: usize = 2000; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)] +pub enum Stratum { + #[serde(rename = "2k")] + K2, + #[serde(rename = "8k")] + K8, + #[serde(rename = "16k")] + K16, + #[serde(rename = "32k")] + K32, +} + +impl Stratum { + pub const ALL: [Self; 4] = [Self::K2, Self::K8, Self::K16, Self::K32]; + + /// Inclusive upper bound in input tokens. + #[must_use] + pub fn upper(self) -> u64 { + match self { + Self::K2 => 2048, + Self::K8 => 8192, + Self::K16 => 16384, + Self::K32 => 32768, + } + } + + #[must_use] + pub fn as_str(self) -> &'static str { + match self { + Self::K2 => "2k", + Self::K8 => "8k", + Self::K16 => "16k", + Self::K32 => "32k", + } + } + + /// The stratum of an input of `tokens` tokens; `None` for 0 or > 32768. + #[must_use] + pub fn of(tokens: u64) -> Option { + if tokens == 0 { + return None; + } + Self::ALL.into_iter().find(|s| tokens <= s.upper()) + } +} + +/// A candidate diff, before sealing and stratification. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Candidate { + /// `owner/repo#N`. + pub group: String, + pub diff_sha256: String, + /// Composed prompt tokens, counted by the replay GGUF's tokenizer. + pub input_tokens: u64, + pub diff: String, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct Item { + pub group: String, + pub diff_sha256: String, + pub input_tokens: u64, + pub stratum: Stratum, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Set { + pub version: String, + pub seed: u64, + pub per_stratum: usize, + pub items: Vec, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum BuildError { + /// Nothing is sealed: the exclusion would prove nothing. + EmptySealedIndex, + /// Strata that could not be filled. + Short(Vec), +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct Short { + pub stratum: Stratum, + pub have: usize, + pub need: usize, +} + +fn rank_key(seed: u64, diff_sha: &str) -> String { + sha256_hex(format!("{seed}\u{0}{diff_sha}").as_bytes()) +} + +/// Build the frozen set (see module docs). +/// +/// # Errors +/// [`BuildError::EmptySealedIndex`] when `sealed` is empty, and +/// [`BuildError::Short`] listing every stratum with fewer than `per_stratum` +/// eligible items. +pub fn build( + version: &str, + seed: u64, + per_stratum: usize, + cands: &[Candidate], + sealed: &Index, +) -> Result { + if sealed.is_empty() { + return Err(BuildError::EmptySealedIndex); + } + // Deterministic order before the one-per-group / one-per-sha dedup, so + // input order never decides which duplicate survives. + let mut eligible: Vec<(String, Item)> = cands + .iter() + .filter(|c| { + sealed + .scan(&format!("{}\n{}", c.diff_sha256, c.diff), &c.group) + .is_empty() + }) + .filter_map(|c| { + Stratum::of(c.input_tokens).map(|stratum| { + ( + rank_key(seed, &c.diff_sha256), + Item { + group: c.group.clone(), + diff_sha256: c.diff_sha256.clone(), + input_tokens: c.input_tokens, + stratum, + }, + ) + }) + }) + .collect(); + eligible.sort_by(|a, b| a.0.cmp(&b.0).then_with(|| a.1.group.cmp(&b.1.group))); + let mut groups = BTreeSet::new(); + let mut shas = BTreeSet::new(); + let mut by: BTreeMap> = BTreeMap::new(); + for (_, it) in eligible { + if groups.contains(&it.group) || shas.contains(&it.diff_sha256) { + continue; + } + let bucket = by.entry(it.stratum).or_default(); + if bucket.len() < per_stratum { + groups.insert(it.group.clone()); + shas.insert(it.diff_sha256.clone()); + bucket.push(it); + } + } + let short: Vec = Stratum::ALL + .into_iter() + .filter_map(|s| { + let have = by.get(&s).map_or(0, Vec::len); + (have < per_stratum).then_some(Short { + stratum: s, + have, + need: per_stratum, + }) + }) + .collect(); + if !short.is_empty() { + return Err(BuildError::Short(short)); + } + Ok(Set { + version: version.to_string(), + seed, + per_stratum, + items: Stratum::ALL + .into_iter() + .flat_map(|s| by.remove(&s).unwrap_or_default()) + .collect(), + }) +} + +impl Set { + /// JSONL: a header line, then one item per line. + #[must_use] + pub fn render(&self) -> String { + let head: serde_json::Map = [ + ("schema", serde_json::Value::from(SCHEMA)), + ("version", self.version.as_str().into()), + ("seed", self.seed.into()), + ("per_stratum", self.per_stratum.into()), + ] + .into_iter() + .map(|(k, v)| (k.to_string(), v)) + .collect(); + let mut out = serde_json::Value::Object(head).to_string(); + out.push('\n'); + for it in &self.items { + out.push_str(&serde_json::to_string(it).unwrap_or_default()); + out.push('\n'); + } + out + } + + /// sha256 of [`Set::render`]; every row carries it. + #[must_use] + pub fn sha(&self) -> String { + sha256_hex(self.render().as_bytes()) + } + + /// Parse a rendered set. + /// + /// # Errors + /// A missing or foreign header, or a malformed item line. + pub fn parse(text: &str) -> Result { + let mut lines = text.lines(); + let head: serde_json::Value = lines + .next() + .and_then(|l| serde_json::from_str(l).ok()) + .ok_or("no header")?; + if head["schema"] != SCHEMA { + return Err(format!("schema is {}, want {SCHEMA}", head["schema"])); + } + let version = head["version"].as_str().ok_or("no version")?.to_string(); + let seed = head["seed"].as_u64().ok_or("no seed")?; + let per_stratum = head["per_stratum"] + .as_u64() + .and_then(|n| usize::try_from(n).ok()) + .ok_or("no per_stratum")?; + let items = lines + .enumerate() + .map(|(i, l)| serde_json::from_str(l).map_err(|e| format!("item {}: {e}", i + 1))) + .collect::>()?; + Ok(Self { + version, + seed, + per_stratum, + items, + }) + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum Engine { + Apr, + LlamaCpp, +} + +/// One `review-replay-receipt-v1` row. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct Row { + pub schema: String, + pub replay_version: String, + pub set_sha: String, + pub diff_sha256: String, + pub stratum: Stratum, + pub engine: Engine, + pub engine_version: String, + /// The apr tag under test (the same on the llama.cpp rows of that run). + pub apr_tag: String, + pub cell: String, + pub gguf_sha256: String, + pub wall_ms: f64, + /// Server-reported prompt (prefill) time: the TTFT of a non-streamed call. + pub prompt_ms: Option, + pub prompt_tps: Option, + pub decode_tps: Option, + pub input_tokens: Option, + pub output_tokens: Option, + pub peak_rss_mb: Option, + pub verdict: Verdict, +} + +/// Engine A over engine B (`apr / llama_cpp`). `ratio` is null when either +/// side is null or B is zero. +#[derive(Debug, Clone, Copy, PartialEq, Serialize, Deserialize)] +pub struct Ratio { + pub apr: Option, + pub llama_cpp: Option, + pub ratio: Option, +} + +impl Ratio { + fn of(apr: Option, llama_cpp: Option) -> Self { + let ratio = match (apr, llama_cpp) { + (Some(a), Some(b)) if b > 0.0 => Some(a / b), + _ => None, + }; + Self { + apr, + llama_cpp, + ratio, + } + } +} + +/// The §9 speed block. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct Speed { + pub replay_version: String, + pub set_sha: String, + pub apr_tag: String, + pub cell: String, + pub gguf_sha256: String, + pub n_items: usize, + pub p50_s: f64, + pub p95_s: f64, + pub p95_ci: [f64; 2], + pub queue_budget_p95_s: f64, + pub queue_budget_lane: String, + pub within_budget: bool, + pub ttft_ms: Ratio, + pub prefill_tps: BTreeMap, + pub decode_tps: Ratio, + pub peak_rss_mb: Ratio, + pub parse_rate: f64, + pub verdict_identity_vs_prev_tag: Option, +} + +#[derive(Debug, Clone, PartialEq)] +pub enum SummaryError { + NoRows, + MixedSet, + Mismatch(&'static str), + DuplicateRow { engine: Engine, diff_sha256: String }, + CoverageDiffers, + NotRun { engine: Engine, diff_sha256: String }, + NoVoterLatency, + PrevSetDiffers, + PrevCoverageDiffers, +} + +/// Per-engine rows keyed by diff sha. +type ByItem<'a> = BTreeMap<&'a str, &'a Row>; + +fn index(rows: &[Row]) -> Result<[ByItem<'_>; 2], SummaryError> { + let mut out = [BTreeMap::new(), BTreeMap::new()]; + for r in rows { + if let Verdict::NotRun(_) = r.verdict { + return Err(SummaryError::NotRun { + engine: r.engine, + diff_sha256: r.diff_sha256.clone(), + }); + } + let slot = usize::from(r.engine == Engine::LlamaCpp); + if out[slot].insert(r.diff_sha256.as_str(), r).is_some() { + return Err(SummaryError::DuplicateRow { + engine: r.engine, + diff_sha256: r.diff_sha256.clone(), + }); + } + } + Ok(out) +} + +fn median(xs: impl Iterator>) -> Option { + // A single null makes the engine's figure null: never a median of a subset. + let v: Option> = xs.collect(); + v.filter(|v| !v.is_empty()).map(|v| percentile(&v, 0.5)) +} + +fn p95_ci(wall_s: &[f64], seed: u64) -> [f64; 2] { + let mut rng = SplitMix64::new(seed); + let mut stats: Vec = (0..BOOTSTRAP) + .map(|_| { + let s: Vec = (0..wall_s.len()) + .map(|_| wall_s[rng.below(wall_s.len())]) + .collect(); + percentile(&s, 0.95) + }) + .collect(); + stats.sort_by(f64::total_cmp); + [ + percentile_sorted(&stats, 0.025), + percentile_sorted(&stats, 0.975), + ] +} + +/// Fold one run's rows into the §9 block. +/// +/// `voters` is `(lane, total seconds per review)` for every counted voter; +/// the queue budget is the largest lane p95. `prev` is the previous tag's +/// apr rows on the same set, for verdict identity. +/// +/// # Errors +/// See [`SummaryError`]; every refusal names what was inconsistent. +pub fn summarize( + rows: &[Row], + voters: &[(String, Vec)], + prev: Option<&[Row]>, + seed: u64, +) -> Result { + let first = rows.first().ok_or(SummaryError::NoRows)?; + for r in rows { + if r.set_sha != first.set_sha || r.replay_version != first.replay_version { + return Err(SummaryError::MixedSet); + } + if r.gguf_sha256 != first.gguf_sha256 { + return Err(SummaryError::Mismatch("gguf_sha256")); + } + if r.cell != first.cell { + return Err(SummaryError::Mismatch("cell")); + } + if r.apr_tag != first.apr_tag { + return Err(SummaryError::Mismatch("apr_tag")); + } + } + let [apr, llama] = index(rows)?; + if apr.is_empty() || !apr.keys().eq(llama.keys()) { + return Err(SummaryError::CoverageDiffers); + } + if voters.is_empty() || voters.iter().any(|(_, v)| v.is_empty()) { + return Err(SummaryError::NoVoterLatency); + } + let (queue_budget_lane, queue_budget_p95_s) = voters + .iter() + .map(|(lane, v)| (lane.clone(), percentile(v, 0.95))) + .max_by(|a, b| a.1.total_cmp(&b.1)) + .ok_or(SummaryError::NoVoterLatency)?; + + let wall_s: Vec = apr.values().map(|r| r.wall_ms / 1000.0).collect(); + let p95_s = percentile(&wall_s, 0.95); + let pair = |f: fn(&Row) -> Option, keep: &dyn Fn(&Row) -> bool| { + Ratio::of( + median(apr.values().filter(|r| keep(r)).map(|r| f(r))), + median(llama.values().filter(|r| keep(r)).map(|r| f(r))), + ) + }; + let all = |_: &Row| true; + let prefill_tps = Stratum::ALL + .into_iter() + .filter(|s| apr.values().any(|r| r.stratum == *s)) + .map(|s| (s, pair(|r| r.prompt_tps, &|r: &Row| r.stratum == s))) + .collect(); + let peak = |m: &ByItem<'_>| -> Option { + let v: Option> = m.values().map(|r| r.peak_rss_mb).collect(); + v.and_then(|v| v.into_iter().reduce(f64::max)) + }; + let parsed = apr + .values() + .filter(|r| matches!(r.verdict, Verdict::Pass | Verdict::Fail)) + .count(); + + let verdict_identity_vs_prev_tag = match prev { + None => None, + Some(prev) => { + let mut p = BTreeMap::new(); + for r in prev.iter().filter(|r| r.engine == Engine::Apr) { + if r.set_sha != first.set_sha { + return Err(SummaryError::PrevSetDiffers); + } + p.insert(r.diff_sha256.as_str(), r.verdict); + } + if !p.keys().eq(apr.keys()) { + return Err(SummaryError::PrevCoverageDiffers); + } + let same = apr + .values() + .filter(|r| p[r.diff_sha256.as_str()] == r.verdict) + .count(); + Some(same as f64 / apr.len() as f64) + } + }; + + Ok(Speed { + replay_version: first.replay_version.clone(), + set_sha: first.set_sha.clone(), + apr_tag: first.apr_tag.clone(), + cell: first.cell.clone(), + gguf_sha256: first.gguf_sha256.clone(), + n_items: apr.len(), + p50_s: percentile(&wall_s, 0.5), + p95_s, + p95_ci: p95_ci(&wall_s, seed), + queue_budget_p95_s, + queue_budget_lane, + within_budget: p95_s <= queue_budget_p95_s, + ttft_ms: pair(|r| r.prompt_ms, &all), + prefill_tps, + decode_tps: pair(|r| r.decode_tps, &all), + peak_rss_mb: Ratio::of(peak(&apr), peak(&llama)), + parse_rate: parsed as f64 / apr.len() as f64, + verdict_identity_vs_prev_tag, + }) +} + +#[cfg(test)] +#[path = "replay_tests.rs"] +mod tests; diff --git a/crates/aprender-review-experiment/src/replay_tests.rs b/crates/aprender-review-experiment/src/replay_tests.rs new file mode 100644 index 0000000000..a726c63565 --- /dev/null +++ b/crates/aprender-review-experiment/src/replay_tests.rs @@ -0,0 +1,415 @@ +use super::*; +use crate::contamination::Index; +use crate::corpus::{hunk_fingerprints, sha256_hex, Sealed}; +use crate::receipt::NotRun; + +fn diff(tag: &str) -> String { + format!("diff --git a/{tag}.rs b/{tag}.rs\n--- a/{tag}.rs\n+++ b/{tag}.rs\n@@ -1,3 +1,4 @@\n fn {tag}() {{\n- old_{tag}();\n+ new_{tag}();\n+ more_{tag}();\n }}\n") +} + +fn cand(group: &str, tag: &str, tokens: u64) -> Candidate { + let d = diff(tag); + Candidate { + group: group.to_string(), + diff_sha256: sha256_hex(d.as_bytes()), + input_tokens: tokens, + diff: d, + } +} + +fn sealed_of(c: &Candidate) -> Sealed { + Sealed { + id: format!("G-{}", c.group), + diff_sha256: c.diff_sha256.clone(), + hunks: hunk_fingerprints(&c.diff), + } +} + +/// `n` fresh candidates per stratum, each its own group. +fn pool(n: usize) -> Vec { + let mut v = Vec::new(); + for s in Stratum::ALL { + for i in 0..n { + let tag = format!("{}_{i}", s.as_str()); + v.push(cand( + &format!("paiml/x#{tag}"), + &format!("f{tag}"), + s.upper() - i as u64, + )); + } + } + v +} + +fn unrelated_seal() -> Index { + Index::new(&[sealed_of(&cand("paiml/sealed#1", "sealedonly", 10))]) +} + +#[test] +fn stratum_bounds_are_inclusive_and_refuse_zero_and_overflow() { + assert_eq!(Stratum::of(0), None); + assert_eq!(Stratum::of(1), Some(Stratum::K2)); + assert_eq!(Stratum::of(2048), Some(Stratum::K2)); + assert_eq!(Stratum::of(2049), Some(Stratum::K8)); + assert_eq!(Stratum::of(8192), Some(Stratum::K8)); + assert_eq!(Stratum::of(8193), Some(Stratum::K16)); + assert_eq!(Stratum::of(16384), Some(Stratum::K16)); + assert_eq!(Stratum::of(16385), Some(Stratum::K32)); + assert_eq!(Stratum::of(32768), Some(Stratum::K32)); + assert_eq!(Stratum::of(32769), None); +} + +/// FALSIFY-REPLAY-001: a sealed item never enters the set, by diff sha, +/// by hunk or by a copy under another group. +#[test] +fn falsify_replay_001_sealed_items_are_excluded() { + let mut cands = pool(3); + let leak = cands[0].clone(); + // Same diff under another PR: the diff-sha / hunk hit must still drop it. + let mut copy = leak.clone(); + copy.group = "paiml/other#9".into(); + cands.push(copy); + let ix = Index::new(&[sealed_of(&leak)]); + let err = build("v1", 4354, 3, &cands, &ix).unwrap_err(); + assert_eq!( + err, + BuildError::Short(vec![Short { + stratum: Stratum::K2, + have: 2, + need: 3 + }]), + "only the sealed diff's stratum may come up short" + ); + let set = build("v1", 4354, 2, &cands, &ix).expect("2 per stratum fits"); + assert!(set.items.iter().all(|i| i.diff_sha256 != leak.diff_sha256)); +} + +#[test] +fn an_empty_sealed_index_is_refused() { + assert_eq!( + build("v1", 1, 1, &pool(1), &Index::default()), + Err(BuildError::EmptySealedIndex) + ); +} + +/// FALSIFY-REPLAY-002: a short stratum is an error, never padded. +#[test] +fn falsify_replay_002_short_strata_are_refused_not_padded() { + let mut cands = pool(2); + cands.retain(|c| Stratum::of(c.input_tokens) != Some(Stratum::K32)); + cands.push(cand("paiml/x#big", "big", 40_000)); + let err = build("v1", 1, 2, &cands, &unrelated_seal()).unwrap_err(); + assert_eq!( + err, + BuildError::Short(vec![Short { + stratum: Stratum::K32, + have: 0, + need: 2 + }]) + ); +} + +/// FALSIFY-REPLAY-003: one item per repo#PR and per diff sha, and the set +/// does not depend on candidate order. +#[test] +fn falsify_replay_003_one_per_group_and_order_free() { + let mut cands = pool(4); + // Two more diffs on an existing PR and a re-post of an existing diff. + let g = cands[0].group.clone(); + cands.push(cand(&g, "extra_a", 100)); + cands.push(cand(&g, "extra_b", 200)); + let mut repost = cands[1].clone(); + repost.group = "paiml/y#1".into(); + cands.push(repost); + let a = build("v1", 7, 4, &cands, &unrelated_seal()).expect("fits"); + cands.reverse(); + let b = build("v1", 7, 4, &cands, &unrelated_seal()).expect("fits"); + assert_eq!(a, b); + assert_eq!(a.sha(), b.sha()); + let groups: BTreeSet<_> = a.items.iter().map(|i| &i.group).collect(); + let shas: BTreeSet<_> = a.items.iter().map(|i| &i.diff_sha256).collect(); + assert_eq!(groups.len(), a.items.len()); + assert_eq!(shas.len(), a.items.len()); + assert_eq!(a.items.len(), 16); + for s in Stratum::ALL { + assert_eq!(a.items.iter().filter(|i| i.stratum == s).count(), 4); + } + let c = build("v1", 8, 4, &cands, &unrelated_seal()).expect("fits"); + assert_ne!(a.sha(), c.sha(), "the seed is part of the frozen identity"); +} + +#[test] +fn render_parse_round_trips_and_rejects_a_foreign_header() { + let set = build("v1", 3, 2, &pool(3), &unrelated_seal()).expect("fits"); + let text = set.render(); + assert_eq!(Set::parse(&text), Ok(set.clone())); + assert!(Set::parse(&text.replace(SCHEMA, "other-v1")).is_err()); + assert!(Set::parse("").is_err()); + assert_eq!(set.sha(), sha256_hex(text.as_bytes())); +} + +fn row(engine: Engine, i: usize, wall_ms: f64) -> Row { + let stratum = Stratum::ALL[i % 4]; + let (pt, dt) = match engine { + Engine::Apr => (500.0, 20.0), + Engine::LlamaCpp => (1000.0, 40.0), + }; + Row { + schema: ROW_SCHEMA.into(), + replay_version: "v1".into(), + set_sha: "s".repeat(64), + diff_sha256: format!("{i:064x}"), + stratum, + engine, + engine_version: "x".into(), + apr_tag: "v0.70.1".into(), + cell: "lambda-4090".into(), + gguf_sha256: "g".repeat(64), + wall_ms, + prompt_ms: Some(if engine == Engine::Apr { 300.0 } else { 100.0 }), + prompt_tps: Some(pt * (1.0 + (i % 4) as f64)), + decode_tps: Some(dt), + input_tokens: Some(1000), + output_tokens: Some(64), + peak_rss_mb: Some(if engine == Engine::Apr { + 3000.0 + i as f64 + } else { + 2000.0 + }), + verdict: if i % 5 == 0 { + Verdict::Unparsed + } else { + Verdict::Pass + }, + } +} + +fn run(n: usize) -> Vec { + let mut v: Vec = (0..n) + .map(|i| row(Engine::Apr, i, 1000.0 * (i + 1) as f64)) + .collect(); + v.extend((0..n).map(|i| row(Engine::LlamaCpp, i, 500.0))); + v +} + +fn voters() -> Vec<(String, Vec)> { + vec![ + ("haiku".into(), (1..=20).map(f64::from).collect()), + ("agy".into(), (1..=20).map(|x| f64::from(x) * 3.0).collect()), + ] +} + +/// FALSIFY-REPLAY-004: ratios are apr / llama_cpp and the queue budget is +/// the slowest counted voter's p95. +#[test] +fn falsify_replay_004_ratios_and_queue_budget() { + let s = summarize(&run(20), &voters(), None, 1).expect("valid run"); + assert_eq!(s.n_items, 20); + assert!((s.p50_s - 10.0).abs() < 1e-9, "p50 {}", s.p50_s); + assert!((s.p95_s - 19.0).abs() < 1e-9, "p95 {}", s.p95_s); + assert!( + s.p95_ci[0] <= s.p95_s && s.p95_s <= s.p95_ci[1], + "{:?}", + s.p95_ci + ); + assert_eq!(s.queue_budget_lane, "agy"); + assert!((s.queue_budget_p95_s - 57.0).abs() < 1e-9); + assert!(s.within_budget); + assert_eq!(s.ttft_ms.ratio, Some(3.0)); + assert_eq!(s.decode_tps.ratio, Some(0.5)); + assert_eq!(s.prefill_tps.len(), 4); + for (st, r) in &s.prefill_tps { + let k = 1.0 + Stratum::ALL.iter().position(|x| x == st).unwrap_or(9) as f64; + assert_eq!(r.apr, Some(500.0 * k), "{st:?}"); + assert_eq!(r.ratio, Some(0.5)); + } + assert_eq!(s.peak_rss_mb.apr, Some(3019.0)); + assert!((s.parse_rate - 0.8).abs() < 1e-9); + assert_eq!(s.verdict_identity_vs_prev_tag, None); + + let slow = vec![("haiku".to_string(), vec![5.0])]; + assert!( + !summarize(&run(20), &slow, None, 1) + .expect("valid") + .within_budget + ); +} + +#[test] +fn falsify_replay_004_a_null_server_timing_is_null_not_a_subset_median() { + let mut rows = run(8); + rows[9].prompt_ms = None; // one llama row + let s = summarize(&rows, &voters(), None, 1).expect("valid"); + assert_eq!(s.ttft_ms.llama_cpp, None); + assert_eq!(s.ttft_ms.ratio, None); + assert_eq!(s.ttft_ms.apr, Some(300.0)); +} + +/// FALSIFY-REPLAY-005: an inconsistent run never prints a number. +#[test] +fn falsify_replay_005_inconsistent_runs_are_refused() { + let v = voters(); + let base = run(8); + assert_eq!(summarize(&[], &v, None, 1), Err(SummaryError::NoRows)); + + let mut r = base.clone(); + r[3].set_sha = "t".repeat(64); + assert_eq!(summarize(&r, &v, None, 1), Err(SummaryError::MixedSet)); + let mut r = base.clone(); + r[3].replay_version = "v2".into(); + assert_eq!(summarize(&r, &v, None, 1), Err(SummaryError::MixedSet)); + let mut r = base.clone(); + r[9].gguf_sha256 = "h".repeat(64); + assert_eq!( + summarize(&r, &v, None, 1), + Err(SummaryError::Mismatch("gguf_sha256")) + ); + let mut r = base.clone(); + r[9].cell = "gx10".into(); + assert_eq!( + summarize(&r, &v, None, 1), + Err(SummaryError::Mismatch("cell")) + ); + let mut r = base.clone(); + r[9].apr_tag = "v0.70.0".into(); + assert_eq!( + summarize(&r, &v, None, 1), + Err(SummaryError::Mismatch("apr_tag")) + ); + + let mut r = base.clone(); + r.pop(); + assert_eq!( + summarize(&r, &v, None, 1), + Err(SummaryError::CoverageDiffers) + ); + let mut r = base.clone(); + r.push(r[0].clone()); + assert!(matches!( + summarize(&r, &v, None, 1), + Err(SummaryError::DuplicateRow { .. }) + )); + let mut r = base.clone(); + r[2].verdict = Verdict::NotRun(NotRun::ContextOverflow); + assert!(matches!( + summarize(&r, &v, None, 1), + Err(SummaryError::NotRun { + engine: Engine::Apr, + .. + }) + )); + let llama_only: Vec = base + .iter() + .filter(|r| r.engine == Engine::LlamaCpp) + .cloned() + .collect(); + assert_eq!( + summarize(&llama_only, &v, None, 1), + Err(SummaryError::CoverageDiffers) + ); + + assert_eq!( + summarize(&base, &[], None, 1), + Err(SummaryError::NoVoterLatency) + ); + let empty_lane = vec![("haiku".to_string(), vec![])]; + assert_eq!( + summarize(&base, &empty_lane, None, 1), + Err(SummaryError::NoVoterLatency) + ); +} + +/// FALSIFY-REPLAY-006: verdict identity compares the same set, item by item. +#[test] +fn falsify_replay_006_verdict_identity_vs_previous_tag() { + let now = run(10); + let mut prev: Vec = now.clone(); + for r in prev.iter_mut().filter(|r| r.engine == Engine::Apr).take(3) { + r.verdict = Verdict::Fail; + r.apr_tag = "v0.70.0".into(); + } + let s = summarize(&now, &voters(), Some(&prev), 1).expect("valid"); + assert_eq!(s.verdict_identity_vs_prev_tag, Some(0.7)); + // llama rows in `prev` are ignored. + let apr_prev: Vec = prev + .iter() + .filter(|r| r.engine == Engine::Apr) + .cloned() + .collect(); + assert_eq!( + summarize(&now, &voters(), Some(&apr_prev), 1) + .expect("valid") + .verdict_identity_vs_prev_tag, + Some(0.7) + ); + let mut other = apr_prev.clone(); + other[0].set_sha = "t".repeat(64); + assert_eq!( + summarize(&now, &voters(), Some(&other), 1), + Err(SummaryError::PrevSetDiffers) + ); + let mut fewer = apr_prev; + fewer.pop(); + assert_eq!( + summarize(&now, &voters(), Some(&fewer), 1), + Err(SummaryError::PrevCoverageDiffers) + ); +} + +#[test] +fn row_serializes_with_stable_engine_and_stratum_names() { + let v = serde_json::to_value(row(Engine::LlamaCpp, 1, 1.0)).expect("serializes"); + assert_eq!(v["engine"], "llama_cpp"); + assert_eq!(v["stratum"], "8k"); + assert_eq!(v["schema"], ROW_SCHEMA); +} + +/// FALSIFY-REPLAY-003: two PRs with many diffs each cannot fill a stratum of 3. +#[test] +fn falsify_replay_003_many_diffs_on_one_pr_count_once() { + let mut cands: Vec = pool(3) + .into_iter() + .filter(|c| Stratum::of(c.input_tokens) != Some(Stratum::K2)) + .collect(); + for i in 0..6 { + cands.push(cand("paiml/a#1", &format!("a{i}"), 100 + i)); + cands.push(cand("paiml/b#2", &format!("b{i}"), 200 + i)); + } + assert_eq!( + build("v1", 1, 3, &cands, &unrelated_seal()), + Err(BuildError::Short(vec![Short { + stratum: Stratum::K2, + have: 2, + need: 3 + }])) + ); +} + +/// FALSIFY-REPLAY-002: a stratum is never over-filled either. +#[test] +fn falsify_replay_002_each_stratum_holds_exactly_per_stratum() { + let set = build("v1", 1, 2, &pool(5), &unrelated_seal()).expect("fits"); + assert_eq!(set.items.len(), 8); + for s in Stratum::ALL { + assert_eq!(set.items.iter().filter(|i| i.stratum == s).count(), 2); + } +} + +#[test] +fn falsify_replay_004_ratio_and_median_edges() { + assert_eq!(Ratio::of(Some(0.5), Some(2.0)).ratio, Some(0.25)); + assert_eq!(Ratio::of(Some(1.0), Some(0.0)).ratio, None); + assert_eq!(Ratio::of(None, Some(2.0)).ratio, None); + let xs: Vec> = (1..=10).map(|x| Some(f64::from(x))).collect(); + assert_eq!(median(xs.into_iter()), Some(5.0)); + assert_eq!(median(std::iter::empty()), None); + assert_eq!(median([Some(1.0), None].into_iter()), None); +} + +#[test] +fn falsify_replay_004_p95_equal_to_the_budget_is_within() { + let exact = vec![("haiku".to_string(), vec![19.0])]; + let s = summarize(&run(20), &exact, None, 1).expect("valid"); + assert!((s.p95_s - s.queue_budget_p95_s).abs() < 1e-12); + assert!(s.within_budget); +} diff --git a/crates/aprender-review-experiment/src/score.rs b/crates/aprender-review-experiment/src/score.rs new file mode 100644 index 0000000000..25fab6345e --- /dev/null +++ b/crates/aprender-review-experiment/src/score.rs @@ -0,0 +1,351 @@ +//! REX-03 scorer: §2.3 metrics and the §3 hypothesis inputs, over receipts. +//! +//! The denominator is always the EXPECTED item set for a (cell, arm), never +//! the receipts that happen to exist: an item with no admissible receipt is +//! scored as `NotRun{Inadmissible}`, so dropping or corrupting a row can only +//! lower a score. The statistics themselves are the frozen `stats.rs`. + +use crate::corpus::{Class, Item}; +use crate::receipt::{admissible, localized, parse_verdict, Expect, NotRun, Receipt, Verdict}; +use crate::stats::{self, Paired, VsVoters}; +use serde::Serialize; +use std::collections::BTreeMap; + +/// A proportion with its Wilson 95 % interval (`None` when n = 0). +#[derive(Debug, Clone, Copy, PartialEq, Serialize)] +pub struct Ratio { + pub k: u64, + pub n: u64, + pub ci: Option<(f64, f64)>, +} + +impl Ratio { + #[must_use] + pub fn new(k: u64, n: u64) -> Self { + Self { + k, + n, + ci: stats::wilson(k, n, stats::Z95), + } + } + + /// Point estimate (`None` when n = 0). + #[must_use] + pub fn value(&self) -> Option { + (self.n > 0).then(|| self.k as f64 / self.n as f64) + } +} + +/// One expected item's outcome for one (cell, arm). +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Scored { + pub id: String, + /// Corpus class; H5 reads class R only (spec v2 §3). + pub class: Class, + pub defect: bool, + pub verdict: Verdict, + /// A finding names the defect file (only set on a FAIL of a defect). + pub localized: bool, + /// sha of the raw output; `None` when nothing ran. + pub output_sha: Option, +} + +/// §2.3 metrics for one (cell, arm). +#[derive(Debug, Clone, PartialEq, Serialize)] +pub struct LaneScore { + pub items: u64, + pub parse_rate: Ratio, + pub recall: Ratio, + pub precision: Ratio, + pub false_refute: Ratio, + pub localization: Ratio, + pub correct: Ratio, + pub unparsed: u64, + pub not_run: BTreeMap, +} + +/// Score a lane. Recall's denominator is every defect item, run or not; +/// precision's is every parsed FAIL; `Unparsed` is attempted but never a +/// FAIL or a PASS; `NotRun` is not attempted and never correct. +#[must_use] +pub fn score(rows: &[Scored]) -> LaneScore { + let n = |f: &dyn Fn(&Scored) -> bool| rows.iter().filter(|r| f(r)).count() as u64; + let fail = |r: &Scored| r.verdict == Verdict::Fail; + let mut not_run = BTreeMap::new(); + for r in rows { + if let Verdict::NotRun(why) = r.verdict { + *not_run.entry(format!("{why:?}")).or_insert(0) += 1; + } + } + LaneScore { + items: rows.len() as u64, + parse_rate: Ratio::new( + n(&|r| matches!(r.verdict, Verdict::Pass | Verdict::Fail)), + n(&|r| r.verdict.executed()), + ), + recall: Ratio::new(n(&|r| r.defect && fail(r)), n(&|r| r.defect)), + precision: Ratio::new(n(&|r| r.defect && fail(r)), n(&fail)), + false_refute: Ratio::new(n(&|r| !r.defect && fail(r)), n(&|r| !r.defect)), + localization: Ratio::new( + n(&|r| r.defect && fail(r) && r.localized), + n(&|r| r.defect && fail(r)), + ), + correct: Ratio::new(n(&|r| r.verdict.correct(r.defect)), rows.len() as u64), + unparsed: n(&|r| r.verdict == Verdict::Unparsed), + not_run, + } +} + +/// Why a receipt line was not used. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Rejected { + pub line: usize, + pub reasons: Vec, +} + +/// Turn receipt lines into one [`Scored`] per expected item for (cell, arm, +/// rerun). A row is used only if it is admissible, belongs to the selection, +/// names an expected item with the right diff sha and class, is the only row +/// for that item, and (when it ran) its raw output exists with the recorded +/// sha. The verdict is RE-PARSED from that raw output; the row's own verdict +/// must agree with it. +pub fn collect( + lines: &str, + expect: Expect<'_>, + expected: &[&Item], + select: impl Fn(&Receipt) -> bool, + read_raw: F, +) -> (Vec, Vec) +where + F: Fn(&str) -> Option, +{ + let by_id: BTreeMap<&str, &Item> = expected.iter().map(|i| (i.id.as_str(), *i)).collect(); + let mut got: BTreeMap = BTreeMap::new(); + let mut dup: Vec = Vec::new(); + let mut rejected = Vec::new(); + for (ix, line) in lines + .lines() + .enumerate() + .filter(|(_, l)| !l.trim().is_empty()) + { + let r = match admissible(line, expect) { + Ok(r) if select(&r) => r, + Ok(_) => continue, + Err(reasons) => { + rejected.push(Rejected { + line: ix + 1, + reasons, + }); + continue; + } + }; + match check(&r, &by_id, &read_raw) { + Ok(s) => { + if got.insert(s.id.clone(), s).is_some() { + dup.push(r.item_id.clone()); + } + } + Err(reasons) => rejected.push(Rejected { + line: ix + 1, + reasons, + }), + } + } + for id in &dup { + got.remove(id); + rejected.push(Rejected { + line: 0, + reasons: vec![format!("{id}: more than one row")], + }); + } + let rows = expected + .iter() + .map(|i| { + got.remove(&i.id).unwrap_or_else(|| Scored { + id: i.id.clone(), + class: i.class, + defect: i.class.is_defect(), + verdict: Verdict::NotRun(NotRun::Inadmissible), + localized: false, + output_sha: None, + }) + }) + .collect(); + (rows, rejected) +} + +fn check(r: &Receipt, by_id: &BTreeMap<&str, &Item>, read_raw: &F) -> Result> +where + F: Fn(&str) -> Option, +{ + let item = by_id + .get(r.item_id.as_str()) + .ok_or_else(|| vec![format!("{}: not an expected item", r.item_id)])?; + if item.diff_sha256 != r.item_sha256 || item.class != r.class { + return Err(vec![format!( + "{}: item sha/class differs from the corpus", + r.item_id + )]); + } + let defect = item.class.is_defect(); + let Some(out) = &r.output else { + return Ok(Scored { + id: r.item_id.clone(), + class: item.class, + defect, + verdict: r.verdict, + localized: false, + output_sha: None, + }); + }; + let text = + read_raw(&out.path).ok_or_else(|| vec![format!("{}: raw output missing", r.item_id)])?; + if crate::corpus::sha256_hex(text.as_bytes()) != out.sha256 { + return Err(vec![format!("{}: raw output sha mismatch", r.item_id)]); + } + let verdict = if r.verdict.executed() { + let v = parse_verdict(&text); + if v != r.verdict { + return Err(vec![format!( + "{}: recorded {:?}, raw parses {v:?}", + r.item_id, r.verdict + )]); + } + v + } else { + r.verdict + }; + Ok(Scored { + id: r.item_id.clone(), + class: item.class, + defect, + verdict, + localized: verdict == Verdict::Fail && defect && localized(&text, &item.defect), + output_sha: Some(out.sha256.clone()), + }) +} + +fn divergence_by(a: &[Scored], b: &[Scored], bytes: bool) -> (u64, u64) { + let bm: BTreeMap<&str, &Scored> = b.iter().map(|s| (s.id.as_str(), s)).collect(); + let (mut divergent, mut missing) = (0, 0); + for s in a { + match bm.get(s.id.as_str()) { + Some(t) if s.verdict.executed() && t.verdict.executed() => { + let differs = s.verdict != t.verdict || (bytes && s.output_sha != t.output_sha); + divergent += u64::from(differs); + } + _ => missing += 1, + } + } + (divergent, missing) +} + +/// H2 input (and the descriptive H1 byte count): items whose verdict or +/// output bytes differ between two runs, and items that did not run on both +/// (reported, not silently dropped). +#[must_use] +pub fn divergence(a: &[Scored], b: &[Scored]) -> (u64, u64) { + divergence_by(a, b, true) +} + +/// H1 input (spec v2 §3): items whose VERDICT differs across cells. Output +/// bytes are descriptive for H1 (`divergence`); a byte-only difference does +/// not reject it. +#[must_use] +pub fn verdict_divergence(a: &[Scored], b: &[Scored]) -> (u64, u64) { + divergence_by(a, b, false) +} + +/// H4/H5 input: items where BOTH arms parsed a verdict, paired; the rest are +/// dropped pairwise and counted (analysis plan). +#[must_use] +pub fn paired_parsed(a: &[Scored], b: &[Scored]) -> (Vec, u64) { + let bm: BTreeMap<&str, &Scored> = b.iter().map(|s| (s.id.as_str(), s)).collect(); + let parsed = |v: Verdict| matches!(v, Verdict::Pass | Verdict::Fail); + let mut out = Vec::new(); + let mut dropped = 0; + for s in a { + match bm.get(s.id.as_str()) { + Some(t) if parsed(s.verdict) && parsed(t.verdict) => out.push(Paired { + defect: s.defect, + a_fail: s.verdict == Verdict::Fail, + b_fail: t.verdict == Verdict::Fail, + }), + _ => dropped += 1, + } + } + (out, dropped) +} + +/// H5 input (spec v2 §3): `paired_parsed` restricted to items of `class` in +/// `a`. H5 decides on class R; class P is reported beside it (mutant confound). +#[must_use] +pub fn paired_parsed_class(a: &[Scored], b: &[Scored], class: Class) -> (Vec, u64) { + let only: Vec = a.iter().filter(|s| s.class == class).cloned().collect(); + paired_parsed(&only, b) +} + +/// H4 input (spec v2 §3): items where apr AND both voters parsed a verdict; +/// the rest are dropped and counted. +#[must_use] +pub fn vs_voters_parsed(apr: &[Scored], voters: [&[Scored]; 2]) -> (Vec, u64) { + let maps = voters.map(|v| { + v.iter() + .map(|s| (s.id.as_str(), s)) + .collect::>() + }); + let parsed = |v: Verdict| matches!(v, Verdict::Pass | Verdict::Fail); + let mut out = Vec::new(); + let mut dropped = 0; + for s in apr { + match (maps[0].get(s.id.as_str()), maps[1].get(s.id.as_str())) { + (Some(h), Some(g)) if parsed(s.verdict) && parsed(h.verdict) && parsed(g.verdict) => { + out.push(VsVoters { + defect: s.defect, + apr_fail: s.verdict == Verdict::Fail, + voter_fail: [h.verdict == Verdict::Fail, g.verdict == Verdict::Fail], + }); + } + _ => dropped += 1, + } + } + (out, dropped) +} + +/// κ input (spec v2 §3, descriptive): per-item error indicators of (a, b) +/// over items both lanes parsed. Error = not correct. +#[must_use] +pub fn error_pairs(a: &[Scored], b: &[Scored]) -> (Vec, Vec) { + paired_parsed(a, b) + .0 + .iter() + .map(|p| (p.a_fail != p.defect, p.b_fail != p.defect)) + .unzip() +} + +/// H3 input: per-item correctness of (challenger, champion) over the +/// champion's items; a missing challenger row is incorrect. +#[must_use] +pub fn correctness_pairs(challenger: &[Scored], champion: &[Scored]) -> Vec<(bool, bool)> { + let cm: BTreeMap<&str, &Scored> = challenger.iter().map(|s| (s.id.as_str(), s)).collect(); + champion + .iter() + .map(|c| { + let ch = cm + .get(c.id.as_str()) + .is_some_and(|s| s.verdict.correct(s.defect)); + (ch, c.verdict.correct(c.defect)) + }) + .collect() +} + +/// H3: one-sided McNemar exact p that the challenger (9B) beats the champion. +#[must_use] +pub fn h3_p(challenger: &[Scored], champion: &[Scored]) -> f64 { + let (b, c) = stats::discordant(&correctness_pairs(challenger, champion)); + stats::mcnemar_exact_one_sided(b, c) +} + +#[cfg(test)] +#[path = "score_tests.rs"] +mod tests; diff --git a/crates/aprender-review-experiment/src/score_tests.rs b/crates/aprender-review-experiment/src/score_tests.rs new file mode 100644 index 0000000000..a5ac454bfb --- /dev/null +++ b/crates/aprender-review-experiment/src/score_tests.rs @@ -0,0 +1,323 @@ +use super::*; +use crate::corpus::{sha256_hex, Class, Loc}; +use crate::receipt::tests::{row, EXPECT}; + +fn s(id: &str, defect: bool, verdict: Verdict, localized: bool) -> Scored { + Scored { + id: id.into(), + class: if defect { Class::P } else { Class::G }, + defect, + verdict, + localized, + output_sha: verdict.executed().then(|| format!("sha-{id}")), + } +} + +/// Hand-computed fixture: 6 defects, 4 good. +fn fixture() -> Vec { + use Verdict::{Fail, Pass, Unparsed}; + vec![ + s("d1", true, Fail, true), + s("d2", true, Fail, false), + s("d3", true, Fail, true), + s("d4", true, Pass, false), + s("d5", true, Unparsed, false), + s("d6", true, Verdict::NotRun(NotRun::ContextOverflow), false), + s("g1", false, Pass, false), + s("g2", false, Fail, false), + s("g3", false, Unparsed, false), + s( + "g4", + false, + Verdict::NotRun(NotRun::NoDeclaredExecutor), + false, + ), + ] +} + +fn kn(r: Ratio) -> (u64, u64) { + (r.k, r.n) +} + +#[test] +fn metrics_match_hand_computation() { + let l = score(&fixture()); + assert_eq!(l.items, 10); + assert_eq!(kn(l.parse_rate), (6, 8), "parsed ÷ executed"); + assert_eq!( + kn(l.recall), + (3, 6), + "NotRun defect stays in the denominator" + ); + assert_eq!(kn(l.precision), (3, 4)); + assert_eq!(kn(l.false_refute), (1, 4)); + assert_eq!(kn(l.localization), (2, 3)); + assert_eq!(kn(l.correct), (4, 10)); + assert_eq!(l.unparsed, 2); + assert_eq!(l.not_run.get("ContextOverflow"), Some(&1)); + assert_eq!(l.not_run.get("NoDeclaredExecutor"), Some(&1)); + // Wilson 3/6 by hand: centre 0.5, half-width 0.31238. + let (lo, hi) = l.recall.ci.expect("ci"); + assert!( + (lo - 0.187_62).abs() < 1e-4 && (hi - 0.812_38).abs() < 1e-4, + "{lo} {hi}" + ); + assert_eq!(Ratio::new(0, 0).value(), None); +} + +#[test] +fn falsify_rxr_002_not_run_is_never_correct() { + let why = [ + NotRun::NoDeclaredExecutor, + NotRun::ContextOverflow, + NotRun::Refused, + NotRun::ServeError, + NotRun::Inadmissible, + ]; + let rows: Vec = why + .iter() + .enumerate() + .flat_map(|(i, w)| { + [ + s(&format!("d{i}"), true, Verdict::NotRun(*w), false), + s(&format!("g{i}"), false, Verdict::NotRun(*w), false), + ] + }) + .collect(); + let l = score(&rows); + assert_eq!(kn(l.correct), (0, 10), "NotRun counted as correct"); + assert_eq!(kn(l.recall), (0, 5)); + assert_eq!(kn(l.false_refute), (0, 5)); + assert_eq!(kn(l.parse_rate), (0, 0), "NotRun is not attempted"); + // A missing receipt becomes NotRun{Inadmissible}, in the denominator. + let item = crate::corpus::Item::new("R-pr1".into(), Class::R, "t".into(), "--- a\n+++ b\n"); + let (got, _) = collect("", EXPECT, &[&item], |_| true, |_| None); + assert_eq!(got[0].verdict, Verdict::NotRun(NotRun::Inadmissible)); + assert_eq!(kn(score(&got).recall), (0, 1)); +} + +#[test] +fn falsify_rxr_003_unparsed_is_never_a_pass() { + let rows = vec![ + s("g1", false, Verdict::Unparsed, false), + s("g2", false, Verdict::Unparsed, false), + s("d1", true, Verdict::Unparsed, false), + ]; + let l = score(&rows); + assert_eq!( + kn(l.correct), + (0, 3), + "Unparsed counted as PASS on a good item" + ); + assert_eq!(kn(l.parse_rate), (0, 3)); + assert_eq!(kn(l.recall), (0, 1), "Unparsed is not a FAIL either"); + assert_eq!(kn(l.false_refute), (0, 2)); + let (pairs, dropped) = paired_parsed(&rows, &rows); + assert!( + pairs.is_empty() && dropped == 3, + "Unparsed never enters a paired rate" + ); +} + +/// Items, receipts and raw outputs for [`collect`]. +fn world() -> (Vec, BTreeMap) { + let d = "--- a/crates/x/src/api/router.rs\n+++ b/crates/x/src/api/router.rs\n@@ -1,1 +1,1 @@\n-a\n+b\n"; + let r = crate::corpus::Item::new("R-pr1".into(), Class::R, "t".into(), d); + let g = crate::corpus::Item::new("G-pr2".into(), Class::G, "t".into(), "--- a/g\n+++ b/g\n"); + let raw = BTreeMap::from([ + ( + "raw/R-pr1.txt".to_string(), + "VERDICT: FAIL\n- api/router.rs: off by one".to_string(), + ), + ("raw/G-pr2.txt".to_string(), "VERDICT: PASS".to_string()), + ]); + (vec![r, g], raw) +} + +fn receipt_for( + item: &crate::corpus::Item, + verdict: Verdict, + raw: &BTreeMap, +) -> Receipt { + let mut r = row(&item.id, item.class, verdict); + r.item_sha256.clone_from(&item.diff_sha256); + let path = format!("raw/{}.txt", item.id); + let o = r.output.as_mut().expect("output"); + o.sha256 = sha256_hex(raw[&path].as_bytes()); + o.path = path; + r +} + +fn lines(rs: &[Receipt]) -> String { + rs.iter() + .map(|r| serde_json::to_string(r).expect("json") + "\n") + .collect() +} + +#[test] +fn collect_verifies_raw_output_and_reparses_the_verdict() { + let (items, raw) = world(); + let exp: Vec<&crate::corpus::Item> = items.iter().collect(); + let read = |p: &str| raw.get(p).cloned(); + let good = [ + receipt_for(&items[0], Verdict::Fail, &raw), + receipt_for(&items[1], Verdict::Pass, &raw), + ]; + let (got, rej) = collect(&lines(&good), EXPECT, &exp, |_| true, read); + assert!(rej.is_empty(), "{rej:?}"); + assert_eq!(got[0].verdict, Verdict::Fail); + assert!(got[0].localized, "api/router.rs names the defect file"); + assert_eq!(kn(score(&got).correct), (2, 2)); + + // Each corruption turns exactly the corrupted item into NotRun{Inadmissible}. + let corruptions: Vec<(&str, Box)> = vec![ + ( + "raw sha", + Box::new(|r| r.output.as_mut().expect("o").sha256 = "f".repeat(64)), + ), + ( + "raw missing", + Box::new(|r| r.output.as_mut().expect("o").path = "raw/nope.txt".into()), + ), + ( + "verdict disagrees with raw", + Box::new(|r| r.verdict = Verdict::Pass), + ), + ("item sha", Box::new(|r| r.item_sha256 = "e".repeat(64))), + ( + "unexpected item", + Box::new(|r| r.item_id = "R-pr999".into()), + ), + ]; + for (name, bad) in corruptions { + let mut rs = good.clone(); + bad(&mut rs[0]); + let (got, rej) = collect(&lines(&rs), EXPECT, &exp, |_| true, read); + assert_eq!(rej.len(), 1, "{name}"); + assert_eq!( + got[0].verdict, + Verdict::NotRun(NotRun::Inadmissible), + "{name}" + ); + assert_eq!( + got[1].verdict, + Verdict::Pass, + "{name}: the other item is untouched" + ); + } + + let dup = [good[0].clone(), good[0].clone(), good[1].clone()]; + let (got, _) = collect(&lines(&dup), EXPECT, &exp, |_| true, read); + assert_eq!( + got[0].verdict, + Verdict::NotRun(NotRun::Inadmissible), + "duplicates void the item" + ); + + let (got, rej) = collect(&lines(&good), EXPECT, &exp, |r| r.cell == "C9", read); + assert!( + rej.is_empty() + && got + .iter() + .all(|g| g.verdict == Verdict::NotRun(NotRun::Inadmissible)) + ); + let _ = Loc { + file: String::new(), + line: 0, + }; +} + +#[test] +fn divergence_pairs_and_mcnemar_by_hand() { + let a = fixture(); + let mut b = fixture(); + b[0].output_sha = Some("other".into()); // bytes differ + b[3].verdict = Verdict::Fail; // verdict differs (and output sha stays) + assert_eq!( + divergence(&a, &b), + (2, 2), + "d6/g4 did not run: missing, not agreeing" + ); + + let (pairs, dropped) = paired_parsed(&a, &b); + assert_eq!((pairs.len(), dropped), (6, 4)); + + // Challenger right / champion wrong on 5, the reverse on 1: + // p = P(X ≥ 5 | n = 6, ½) = (6 + 1) / 64. + let champ: Vec = (0..8) + .map(|i| { + s( + &format!("d{i}"), + true, + if i == 5 { Verdict::Fail } else { Verdict::Pass }, + false, + ) + }) + .collect(); + let chall: Vec = (0..8) + .map(|i| { + s( + &format!("d{i}"), + true, + if i < 5 { Verdict::Fail } else { Verdict::Pass }, + false, + ) + }) + .collect(); + assert!((h3_p(&chall, &champ) - 7.0 / 64.0).abs() < 1e-12); + assert_eq!( + correctness_pairs(&[], &champ[..1]), + vec![(false, false)], + "missing challenger is wrong" + ); +} + +#[test] +fn falsify_h1_is_verdict_identity_bytes_are_descriptive() { + let a = fixture(); + let mut b = fixture(); + b[0].output_sha = Some("other".into()); // bytes only + assert_eq!( + verdict_divergence(&a, &b).0, + 0, + "a byte-only diff must not reject H1" + ); + assert_eq!( + divergence(&a, &b).0, + 1, + "H2/descriptive still sees the bytes" + ); + b[3].verdict = Verdict::Fail; + assert_eq!(verdict_divergence(&a, &b).0, 1, "a verdict diff rejects H1"); +} + +#[test] +fn h5_reads_class_r_only_and_h4_needs_three_parsed_lanes() { + let mut a = fixture(); + a[1].class = Class::R; // d2 + let (r, _) = paired_parsed_class(&a, &a, Class::R); + assert_eq!(r.len(), 1, "one parsed class-R item"); + let (p, _) = paired_parsed_class(&a, &a, Class::P); + assert!(p.iter().all(|x| x.defect) && !p.is_empty()); + + let (all, dropped) = paired_parsed(&a, &a); + let (three, dropped3) = vs_voters_parsed(&a, [&a, &a]); + assert_eq!((three.len(), dropped3), (all.len(), dropped)); + let mut agy = fixture(); + agy[0].verdict = Verdict::Unparsed; + let (three, dropped3) = vs_voters_parsed(&a, [&a, &agy]); + assert_eq!((three.len(), dropped3), (all.len() - 1, dropped + 1)); +} + +#[test] +fn error_pairs_mark_wrong_parsed_verdicts() { + let a = fixture(); + let mut b = fixture(); + b[0].verdict = Verdict::Pass; // d1 missed by b + let (ea, eb) = error_pairs(&a, &b); + assert_eq!(ea.len(), eb.len()); + assert_eq!( + eb.iter().filter(|e| **e).count(), + ea.iter().filter(|e| **e).count() + 1 + ); +} diff --git a/crates/aprender-review-experiment/src/secret.rs b/crates/aprender-review-experiment/src/secret.rs new file mode 100644 index 0000000000..fabb811d9e --- /dev/null +++ b/crates/aprender-review-experiment/src/secret.rs @@ -0,0 +1,530 @@ +//! Secret scan for trace admission (contract `trace-admission-secret-v1`). +//! +//! Two independent scanners, and a row is a hit when EITHER fires: +//! +//! - **builtin**: hand-written detectors (AWS key id / secret, GitHub token, +//! PEM private key, Anthropic key, Slack token, JWT, high-entropy assignment). +//! It honours no allowlist, so the AWS documentation key `AKIA…EXAMPLE`, which +//! gitleaks allowlists, is still caught here. +//! - **gitleaks**: the upstream gitleaks v8.30.1 ruleset, vendored verbatim as +//! data (`data/gitleaks/`, MIT) and interpreted in-process: keyword prefilter, +//! regex, secret group, entropy floor, per-rule and global allowlists. +//! +//! Where gitleaks keys a rule or allowlist on a file PATH, the scan has none, so +//! it fails safe: a path-scoped rule runs on all content, and a path-conditioned +//! allowlist allows nothing. A rule with no regex (path-only) cannot fire on +//! content and is listed in [`PATH_ONLY_RULES`]. + +use std::sync::OnceLock; + +use regex::{Regex, RegexBuilder}; +use serde::Deserialize; + +/// The vendored ruleset, byte-for-byte as released upstream. +pub const GITLEAKS_TOML: &str = include_str!("../data/gitleaks/gitleaks-v8.30.1.toml"); +/// sha256 of [`GITLEAKS_TOML`]; a test pins it. +pub const GITLEAKS_SHA256: &str = + "e163e53b9e7e8a8511e77271e2b323ed057759542a6d988258afe3a1fa329caf"; +/// The scanner versions a `secret_scan` record carries. +pub const SCANNER_VERSIONS: &[&str] = &["builtin-1", "gitleaks-v8.30.1"]; +/// gitleaks rules that match on a path alone and so never fire on content. +pub const PATH_ONLY_RULES: &[&str] = &["pkcs12-file"]; + +/// Which scanner fired. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub enum Scanner { + Builtin, + Gitleaks, +} + +/// One rule that fired on the scanned text. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub struct Hit { + pub scanner: Scanner, + pub rule: &'static str, +} + +/// Every (scanner, rule) hit in `text` or in any JSON string value on one of +/// its lines (an escape such as `\u0041KIA…` cannot hide a key), sorted, each +/// at most once. +#[must_use] +pub fn scan(text: &str) -> Vec { + scan_without(text, &[]) +} + +/// [`scan`] with the named rules switched off: the ablation the recall +/// falsifier uses to show every rule is load-bearing for its canary. +#[must_use] +pub fn scan_without(text: &str, disabled: &[Hit]) -> Vec { + let texts = texts(text); + let on = |h: &Hit| !disabled.contains(h); + let mut hits: Vec = BUILTIN + .iter() + .filter_map(|&(rule, found)| { + let hit = Hit { + scanner: Scanner::Builtin, + rule, + }; + (on(&hit) && texts.iter().any(|t| found(t))).then_some(hit) + }) + .collect(); + let gl = gitleaks(); + for rule in &gl.rules { + let hit = Hit { + scanner: Scanner::Gitleaks, + rule: rule.id, + }; + if on(&hit) && texts.iter().any(|t| rule.fires(t, &gl.global)) { + hits.push(hit); + } + } + hits.sort(); + hits.dedup(); + hits +} + +/// The raw text plus every JSON string value found on its lines. +fn texts(text: &str) -> Vec { + let mut out = vec![text.to_string()]; + for line in text.lines() { + if let Ok(v) = serde_json::from_str::(line) { + collect_strings(&v, &mut out); + } + } + out +} + +fn collect_strings(v: &serde_json::Value, out: &mut Vec) { + match v { + serde_json::Value::String(s) => out.push(s.clone()), + serde_json::Value::Array(a) => a.iter().for_each(|x| collect_strings(x, out)), + serde_json::Value::Object(o) => o.values().for_each(|x| collect_strings(x, out)), + _ => {} + } +} + +/// Shannon entropy of `s` in bits per byte. +#[must_use] +pub fn entropy(s: &str) -> f64 { + let mut counts = [0usize; 256]; + s.bytes().for_each(|b| counts[usize::from(b)] += 1); + let n = s.len() as f64; + counts + .iter() + .filter(|&&c| c > 0) + .map(|&c| { + let p = c as f64 / n; + -p * p.log2() + }) + .sum() +} + +// --------------------------------------------------------------------------- +// builtin +// --------------------------------------------------------------------------- + +type Detector = fn(&str) -> bool; + +/// The builtin detectors, by rule name. +pub const BUILTIN: &[(&str, Detector)] = &[ + ("aws-access-key-id", aws_access_key_id), + ("aws-secret-access-key", aws_secret_access_key), + ("github-token", github_token), + ("private-key", private_key), + ("anthropic-key", anthropic_key), + ("slack-token", slack_token), + ("jwt", jwt), + ("high-entropy-assignment", high_entropy_assignment), +]; + +fn is_word(c: u8) -> bool { + c.is_ascii_alphanumeric() || c == b'_' +} + +/// Byte offsets just past each occurrence of `prefix` that does not start +/// inside a longer word. +fn after<'a>(text: &'a str, prefix: &'a str) -> impl Iterator + 'a { + let b = text.as_bytes(); + text.match_indices(prefix) + .filter(move |(i, _)| *i == 0 || !is_word(b[i - 1])) + .map(move |(i, _)| i + prefix.len()) +} + +/// Length of the run at `from` whose bytes satisfy `ok`. +fn run(text: &str, from: usize, ok: impl Fn(u8) -> bool) -> usize { + text.as_bytes()[from..] + .iter() + .take_while(|&&c| ok(c)) + .count() +} + +/// `AKIA`/`ASIA`/`ABIA`/`ACCA` + exactly 16 of `[A-Z0-9]`, as a whole word. +fn aws_access_key_id(text: &str) -> bool { + let key = |c: u8| c.is_ascii_uppercase() || c.is_ascii_digit(); + ["AKIA", "ASIA", "ABIA", "ACCA"] + .iter() + .any(|p| after(text, p).any(|i| run(text, i, is_word) == 16 && run(text, i, key) == 16)) +} + +/// `aws_secret_access_key` then `=`/`:`/quotes/space, then a 40-char key. +fn aws_secret_access_key(text: &str) -> bool { + let lower = text.to_ascii_lowercase(); + let key = |c: u8| c.is_ascii_alphanumeric() || c == b'/' || c == b'+'; + let found = after(&lower, "aws_secret_access_key").any(|i| { + let j = i + run(text, i, |c| { + matches!(c, b'=' | b':' | b'"' | b'\'' | b' ' | b'\t') + }); + j > i && run(text, j, key) == 40 + }); + found +} + +/// Classic `gh[pousr]_` + 36 alphanumerics, or a fine-grained `github_pat_`. +fn github_token(text: &str) -> bool { + ["ghp_", "gho_", "ghu_", "ghs_", "ghr_"] + .iter() + .any(|p| after(text, p).any(|i| run(text, i, |c| c.is_ascii_alphanumeric()) == 36)) + || after(text, "github_pat_").any(|i| run(text, i, is_word) >= 22) +} + +/// A PEM `-----BEGIN … PRIVATE KEY-----` armour line. +fn private_key(text: &str) -> bool { + text.match_indices("-----BEGIN ").any(|(i, m)| { + let rest = &text[i + m.len()..]; + rest.find("-----") + .is_some_and(|end| rest[..end].ends_with("PRIVATE KEY") && !rest[..end].contains('\n')) + }) +} + +fn anthropic_key(text: &str) -> bool { + after(text, "sk-ant-").any(|i| run(text, i, |c| is_word(c) || c == b'-') >= 20) +} + +fn slack_token(text: &str) -> bool { + ["xoxb-", "xoxa-", "xoxp-", "xoxr-", "xoxs-"] + .iter() + .any(|p| { + after(text, p).any(|i| run(text, i, |c| c.is_ascii_alphanumeric() || c == b'-') >= 10) + }) +} + +/// `eyJ.eyJ.`: a JWT whose header and payload are both +/// JSON objects, with a signature of at least 16 characters. +fn jwt(text: &str) -> bool { + let b64 = |c: u8| c.is_ascii_alphanumeric() || c == b'-' || c == b'_'; + after(text, "eyJ").any(|i| { + let h = run(text, i, b64); + let p = i + h; + if h < 10 || !text[p..].starts_with(".eyJ") { + return false; + } + let q = p + 4; + let body = run(text, q, b64); + let s = q + body; + body >= 10 && text[s..].starts_with('.') && run(text, s + 1, b64) >= 16 + }) +} + +/// `secret`/`token`/`password`/`api_key`/… assigned a value of at least 24 +/// characters whose entropy exceeds hex's 4 bits/char: random material, not a +/// hash, a path, or a call such as `token = self.next()`. +fn high_entropy_assignment(text: &str) -> bool { + const NAMES: &[&str] = &[ + "secret", + "token", + "password", + "passwd", + "api_key", + "apikey", + "access_key", + "private_key", + "client_secret", + ]; + let lower = text.to_ascii_lowercase(); + let value = |c: u8| c.is_ascii_alphanumeric() || matches!(c, b'+' | b'/' | b'_' | b'-' | b'='); + let found = NAMES.iter().any(|name| { + lower.match_indices(name).any(|(i, m)| { + let k = i + m.len() + run(&lower, i + m.len(), is_word); + let sep = run(text, k, |c| { + matches!(c, b'=' | b':' | b'"' | b'\'' | b' ' | b'\t' | b'>') + }); + if sep == 0 || !text[k..k + sep].contains(['=', ':']) { + return false; + } + let v = k + sep; + let len = run(text, v, value); + len >= 24 && entropy(&text[v..v + len]) > 4.2 + }) + }); + found +} + +// --------------------------------------------------------------------------- +// gitleaks as data +// --------------------------------------------------------------------------- + +#[derive(Deserialize)] +struct ConfigDef { + #[serde(default)] + allowlist: Option, + #[serde(default)] + rules: Vec, +} + +#[derive(Deserialize)] +#[serde(rename_all = "camelCase")] +struct RuleDef { + id: String, + regex: Option, + secret_group: Option, + entropy: Option, + #[serde(default)] + keywords: Vec, + #[serde(default)] + allowlists: Vec, +} + +#[derive(Deserialize)] +#[serde(rename_all = "camelCase")] +struct AllowDef { + #[serde(default)] + regexes: Vec, + #[serde(default)] + stopwords: Vec, + #[serde(default)] + paths: Vec, + regex_target: Option, + condition: Option, +} + +#[derive(Clone, Copy, PartialEq, Eq)] +enum Target { + Secret, + Match, + Line, +} + +struct Allow { + regexes: Vec, + stopwords: Vec, + target: Target, + /// `condition = "AND"` with paths: a path we do not have is a condition we + /// cannot meet, so this allowlist allows nothing. + inert: bool, +} + +impl Allow { + fn allows(&self, secret: &str, whole: &str, line: &str) -> bool { + if self.inert { + return false; + } + let target = match self.target { + Target::Secret => secret, + Target::Match => whole, + Target::Line => line, + }; + let lower = secret.to_lowercase(); + self.regexes.iter().any(|r| r.is_match(target)) + || self.stopwords.iter().any(|w| lower.contains(w.as_str())) + } +} + +/// One compiled gitleaks rule. +pub struct Rule { + pub id: &'static str, + regex: Regex, + secret_group: Option, + entropy: Option, + keywords: Vec, + allows: Vec, +} + +impl Rule { + fn fires(&self, text: &str, global: &Allow) -> bool { + if !self.keywords.is_empty() { + let lower = text.to_lowercase(); + if !self.keywords.iter().any(|k| lower.contains(k.as_str())) { + return false; + } + } + self.regex.captures_iter(text).any(|caps| { + let whole = caps.get(0).map_or("", |m| m.as_str()); + let secret = self + .secret_group + .and_then(|g| caps.get(g)) + .or_else(|| { + caps.iter() + .skip(1) + .flatten() + .find(|m| !m.as_str().is_empty()) + }) + .map_or(whole, |m| m.as_str()); + if let Some(floor) = self.entropy { + if entropy(secret) <= floor { + return false; + } + if self.id.starts_with("generic") && !secret.bytes().any(|b| b.is_ascii_digit()) { + return false; + } + } + let start = caps.get(0).map_or(0, |m| m.start()); + let end = caps.get(0).map_or(0, |m| m.end()); + let ls = text[..start].rfind('\n').map_or(0, |i| i + 1); + let le = text[end..].find('\n').map_or(text.len(), |i| end + i); + let line = &text[ls..le]; + !global.allows(secret, whole, line) + && !self.allows.iter().any(|a| a.allows(secret, whole, line)) + }) + } +} + +/// The compiled ruleset. +pub struct Gitleaks { + pub rules: Vec, + global: Allow, + /// Rules whose regex failed to compile: must be empty (a test asserts it). + pub failed: Vec<(String, String)>, +} + +/// The vendored ruleset, compiled once. +/// +/// # Panics +/// The vendored TOML does not parse; the sha-pin test catches that first. +pub fn gitleaks() -> &'static Gitleaks { + static GL: OnceLock = OnceLock::new(); + GL.get_or_init(|| compile(GITLEAKS_TOML).expect("vendored gitleaks toml parses")) +} + +fn build(pattern: &str) -> Result { + RegexBuilder::new(&go_to_rust(pattern)) + .size_limit(1 << 28) + .dfa_size_limit(1 << 28) + .build() +} + +fn allow(def: AllowDef, failed: &mut Vec<(String, String)>, owner: &str) -> Allow { + let target = match def.regex_target.as_deref() { + Some("match") => Target::Match, + Some("line") => Target::Line, + _ => Target::Secret, + }; + let and = def.condition.as_deref() == Some("AND"); + let regexes = def + .regexes + .iter() + .filter_map(|p| { + build(p) + .map_err(|e| failed.push((format!("{owner} allowlist"), e.to_string()))) + .ok() + }) + .collect(); + Allow { + regexes, + stopwords: def.stopwords.iter().map(|w| w.to_lowercase()).collect(), + target, + inert: and && !def.paths.is_empty(), + } +} + +fn compile(toml_text: &str) -> Result { + let def: ConfigDef = toml::from_str(toml_text)?; + let mut failed = Vec::new(); + let global = def.allowlist.map_or( + Allow { + regexes: Vec::new(), + stopwords: Vec::new(), + target: Target::Secret, + inert: true, + }, + |a| allow(a, &mut failed, "global"), + ); + let mut rules = Vec::new(); + for r in def.rules { + let Some(pattern) = r.regex else { continue }; + let id: &'static str = Box::leak(r.id.into_boxed_str()); + match build(&pattern) { + Ok(regex) => rules.push(Rule { + id, + regex, + secret_group: r.secret_group, + entropy: r.entropy, + keywords: r.keywords.iter().map(|k| k.to_lowercase()).collect(), + allows: r + .allowlists + .into_iter() + .map(|a| allow(a, &mut failed, id)) + .collect(), + }), + Err(e) => failed.push((id.to_string(), e.to_string())), + } + } + Ok(Gitleaks { + rules, + global, + failed, + }) +} + +/// Go RE2 takes a `{` that does not open a valid repetition as a literal; Rust +/// rejects it. Escape those, outside character classes; nothing else differs +/// for this ruleset (the compile-all test is the proof). +fn go_to_rust(p: &str) -> String { + let b = p.as_bytes(); + let mut out = String::with_capacity(p.len() + 8); + let (mut i, mut class, mut prev_atom) = (0, false, false); + while i < b.len() { + let c = b[i]; + if c == b'\\' && i + 1 < b.len() { + let n = p[i + 1..].chars().next().map_or(1, char::len_utf8); + out.push_str(&p[i..i + 1 + n]); + i += 1 + n; + prev_atom = true; + continue; + } + if class { + class = c != b']'; + } else if c == b'[' { + class = true; + if b.get(i + 1) == Some(&b'^') { + out.push_str("[^"); + i += 2; + if b.get(i) == Some(&b']') { + out.push(']'); + i += 1; + } + continue; + } + if b.get(i + 1) == Some(&b']') { + out.push_str("[]"); + i += 2; + continue; + } + } else if c == b'{' && !(prev_atom && is_repetition(&p[i..])) { + out.push_str("\\{"); + i += 1; + prev_atom = true; + continue; + } + prev_atom = class || !matches!(c, b'(' | b'|'); + let n = p[i..].chars().next().map_or(1, char::len_utf8); + out.push_str(&p[i..i + n]); + i += n; + } + out +} + +/// `{n}`, `{n,}` or `{n,m}` at the start of `s`. +fn is_repetition(s: &str) -> bool { + let Some(end) = s.find('}') else { + return false; + }; + let inner = &s[1..end]; + let (lo, hi) = inner.split_once(',').unwrap_or((inner, "0")); + !lo.is_empty() + && lo.bytes().all(|c| c.is_ascii_digit()) + && hi.bytes().all(|c| c.is_ascii_digit()) +} + +#[cfg(test)] +#[path = "secret_tests.rs"] +mod tests; diff --git a/crates/aprender-review-experiment/src/secret_tests.rs b/crates/aprender-review-experiment/src/secret_tests.rs new file mode 100644 index 0000000000..c45689cd51 --- /dev/null +++ b/crates/aprender-review-experiment/src/secret_tests.rs @@ -0,0 +1,359 @@ +//! Falsifiers for `trace-admission-secret-v1` (PRA-001 T6, G-SEC / G-SEC-FN). +//! +//! Every canary is BUILT at run time, from pieces or from a seeded generator, +//! so no key-shaped literal sits in the source for a repo scanner to flag. + +// `json!` expands to an `unwrap` of an infallible `to_value`. +#![allow(clippy::disallowed_methods)] + +use super::*; +use sha2::{Digest, Sha256}; + +/// A splitmix64 stream: the same seed yields the same canaries on every run. +struct Gen(u64); + +impl Gen { + fn next(&mut self) -> u64 { + self.0 = self.0.wrapping_add(0x9E37_79B9_7F4A_7C15); + let mut z = self.0; + z = (z ^ (z >> 30)).wrapping_mul(0xBF58_476D_1CE4_E5B9); + z = (z ^ (z >> 27)).wrapping_mul(0x94D0_49BB_1331_11EB); + z ^ (z >> 31) + } + + /// `n` characters drawn from `alphabet`. + fn take(&mut self, alphabet: &str, n: usize) -> String { + let a = alphabet.as_bytes(); + (0..n) + .map(|_| char::from(a[(self.next() % a.len() as u64) as usize])) + .collect() + } +} + +const UPPER: &str = "ABCDEFGHIJKLMNOPQRSTUVWXYZ"; +const B32: &str = "ABCDEFGHIJKLMNOPQRSTUVWXYZ234567"; +const ALNUM: &str = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"; +const LOWER_NUM: &str = "abcdefghijklmnopqrstuvwxyz0123456789"; +const HEX: &str = "0123456789abcdef"; +const DIGITS: &str = "0123456789"; + +fn b64url() -> String { + [ALNUM, "-_"].concat() +} + +fn b64() -> String { + [ALNUM, "+/"].concat() +} + +fn hit(scanner: Scanner, rule: &'static str) -> Hit { + Hit { scanner, rule } +} + +/// The AWS documentation key: gitleaks allowlists it (`.+EXAMPLE$`), which is +/// exactly why the builtin scanner must catch it on its own. +fn planted_aws_key() -> String { + ["AKIA", "IOSFODNN7", "EXAMPLE"].concat() +} + +/// The 20 canaries, each labelled with the ONE rule it is planted for. +fn canaries() -> Vec<(Hit, String)> { + use Scanner::{Builtin as B, Gitleaks as G}; + let mut g = Gen(0x5EC0_0001); + let pem = |g: &mut Gen, kind: &str| { + let body = (0..4) + .map(|_| g.take(&b64(), 64)) + .collect::>() + .join("\n"); + format!("-----BEGIN {kind}PRIVATE KEY-----\n{body}\n-----END {kind}PRIVATE KEY-----\n") + }; + let jwt = |g: &mut Gen| { + format!( + "Authorization: Bearer eyJ{}.eyJ{}.{}\n", + g.take(ALNUM, 30), + g.take(ALNUM, 60), + g.take(&b64url(), 43) + ) + }; + vec![ + // builtin + ( + hit(B, "aws-access-key-id"), + format!("export AWS_ACCESS_KEY_ID={}", planted_aws_key()), + ), + ( + hit(B, "aws-secret-access-key"), + format!( + "aws configure set aws_secret_access_key \"{}\"", + g.take(&b64(), 40) + ), + ), + ( + hit(B, "github-token"), + ["gho_", &g.take(ALNUM, 36)].concat(), + ), + (hit(B, "private-key"), pem(&mut g, "OPENSSH ")), + ( + hit(B, "anthropic-key"), + ["sk-ant-", "admin01-", &g.take(ALNUM, 40)].concat(), + ), + ( + hit(B, "slack-token"), + ["xoxp-", &g.take(DIGITS, 11), "-", &g.take(ALNUM, 24)].concat(), + ), + (hit(B, "jwt"), jwt(&mut g)), + ( + hit(B, "high-entropy-assignment"), + format!("client_secret: \"{}\"", g.take(ALNUM, 32)), + ), + // gitleaks + ( + hit(G, "aws-access-token"), + format!("key {}{} ", "ASIA", g.take(B32, 16)), + ), + (hit(G, "github-pat"), ["ghp_", &g.take(ALNUM, 36)].concat()), + ( + hit(G, "github-fine-grained-pat"), + ["github_pat_", &g.take(ALNUM, 82)].concat(), + ), + (hit(G, "private-key"), pem(&mut g, "")), + (hit(G, "jwt"), jwt(&mut g)), + ( + hit(G, "generic-api-key"), + format!("api_token = \"{}{}\"", g.take(ALNUM, 30), g.take(DIGITS, 2)), + ), + ( + hit(G, "slack-bot-token"), + [ + "xoxb-", + &g.take(DIGITS, 12), + "-", + &g.take(DIGITS, 12), + "-", + &g.take(ALNUM, 24), + ] + .concat(), + ), + ( + hit(G, "stripe-access-token"), + format!("{}{} ", "sk_live_", g.take(ALNUM, 24)), + ), + ( + hit(G, "gitlab-pat"), + ["glpat-", &g.take(ALNUM, 20)].concat(), + ), + ( + hit(G, "npm-access-token"), + format!("{}{} ", "npm_", g.take(LOWER_NUM, 36)), + ), + ( + hit(G, "gcp-api-key"), + format!("{}{} ", "AIza", g.take(&b64url(), 35)), + ), + ( + hit(G, "digitalocean-pat"), + format!("{}{} ", "dop_v1_", g.take(HEX, 64)), + ), + ] +} + +/// The canary set covers what G-SEC names: AWS, GitHub PAT, private key, JWT, +/// high-entropy generic; 20 of them, each labelled with a distinct rule. +#[test] +fn canary_set_is_twenty_distinct_labelled_rules() { + let c = canaries(); + assert_eq!(c.len(), 20); + let mut labels: Vec = c.iter().map(|(h, _)| *h).collect(); + labels.sort(); + labels.dedup(); + assert_eq!(labels.len(), 20, "each canary names a distinct rule"); + for (h, _) in &c { + match h.scanner { + Scanner::Builtin => assert!(BUILTIN.iter().any(|(r, _)| r == &h.rule)), + Scanner::Gitleaks => assert!(gitleaks().rules.iter().any(|r| r.id == h.rule)), + } + } + assert!(c[0].1.contains(&planted_aws_key())); +} + +/// FALSIFY-TAS-001 (G-SEC): 100% canary recall. Each canary fires the rule it is +/// labelled with, bare and inside a captured JSONL row. +#[test] +fn falsify_tas_001_every_canary_fires_its_labelled_rule() { + for (label, text) in canaries() { + let row = serde_json::json!({"lane": "haiku-4-5", "output": text}).to_string(); + for t in [&text, &row] { + assert!(scan(t).contains(&label), "{label:?} missed: {t}"); + } + } +} + +/// FALSIFY-TAS-002 (G-SEC-FN): remove one rule and the canary planted for it +/// goes RED: its scanner then reports NOTHING on it, so no other rule of that +/// scanner is quietly carrying the recall check above. +#[test] +fn falsify_tas_002_removing_a_rule_turns_its_canary_red() { + for (label, text) in canaries() { + let rest: Vec = scan_without(&text, &[label]) + .into_iter() + .filter(|h| h.scanner == label.scanner) + .collect(); + assert_eq!( + rest, + Vec::::new(), + "{label:?} with its rule removed: {text}" + ); + } +} + +/// FALSIFY-TAS-003: the AWS documentation key is allowlisted by gitleaks and +/// caught by the builtin scanner; the union quarantines it. Removing the +/// builtin rule leaves NOTHING to catch it: the union is load-bearing. +#[test] +fn falsify_tas_003_aws_doc_key_needs_the_builtin_scanner() { + let text = format!("export AWS_ACCESS_KEY_ID={}", planted_aws_key()); + let aws = hit(Scanner::Builtin, "aws-access-key-id"); + assert_eq!(scan(&text), vec![aws]); + assert!(scan_without(&text, &[aws]).is_empty()); +} + +/// The vendored ruleset is the released v8.30.1 file, byte for byte. +#[test] +fn vendored_gitleaks_toml_is_sha_pinned() { + let got = format!("{:x}", Sha256::digest(GITLEAKS_TOML.as_bytes())); + assert_eq!(got, GITLEAKS_SHA256); +} + +/// Every gitleaks rule with a regex compiles, and the only rules without one +/// are the listed path-only rules: no rule is silently dropped. +#[test] +fn every_gitleaks_rule_compiles() { + let gl = gitleaks(); + assert!(gl.failed.is_empty(), "failed to compile: {:?}", gl.failed); + let declared = GITLEAKS_TOML + .lines() + .filter(|l| l.trim() == "[[rules]]") + .count(); + assert_eq!(declared, 222); + assert_eq!(gl.rules.len() + PATH_ONLY_RULES.len(), declared); + for id in PATH_ONLY_RULES { + assert!(gl.rules.iter().all(|r| r.id != *id)); + assert!(GITLEAKS_TOML.contains(&format!("id = \"{id}\""))); + } +} + +/// Go's literal `{` becomes an escaped one; real repetitions are untouched. +#[test] +fn go_regex_translation_case_table() { + for (go, rust) in [ + (r"^\$(?:\d+|{\d+})$", r"^\$(?:\d+|\{\d+})$"), + (r"a{2,5}", r"a{2,5}"), + (r"a{2}", r"a{2}"), + (r"[{]x", r"[{]x"), + (r"{{x}}", r"\{\{x}}"), + (r"a{,5}", r"a\{,5}"), + (r"\{\d{0,2}}", r"\{\d{0,2}}"), + ] { + assert_eq!(go_to_rust(go), rust, "{go}"); + assert!(Regex::new(&go_to_rust(go)).is_ok(), "{go}"); + } +} + +/// Builtin case table: must-match rows fire exactly their rule; must-not-match +/// rows fire nothing in the builtin scanner. +#[test] +fn builtin_case_table() { + let builtin = |t: &str| -> Vec<&'static str> { + scan(t) + .into_iter() + .filter(|h| h.scanner == Scanner::Builtin) + .map(|h| h.rule) + .collect() + }; + let hits = [ + (planted_aws_key(), "aws-access-key-id"), + (["ASIA", "Y34FZKBOKMUTVV7A"].concat(), "aws-access-key-id"), + (["ghp_", &"a".repeat(36)].concat(), "github-token"), + (["github_pat_", &"B".repeat(40)].concat(), "github-token"), + ( + ["-----BEGIN RSA ", "PRIVATE KEY-----"].concat(), + "private-key", + ), + (["-----BEGIN ", "PRIVATE KEY-----"].concat(), "private-key"), + ( + ["sk-ant-", "api03-", &"x".repeat(40)].concat(), + "anthropic-key", + ), + ( + ["xoxb-", "123456789012-", &"q".repeat(24)].concat(), + "slack-token", + ), + ]; + let escaped = format!("{{\"output\": \"\\u0041{}\"}}", &planted_aws_key()[1..]); + assert!( + !escaped.contains("AKIA"), + "the fixture must hide the literal" + ); + assert_eq!(builtin(&escaped), vec!["aws-access-key-id"]); + for (text, kind) in &hits { + assert_eq!(builtin(text), vec![*kind], "must match: {text}"); + } + let misses = [ + "AKIA".to_string(), + ["AKIA", "iosfodnn7example"].concat(), + ["XAKIA", "IOSFODNN7EXAMPLE"].concat(), + ["AKIA", "IOSFODNN7EXAMPLEX"].concat(), + "ghp_short".to_string(), + "-----BEGIN PUBLIC KEY-----".to_string(), + "the aws_secret_access_key field must be set".to_string(), + "sk-ant is the Anthropic key prefix".to_string(), + "eyJhbGciOiJIUzI1NiJ9 alone is a header".to_string(), + format!("token = {}", "0123456789abcdef".repeat(4)), + ]; + for text in &misses { + assert!(builtin(text).is_empty(), "must not match: {text}"); + } +} + +/// Ordinary review traffic fires neither scanner: quarantine is for secrets, +/// not for every row that mentions a token. +#[test] +fn ordinary_review_rows_are_clean() { + let upper = Gen(7).take(UPPER, 8); + for text in [ + "LGTM: the bound check is inclusive now.".to_string(), + "let token = lexer.next_token()?;".to_string(), + "+ password: String,\n- passwd: &'static str,".to_string(), + format!("sha256 {}", "0123456789abcdef".repeat(4)), + "fn secret_scan(rows: &str) -> Vec {".to_string(), + format!("see crates/aprender-serve/src/api/{upper}.rs"), + "--- a/src/lib.rs\n+++ b/src/lib.rs\n@@ -1,3 +1,3 @@\n-x\n+y".to_string(), + ] { + assert!(scan(&text).is_empty(), "{:?} fired on: {text}", scan(&text)); + } +} + +/// gitleaks semantics the canaries alone do not pin: the entropy floor drops a +/// low-entropy match, and an allowlist conditioned (AND) on a path we do not +/// have allows nothing, so the scan fails safe. +#[test] +fn gitleaks_semantics_case_table() { + let gl_rules = |t: &str| -> Vec<&'static str> { + scan(t) + .into_iter() + .filter(|h| h.scanner == Scanner::Gitleaks) + .map(|h| h.rule) + .collect() + }; + // `ghp_aaaa…` matches github-pat's regex but not its entropy floor of 3. + let flat = ["ghp_", &"a".repeat(36)].concat(); + assert!(!gl_rules(&flat).contains(&"github-pat"), "{flat}"); + // generic-api-key's `SRC…="…"` allowlist applies only to BitBake paths + // (condition AND); with no path it must not suppress the hit. + let mut g = Gen(0x5EC0_0002); + let bitbake = format!( + "SRC_API_TOKEN = \"{}{}\"", + g.take(ALNUM, 30), + g.take(DIGITS, 2) + ); + assert!(gl_rules(&bitbake).contains(&"generic-api-key"), "{bitbake}"); +} diff --git a/crates/aprender-review-experiment/src/sparse_logits.rs b/crates/aprender-review-experiment/src/sparse_logits.rs new file mode 100644 index 0000000000..1dfee0e549 --- /dev/null +++ b/crates/aprender-review-experiment/src/sparse_logits.rs @@ -0,0 +1,345 @@ +//! PRA-001 T11: the `sparse-logits-v1` blob (spec PRA-001 §2.7). +//! +//! Per output token: the sampled id, the top-k ids with their log-softmax at +//! temperature 1.0 (before any sampling filter), the full-vocab logsumexp, and +//! the retained mass `M = Σ exp(logprobs) ≤ 1`. Storing `M` is the point: a +//! top-k cache without its residual mass is a biased teacher estimate, and +//! the tail-bucket correction of a sparse forward-KL objective needs it. +//! +//! Layout: `MAGIC`, a u32-LE header length, the header JSON, then one zstd +//! frame holding the columns (all LE): sampled ids `n×u32`, ids `n·k×u32`, +//! logprobs `n·k×f16`, logsumexp `n×f32`, mass `n×f16`. The decoder is +//! strict: an unknown header key, a short column, a trailing byte, or a mass +//! that disagrees with its own logprobs is an error, never a best effort. + +use half::f16; +use serde::{Deserialize, Serialize}; + +pub const SCHEME: &str = "sparse-logits-v1"; +/// Magic prefix; the trailing newline keeps `head -c` output readable. +pub const MAGIC: &[u8; 8] = b"SPLOGV1\n"; +/// PRA-001 §2.7 default `k` `[A]`. +pub const DEFAULT_K: u8 = 20; +/// zstd level for the column frame. +const ZSTD_LEVEL: i32 = 3; +/// f16 has an 11-bit significand: Σ of k rounded exps can drift this much +/// from the rounded mass. Anything larger is a producer defect. +const MASS_TOLERANCE: f32 = 1.0 / 256.0; + +/// How the token set was chosen. `RsSample` is reserved (§2.7) so the 27B +/// teacher can write importance-sampled sets later without a schema change. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub enum Mode { + Topk, + RsSample, +} + +/// Per-blob header (§2.7). +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Header { + pub schema: String, + pub mode: Mode, + pub model_sha256: String, + pub tokenizer_sha256: String, + pub apr_tag: String, + /// The backend that actually served the row (`served_by`). + pub backend: String, + /// Sampling temperature of the recorded run. Logprobs are always T = 1.0. + pub temperature_of_record: f32, + pub k: u8, + pub n_tokens: u32, +} + +/// One output token's sparse distribution. +#[derive(Debug, Clone, PartialEq)] +pub struct TokenLogits { + pub token_id_sampled: u32, + /// Top-k ids, descending by logprob. u32: Qwen's vocab exceeds 2^16. + pub ids: Vec, + /// log-softmax at T = 1.0, same order as `ids`. + pub logprobs: Vec, + pub logsumexp_full: f32, + /// Retained mass `M = Σ exp(logprobs)`. + pub topk_mass: f16, +} + +impl TokenLogits { + /// Build from a full-vocab logit row at T = 1.0. `None` when `k` is 0, + /// exceeds the vocab, or a logit is not finite. + #[must_use] + pub fn from_full_logits(logits: &[f32], k: u8, token_id_sampled: u32) -> Option { + let k = usize::from(k); + if k == 0 || k > logits.len() || logits.iter().any(|x| !x.is_finite()) { + return None; + } + let max = logits.iter().copied().fold(f32::NEG_INFINITY, f32::max); + let sum: f64 = logits.iter().map(|&x| f64::from(x - max).exp()).sum(); + let lse = f64::from(max) + sum.ln(); + let mut order: Vec = (0..logits.len()).collect(); + // Descending logit, ascending id on ties: deterministic. + order.sort_by(|&a, &b| logits[b].total_cmp(&logits[a]).then(a.cmp(&b))); + order.truncate(k); + let lp: Vec = order.iter().map(|&i| f64::from(logits[i]) - lse).collect(); + let mass: f64 = lp.iter().map(|x| x.exp()).sum(); + Some(Self { + token_id_sampled, + ids: order + .iter() + .map(|&i| u32::try_from(i).ok()) + .collect::>()?, + logprobs: lp.iter().map(|&x| f16::from_f64(x)).collect(), + logsumexp_full: lse as f32, + topk_mass: f16::from_f64(mass.min(1.0)), + }) + } + + /// Residual (tail) mass `1 − M`, clamped at 0. + #[must_use] + pub fn residual_mass(&self) -> f32 { + (1.0 - self.topk_mass.to_f32()).max(0.0) + } + + fn check(&self, k: usize, at: usize) -> Result<(), String> { + if self.ids.len() != k || self.logprobs.len() != k { + return Err(format!( + "token {at}: {} ids / {} logprobs, header k = {k}", + self.ids.len(), + self.logprobs.len() + )); + } + if !self.logsumexp_full.is_finite() { + return Err(format!("token {at}: logsumexp_full is not finite")); + } + let mut sum = 0.0_f32; + for lp in &self.logprobs { + let v = lp.to_f32(); + if !v.is_finite() || v > 0.0 { + return Err(format!( + "token {at}: logprob {v} is not a finite log-probability" + )); + } + sum += v.exp(); + } + let m = self.topk_mass.to_f32(); + if !(0.0..=1.0 + MASS_TOLERANCE).contains(&m) { + return Err(format!("token {at}: topk_mass {m} outside [0, 1]")); + } + if (sum - m).abs() > MASS_TOLERANCE { + return Err(format!("token {at}: topk_mass {m} != Σexp(logprobs) {sum}")); + } + Ok(()) + } +} + +/// PRM C11 capture: the `choices[0].logprobs.content` of an `apr serve` +/// `/v1/chat/completions` body (#4026, `top_logprobs = k`) as one +/// [`TokenLogits`] per output token, ready for [`encode`]. +/// +/// Refused, never guessed: a body without `logsumexp_full` (a hosted or +/// foreign server — its tail mass is unknowable, and hosted output never +/// enters the corpus), fewer than `k` alternatives, an entry without +/// `token_id`, or logprobs that are not a descending, finite log-softmax. +pub fn from_chat_completion(body: &serde_json::Value, k: u8) -> Result, String> { + let content = body + .pointer("/choices/0/logprobs/content") + .and_then(serde_json::Value::as_array) + .ok_or("no choices[0].logprobs.content: the request did not ask for logprobs")?; + content + .iter() + .enumerate() + .map(|(at, entry)| { + token_from_serve_entry(entry, usize::from(k)).map_err(|e| format!("token {at}: {e}")) + }) + .collect() +} + +fn serve_id(v: &serde_json::Value) -> Result { + v.get("token_id") + .and_then(serde_json::Value::as_u64) + .and_then(|id| u32::try_from(id).ok()) + .ok_or_else(|| "no u32 token_id".to_string()) +} + +fn token_from_serve_entry(entry: &serde_json::Value, k: usize) -> Result { + if k == 0 { + return Err("k = 0".into()); + } + let lse = entry + .get("logsumexp_full") + .and_then(serde_json::Value::as_f64) + .filter(|x| x.is_finite()) + .ok_or("no finite logsumexp_full: not an apr serve #4026 body")?; + let top = entry + .get("top_logprobs") + .and_then(serde_json::Value::as_array) + .ok_or("no top_logprobs")?; + if top.len() < k { + return Err(format!("{} alternatives, k = {k}", top.len())); + } + let mut ids = Vec::with_capacity(k); + let mut lps = Vec::with_capacity(k); + for alt in &top[..k] { + let lp = alt + .get("logprob") + .and_then(serde_json::Value::as_f64) + .filter(|x| x.is_finite() && *x <= 0.0) + .ok_or("an alternative's logprob is not a finite value ≤ 0")?; + if lps.last().is_some_and(|&prev: &f64| lp > prev) { + return Err("top_logprobs are not descending".into()); + } + ids.push(serve_id(alt)?); + lps.push(lp); + } + let mass: f64 = lps.iter().map(|x| x.exp()).sum(); + Ok(TokenLogits { + token_id_sampled: serve_id(entry)?, + ids, + logprobs: lps.iter().map(|&x| f16::from_f64(x)).collect(), + logsumexp_full: lse as f32, + topk_mass: f16::from_f64(mass.min(1.0)), + }) +} + +fn check_all(h: &Header, tokens: &[TokenLogits]) -> Result<(), String> { + if h.schema != SCHEME { + return Err(format!("schema {} is not {SCHEME}", h.schema)); + } + if h.k == 0 { + return Err("k = 0".into()); + } + if usize::try_from(h.n_tokens).ok() != Some(tokens.len()) { + return Err(format!( + "header n_tokens {} != {} tokens", + h.n_tokens, + tokens.len() + )); + } + if !h.temperature_of_record.is_finite() || h.temperature_of_record < 0.0 { + return Err("temperature_of_record is not a finite value ≥ 0".into()); + } + for (name, v) in [ + ("model_sha256", &h.model_sha256), + ("tokenizer_sha256", &h.tokenizer_sha256), + ] { + if v.len() != 64 || !v.bytes().all(|b| b.is_ascii_hexdigit()) { + return Err(format!("{name} is not a sha256 hex digest")); + } + } + tokens + .iter() + .enumerate() + .try_for_each(|(i, t)| t.check(usize::from(h.k), i)) +} + +/// Encode one blob. Refuses a header/token set the decoder would reject, so +/// a blob that exists is a blob that reads. +pub fn encode(h: &Header, tokens: &[TokenLogits]) -> Result, String> { + check_all(h, tokens)?; + let head = serde_json::to_vec(h).map_err(|e| e.to_string())?; + let mut cols = Vec::with_capacity(tokens.len() * (usize::from(h.k) * 6 + 10)); + for t in tokens { + cols.extend_from_slice(&t.token_id_sampled.to_le_bytes()); + } + for t in tokens { + t.ids + .iter() + .for_each(|x| cols.extend_from_slice(&x.to_le_bytes())); + } + for t in tokens { + t.logprobs + .iter() + .for_each(|x| cols.extend_from_slice(&x.to_bits().to_le_bytes())); + } + for t in tokens { + cols.extend_from_slice(&t.logsumexp_full.to_le_bytes()); + } + for t in tokens { + cols.extend_from_slice(&t.topk_mass.to_bits().to_le_bytes()); + } + let body = zstd::bulk::compress(&cols, ZSTD_LEVEL).map_err(|e| e.to_string())?; + let len = u32::try_from(head.len()).map_err(|_| "header too large".to_string())?; + let mut out = Vec::with_capacity(MAGIC.len() + 4 + head.len() + body.len()); + out.extend_from_slice(MAGIC); + out.extend_from_slice(&len.to_le_bytes()); + out.extend_from_slice(&head); + out.extend_from_slice(&body); + Ok(out) +} + +/// Uncompressed column bytes for `n` tokens at `k` (the M6 "raw" figure). +#[must_use] +pub fn raw_column_bytes(n: usize, k: usize) -> usize { + n * (4 + k * (4 + 2) + 4 + 2) +} + +struct Cursor<'a>(&'a [u8]); + +impl Cursor<'_> { + fn take(&mut self) -> Result<[u8; N], String> { + if self.0.len() < N { + return Err("column data is short".into()); + } + let (a, b) = self.0.split_at(N); + self.0 = b; + a.try_into().map_err(|_| "column data is short".to_string()) + } +} + +/// Decode and validate one blob. +pub fn decode(blob: &[u8]) -> Result<(Header, Vec), String> { + let rest = blob.strip_prefix(MAGIC.as_slice()).ok_or("bad magic")?; + if rest.len() < 4 { + return Err("truncated header length".into()); + } + let (len, rest) = rest.split_at(4); + let len = u32::from_le_bytes(len.try_into().map_err(|_| "header length")?) as usize; + if rest.len() < len { + return Err("truncated header".into()); + } + let (head, body) = rest.split_at(len); + let h: Header = serde_json::from_slice(head).map_err(|e| format!("header: {e}"))?; + let (n, k) = (h.n_tokens as usize, usize::from(h.k)); + let want = raw_column_bytes(n, k); + let cols = zstd::bulk::decompress(body, want + 1).map_err(|e| format!("columns: {e}"))?; + if cols.len() != want { + return Err(format!( + "columns hold {} bytes, header implies {want}", + cols.len() + )); + } + let mut c = Cursor(&cols); + let mut sampled = Vec::with_capacity(n); + for _ in 0..n { + sampled.push(u32::from_le_bytes(c.take()?)); + } + let mut ids = Vec::with_capacity(n * k); + for _ in 0..n * k { + ids.push(u32::from_le_bytes(c.take()?)); + } + let mut lps = Vec::with_capacity(n * k); + for _ in 0..n * k { + lps.push(f16::from_bits(u16::from_le_bytes(c.take()?))); + } + let mut lse = Vec::with_capacity(n); + for _ in 0..n { + lse.push(f32::from_le_bytes(c.take()?)); + } + let mut tokens = Vec::with_capacity(n); + for (i, (&s, &l)) in sampled.iter().zip(&lse).enumerate() { + tokens.push(TokenLogits { + token_id_sampled: s, + ids: ids[i * k..(i + 1) * k].to_vec(), + logprobs: lps[i * k..(i + 1) * k].to_vec(), + logsumexp_full: l, + topk_mass: f16::from_bits(u16::from_le_bytes(c.take()?)), + }); + } + check_all(&h, &tokens)?; + Ok((h, tokens)) +} + +#[cfg(test)] +#[path = "sparse_logits_tests.rs"] +mod tests; diff --git a/crates/aprender-review-experiment/src/sparse_logits_tests.rs b/crates/aprender-review-experiment/src/sparse_logits_tests.rs new file mode 100644 index 0000000000..381c9cba6b --- /dev/null +++ b/crates/aprender-review-experiment/src/sparse_logits_tests.rs @@ -0,0 +1,308 @@ +// `json!` expands to an `unwrap` of an infallible `to_value`. +#![allow(clippy::disallowed_methods)] + +use super::*; + +const SHA: &str = "70dffe5625e1849661303b82bb1fdf5254ab1d4560e2fe3eacc147e5d2588bba"; + +fn header(k: u8, n: u32) -> Header { + Header { + schema: SCHEME.into(), + mode: Mode::Topk, + model_sha256: SHA.into(), + tokenizer_sha256: SHA.into(), + apr_tag: "v0.70.0-rc.1".into(), + backend: "gx10-cuda".into(), + temperature_of_record: 0.7, + k, + n_tokens: n, + } +} + +/// A deterministic vocab row bigger than 2^16 so id width is exercised. +fn row(seed: u32, vocab: usize) -> Vec { + (0..vocab) + .map(|i| { + let x = (i as u32) + .wrapping_mul(2_654_435_761) + .wrapping_add(seed.wrapping_mul(40_503)); + f32::from((x >> 20) as u16) / 256.0 + }) + .collect() +} + +fn tokens(n: u32, vocab: usize) -> Vec { + (0..n) + .map(|s| { + TokenLogits::from_full_logits(&row(s, vocab), DEFAULT_K, s * 7).expect("finite row") + }) + .collect() +} + +/// FALSIFY-SPL-001: header and every column survive a round trip exactly. +#[test] +fn falsify_spl_001_round_trip_is_exact() { + let t = tokens(16, 70_000); + let h = header(DEFAULT_K, 16); + let (h2, t2) = decode(&encode(&h, &t).expect("encode")).expect("decode"); + assert_eq!(h2, h); + assert_eq!(t2, t); + assert!( + t.iter().flat_map(|x| &x.ids).any(|&id| id > 65_535), + "fixture must carry ids > u16" + ); +} + +/// FALSIFY-SPL-002: the stored mass is Σexp(logprobs) of the top-k of the +/// full softmax, and 1 − M is the true tail mass. +#[test] +fn falsify_spl_002_residual_mass_is_the_true_tail() { + let logits = [3.0_f32, 1.0, 0.5, 0.0, -1.0, -2.0]; + let t = TokenLogits::from_full_logits(&logits, 2, 0).expect("row"); + let z: f64 = logits.iter().map(|&x| f64::from(x).exp()).sum(); + let tail = (f64::from(0.5_f32).exp() + 1.0 + (-1.0_f64).exp() + (-2.0_f64).exp()) / z; + assert_eq!(t.ids, vec![0, 1]); + assert!( + (f64::from(t.residual_mass()) - tail).abs() < 1e-3, + "{} vs {tail}", + t.residual_mass() + ); + assert!((f64::from(t.logsumexp_full) - z.ln()).abs() < 1e-5); + assert!(t.logprobs.iter().all(|lp| lp.to_f32() <= 0.0)); + let (_, back) = decode(&encode(&header(2, 1), &[t.clone()]).expect("encode")).expect("decode"); + assert_eq!(back[0].residual_mass(), t.residual_mass()); +} + +/// FALSIFY-SPL-003: a mass that disagrees with its own logprobs is refused +/// at BOTH ends — a producer cannot write it, a reader cannot accept it. +#[test] +fn falsify_spl_003_inconsistent_mass_is_refused() { + let mut t = tokens(1, 1_000); + t[0].topk_mass = f16::from_f32(1.0); + assert!(encode(&header(DEFAULT_K, 1), &t).is_err()); + let good = tokens(1, 1_000); + let mut blob = encode(&header(DEFAULT_K, 1), &good).expect("encode"); + // Re-encode the columns with a forged mass, header untouched. + let hl = u32::from_le_bytes(blob[8..12].try_into().expect("len")) as usize; + let mut cols = zstd::bulk::decompress(&blob[12 + hl..], 1 << 20).expect("cols"); + let n = cols.len(); + cols[n - 2..].copy_from_slice(&f16::from_f32(1.0).to_bits().to_le_bytes()); + blob.truncate(12 + hl); + blob.extend(zstd::bulk::compress(&cols, 3).expect("zstd")); + assert!(decode(&blob) + .expect_err("forged mass") + .contains("topk_mass")); +} + +/// FALSIFY-SPL-004: the header is closed and the columns are exact-length. +#[test] +fn falsify_spl_004_decoder_is_strict() { + let t = tokens(2, 500); + let blob = encode(&header(DEFAULT_K, 2), &t).expect("encode"); + let hl = u32::from_le_bytes(blob[8..12].try_into().expect("len")) as usize; + let head = std::str::from_utf8(&blob[12..12 + hl]).expect("utf8"); + // Unknown key. + let extra = head.replacen('{', "{\"extra\":1,", 1); + let mut b = MAGIC.to_vec(); + b.extend((extra.len() as u32).to_le_bytes()); + b.extend(extra.as_bytes()); + b.extend(&blob[12 + hl..]); + assert!(decode(&b).is_err(), "unknown header key accepted"); + // n_tokens lies about the column length. + assert!(encode(&header(DEFAULT_K, 3), &t).is_err()); + let lie = head.replace("\"n_tokens\":2", "\"n_tokens\":1"); + let mut b = MAGIC.to_vec(); + b.extend((lie.len() as u32).to_le_bytes()); + b.extend(lie.as_bytes()); + b.extend(&blob[12 + hl..]); + assert!(decode(&b).is_err(), "column overrun accepted"); + // Truncation anywhere and a bad magic. + for cut in [0, 7, 11, 12 + hl - 1, blob.len() - 1] { + assert!(decode(&blob[..cut]).is_err(), "truncated at {cut} accepted"); + } + let mut bad = blob.clone(); + bad[0] = b'X'; + assert!(decode(&bad).is_err()); + // A trailing byte after the column frame. + let mut long = blob.clone(); + long.push(0); + assert!(decode(&long).is_err(), "trailing byte accepted"); + // Exactly one extra column byte inside a valid zstd frame. + let mut cols = zstd::bulk::decompress(&blob[12 + hl..], 1 << 20).expect("cols"); + cols.push(0); + let mut b = blob[..12 + hl].to_vec(); + b.extend(zstd::bulk::compress(&cols, 3).expect("zstd")); + assert!(decode(&b).is_err(), "one extra column byte accepted"); +} + +#[test] +fn from_full_logits_rejects_degenerate_rows() { + assert!(TokenLogits::from_full_logits(&[1.0, 2.0], 0, 0).is_none()); + assert!(TokenLogits::from_full_logits(&[1.0, 2.0], 3, 0).is_none()); + assert!(TokenLogits::from_full_logits(&[1.0, f32::NAN], 1, 0).is_none()); + assert!(TokenLogits::from_full_logits(&[1.0, f32::INFINITY], 1, 0).is_none()); + // k == vocab: all the mass is kept, residual ≈ 0. + let t = TokenLogits::from_full_logits(&[0.0, 0.0, 0.0], 3, 0).expect("row"); + assert!(t.residual_mass() < 1e-3); + assert_eq!(t.ids, vec![0, 1, 2], "ties break by ascending id"); +} + +#[test] +fn header_rejects_non_sha_and_bad_schema() { + let t = tokens(1, 100); + let mut h = header(DEFAULT_K, 1); + h.model_sha256 = "abc".into(); + assert!(encode(&h, &t).is_err()); + let mut h = header(DEFAULT_K, 1); + h.schema = "sparse-logits-v0".into(); + assert!(encode(&h, &t).is_err()); + let mut h = header(DEFAULT_K, 1); + h.temperature_of_record = f32::NAN; + assert!(encode(&h, &t).is_err()); +} + +/// M6: the RAW figure is exact (130 B/token at k = 20; §2.7 estimated ≈128). +/// The encoded figure here is from a SYNTHETIC, highly regular fixture and is +/// a codec smoke only — it is not M6. M6's encoded bytes/token are measured +/// on real `apr serve` rows once top-k capture exists. +#[test] +fn m6_raw_bytes_per_token_at_k20() { + assert_eq!(raw_column_bytes(1, 20), 130); + let n = 256; + let blob = encode(&header(DEFAULT_K, n), &tokens(n, 151_936)).expect("encode"); + let raw = raw_column_bytes(n as usize, 20); + // Column frame + header is bounded by raw + zstd frame overhead + header. + assert!( + blob.len() < raw + 1024, + "encoded {} > raw {raw} + 1 KiB", + blob.len() + ); +} + +/// A body shaped as `apr serve` writes it for `top_logprobs = k` (#4026): +/// `logprob = logit − logsumexp_full` over the full row, most likely first. +fn serve_body(rows: &[Vec], k: usize, sampled: &[u32]) -> serde_json::Value { + let content: Vec = rows + .iter() + .zip(sampled) + .map(|(r, &s)| { + let t = TokenLogits::from_full_logits(r, 1, s).expect("finite row"); + let lse = f64::from(t.logsumexp_full); + let mut order: Vec = (0..r.len()).collect(); + order.sort_by(|&a, &b| r[b].total_cmp(&r[a]).then(a.cmp(&b))); + let top: Vec = order[..k] + .iter() + .map(|&i| serde_json::json!({"token_id": i, "logprob": f64::from(r[i]) - lse})) + .collect(); + serde_json::json!({ + "token_id": s, + "logprob": f64::from(r[s as usize]) - lse, + "top_logprobs": top, + "logsumexp_full": lse, + }) + }) + .collect(); + serde_json::json!({"choices": [{"logprobs": {"content": content}}]}) +} + +/// PRM C11: a serve body captures to the same record the full row gives, and +/// the capture encodes and decodes. +#[test] +fn c11_serve_capture_matches_the_full_row_and_round_trips() { + let rows: Vec> = (0..3).map(|s| row(s, 70_000)).collect(); + let sampled = [5_u32, 69_999, 12]; + let got = from_chat_completion(&serve_body(&rows, 20, &sampled), DEFAULT_K).expect("capture"); + assert_eq!(got.len(), 3); + for ((g, r), &s) in got.iter().zip(&rows).zip(&sampled) { + let want = TokenLogits::from_full_logits(r, DEFAULT_K, s).expect("row"); + assert_eq!(g.token_id_sampled, s); + assert_eq!(g.ids, want.ids); + for (a, b) in g.logprobs.iter().zip(&want.logprobs) { + assert!((a.to_f32() - b.to_f32()).abs() <= 1e-2, "{a} vs {b}"); + } + assert_eq!(g.logsumexp_full, want.logsumexp_full); + assert!((g.topk_mass.to_f32() - want.topk_mass.to_f32()).abs() <= MASS_TOLERANCE); + } + let blob = encode(&header(DEFAULT_K, 3), &got).expect("encode"); + assert_eq!(decode(&blob).expect("decode").1, got); +} + +/// PRM C11: every body the encoder could not honestly record is refused by name. +#[test] +fn c11_serve_capture_refuses_what_it_cannot_record() { + let good = serve_body(&[row(1, 64)], 4, &[3]); + let entry = "/choices/0/logprobs/content/0"; + let cases: &[(&str, fn(&mut serde_json::Value), &str)] = &[ + ( + "no logprobs", + |b| *b = serde_json::json!({"choices": [{}]}), + "did not ask", + ), + ( + "hosted body", + |b| { + b.pointer_mut("/choices/0/logprobs/content/0") + .expect("e") + .as_object_mut() + .expect("o") + .remove("logsumexp_full"); + }, + "logsumexp_full", + ), + ( + "too few", + |b| { + b.pointer_mut("/choices/0/logprobs/content/0/top_logprobs") + .expect("t") + .as_array_mut() + .expect("a") + .truncate(2); + }, + "2 alternatives, k = 4", + ), + ( + "not descending", + |b| { + b.pointer_mut("/choices/0/logprobs/content/0/top_logprobs") + .expect("t") + .as_array_mut() + .expect("a") + .swap(0, 1); + }, + "descending", + ), + ( + "positive logprob", + |b| { + b["choices"][0]["logprobs"]["content"][0]["top_logprobs"][0]["logprob"] = + serde_json::json!(0.5); + }, + "≤ 0", + ), + ( + "no token_id", + |b| { + b.pointer_mut("/choices/0/logprobs/content/0") + .expect("e") + .as_object_mut() + .expect("o") + .remove("token_id"); + }, + "token_id", + ), + ]; + assert!( + from_chat_completion(&good, 4).is_ok(), + "control: the unplanted body captures" + ); + assert!(good.pointer(entry).is_some()); + for (name, plant, want) in cases { + let mut b = good.clone(); + plant(&mut b); + let err = from_chat_completion(&b, 4).expect_err(name); + assert!(err.contains(want), "{name}: {err}"); + } + assert!(from_chat_completion(&good, 0) + .expect_err("k=0") + .contains("k = 0")); +} diff --git a/crates/aprender-review-experiment/src/split_guard.rs b/crates/aprender-review-experiment/src/split_guard.rs new file mode 100644 index 0000000000..f4125af875 --- /dev/null +++ b/crates/aprender-review-experiment/src/split_guard.rs @@ -0,0 +1,222 @@ +//! PRM-C9 `trace-split-guard-v1`: the train/val/test split (gate G-DUP; +//! contract `trace-split-guard-v1`, FALSIFY-TSG-001..004). +//! +//! - **Unit**: `repo#PR`. Every row of a PR lands in one split. +//! - **Time-ordered**: a PR's split comes from its first-seen time against +//! the plan's `val_from` / `test_from` dates. +//! - **Embargo**: a PR first seen less than [`EMBARGO_DAYS`] before a +//! boundary is in no split, so nothing in a later split is within 14 d of +//! an earlier one. +//! - **Closure**: PRs that share a [`crate::dedup`] cluster are one component, +//! split by the component's earliest time: a copy of a train diff opened +//! later is train too, never test. +//! - **Sealed**: a component holding a near-dup of a sealed item is +//! [`Split::Sealed`], i.e. refused from every pool. +//! +//! [`audit`] recomputes nothing: it checks any assignment for PRs or +//! clusters that straddle splits. G-DUP requires both lists empty. + +use std::collections::{BTreeMap, BTreeSet, HashMap}; + +use serde::{Deserialize, Serialize}; + +use crate::dedup::{self, Item}; + +pub const SCHEME: &str = "trace-split-guard-v1"; + +/// Minimum gap between a PR's first-seen time and the next split's start. +pub const EMBARGO_DAYS: i64 = 14; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub enum Split { + Train, + Val, + Test, + /// Near-dup of a sealed item: in no pool. + Sealed, +} + +/// One trace diff to place. +#[derive(Debug, Clone, Copy)] +pub struct Unit<'a> { + /// `diff_sha256`. + pub id: &'a str, + pub repo: &'a str, + pub pr: u64, + /// RFC 3339 UTC. + pub at: &'a str, + pub diff: &'a str, +} + +/// Split boundaries, `YYYY-MM-DD`. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Plan { + pub val_from: String, + pub test_from: String, +} + +/// The `split_guard` a trace row carries. `split: None` is embargoed. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Guard { + pub group_key: String, + pub split: Option, + pub dedup_cluster: String, +} + +/// Days since 1970-01-01 of a `YYYY-MM-DD` prefix; `None` if it is not one. +#[must_use] +pub fn day(s: &str) -> Option { + let b = s.as_bytes(); + if b.len() < 10 || b[4] != b'-' || b[7] != b'-' { + return None; + } + let num = |r: std::ops::Range| s.get(r)?.parse::().ok(); + let (y, m, d) = (num(0..4)?, num(5..7)?, num(8..10)?); + if !(1..=12).contains(&m) || !(1..=31).contains(&d) { + return None; + } + // Howard Hinnant's days_from_civil. + let y = if m <= 2 { y - 1 } else { y }; + let era = y.div_euclid(400); + let yoe = y - era * 400; + let doy = (153 * (m + if m > 2 { -3 } else { 9 }) + 2) / 5 + d - 1; + let doe = yoe * 365 + yoe / 4 - yoe / 100 + doy; + Some(era * 146_097 + doe - 719_468) +} + +/// The `YYYY-MM-DD` of a [`day`] number (Hinnant's civil_from_days). +#[must_use] +pub fn date(days: i64) -> String { + let z = days + 719_468; + let era = z.div_euclid(146_097); + let doe = z - era * 146_097; + let yoe = (doe - doe / 1460 + doe / 36_524 - doe / 146_096) / 365; + let doy = doe - (365 * yoe + yoe / 4 - yoe / 100); + let mp = (5 * doy + 2) / 153; + let d = doy - (153 * mp + 2) / 5 + 1; + let m = if mp < 10 { mp + 3 } else { mp - 9 }; + let y = yoe + era * 400 + i64::from(m <= 2); + format!("{y:04}-{m:02}-{d:02}") +} + +/// Assign each unit its guard, in input order. A plan whose val window is +/// shorter than the embargo, or an unreadable date, is an error. +pub fn assign(units: &[Unit], sealed: &[(&str, &str)], plan: &Plan) -> Result, String> { + let (v, t) = ( + day(&plan.val_from).ok_or("val_from is not a date")?, + day(&plan.test_from).ok_or("test_from is not a date")?, + ); + if t - v <= EMBARGO_DAYS { + return Err(format!( + "val window {} d is not longer than the {EMBARGO_DAYS} d embargo", + t - v + )); + } + let days = units + .iter() + .map(|u| day(u.at).ok_or_else(|| format!("{}: at {:?} is not a date", u.id, u.at))) + .collect::, String>>()?; + // Sealed items sort before every unit (empty `at`), so they name their clusters. + let items: Vec = sealed + .iter() + .map(|&(id, diff)| Item { id, at: "", diff }) + .chain(units.iter().map(|u| Item { + id: u.id, + at: u.at, + diff: u.diff, + })) + .collect(); + let all = dedup::clusters(&items); + let (sealed_ids, clusters) = all.split_at(sealed.len()); + let sealed_ids: BTreeSet<&String> = sealed_ids.iter().collect(); + // Components: units joined by a shared PR or a shared cluster. + let keys: Vec = units + .iter() + .map(|u| format!("{}#{}", u.repo, u.pr)) + .collect(); + let mut parent: Vec = (0..units.len()).collect(); + let mut first: HashMap<&str, usize> = HashMap::new(); + for i in 0..units.len() { + for k in [keys[i].as_str(), clusters[i].as_str()] { + let j = *first.entry(k).or_insert(i); + let (ri, rj) = (root(&mut parent, i), root(&mut parent, j)); + parent[ri.max(rj)] = ri.min(rj); + } + } + let mut comp: BTreeMap = BTreeMap::new(); + for i in 0..units.len() { + let r = root(&mut parent, i); + let e = comp.entry(r).or_insert((days[i], false)); + e.0 = e.0.min(days[i]); + e.1 |= sealed_ids.contains(&clusters[i]); + } + Ok((0..units.len()) + .map(|i| { + let (d, is_sealed) = comp[&root(&mut parent, i)]; + let split = if is_sealed { + Some(Split::Sealed) + } else if d < v - EMBARGO_DAYS { + Some(Split::Train) + } else if d < v { + None + } else if d < t - EMBARGO_DAYS { + Some(Split::Val) + } else if d < t { + None + } else { + Some(Split::Test) + }; + Guard { + group_key: keys[i].clone(), + split, + dedup_cluster: clusters[i].clone(), + } + }) + .collect()) +} + +fn root(parent: &mut [usize], mut i: usize) -> usize { + while parent[i] != i { + parent[i] = parent[parent[i]]; + i = parent[i]; + } + i +} + +/// What straddles splits in an assignment. G-DUP holds when both are empty. +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize)] +pub struct Audit { + pub straddling_groups: Vec, + pub cross_split_clusters: Vec, +} + +/// Check any assignment. Embargoed members (`None`) are in no pool, so they +/// never make a straddle. +#[must_use] +pub fn audit(guards: &[Guard]) -> Audit { + let mut by_group: BTreeMap<&str, BTreeSet> = BTreeMap::new(); + let mut by_cluster: BTreeMap<&str, BTreeSet> = BTreeMap::new(); + for g in guards { + if let Some(s) = g.split { + by_group.entry(&g.group_key).or_default().insert(s); + by_cluster.entry(&g.dedup_cluster).or_default().insert(s); + } + } + let straddles = |m: BTreeMap<&str, BTreeSet>| -> Vec { + m.into_iter() + .filter(|(_, s)| s.len() > 1) + .map(|(k, _)| k.to_owned()) + .collect() + }; + Audit { + straddling_groups: straddles(by_group), + cross_split_clusters: straddles(by_cluster), + } +} + +#[cfg(test)] +#[path = "split_guard_tests.rs"] +mod tests; diff --git a/crates/aprender-review-experiment/src/split_guard_tests.rs b/crates/aprender-review-experiment/src/split_guard_tests.rs new file mode 100644 index 0000000000..98b98a048a --- /dev/null +++ b/crates/aprender-review-experiment/src/split_guard_tests.rs @@ -0,0 +1,144 @@ +use super::*; + +/// Segments `from..to` of a synthetic diff; each segment's literals are its own. +fn segs(from: u32, to: u32) -> String { + (from..to) + .flat_map(|k| { + (0..4).map(move |j| { + format!("+ let v = w.get({k}).map(|x| x * {k}{j}).unwrap_or({j});\n") + }) + }) + .collect() +} + +/// The same change re-indented and renamed. +fn perturb(d: &str) -> String { + d.replace(" ", " ").replace("w.get", "table.get") +} + +fn plan() -> Plan { + Plan { + val_from: "2026-09-01".into(), + test_from: "2026-10-01".into(), + } +} + +fn unit<'a>(id: &'a str, pr: u64, at: &'a str, diff: &'a str) -> Unit<'a> { + Unit { + id, + repo: "paiml/aprender", + pr, + at, + diff, + } +} + +fn splits(g: &[Guard]) -> Vec> { + g.iter().map(|g| g.split).collect() +} + +#[test] +fn falsify_tsg_001_a_pr_never_straddles_splits() { + let (a, b) = (segs(1, 6), segs(20, 25)); + // PR 1 opened in the train window; its second round lands in test time. + let u = [ + unit("a", 1, "2026-08-01T00:00:00Z", &a), + unit("b", 1, "2026-10-05T00:00:00Z", &b), + ]; + let g = assign(&u, &[], &plan()).expect("assigns"); + assert_eq!(splits(&g), [Some(Split::Train), Some(Split::Train)]); + assert_eq!(audit(&g), Audit::default()); + let mut bad = g.clone(); + bad[1].split = Some(Split::Test); + assert_eq!(audit(&bad).straddling_groups, ["paiml/aprender#1"]); +} + +#[test] +fn falsify_tsg_002_a_near_dup_cluster_closes_to_its_earliest_members_split() { + let (a, b, c) = (segs(1, 6), segs(20, 25), segs(40, 45)); + let (a2, b2) = (perturb(&a), perturb(&b)); + let u = [ + // A chain: PR 1 {a} ~ PR 2 {a', b} ~ PR 3 {b'}; PR 4 unrelated. + unit("b2", 3, "2026-10-09T00:00:00Z", &b2), + unit("a", 1, "2026-08-01T00:00:00Z", &a), + unit("a2", 2, "2026-10-05T00:00:00Z", &a2), + unit("b", 2, "2026-10-05T00:00:00Z", &b), + unit("c", 4, "2026-10-06T00:00:00Z", &c), + ]; + let g = assign(&u, &[], &plan()).expect("assigns"); + let train = Some(Split::Train); + assert_eq!(splits(&g), [train, train, train, train, Some(Split::Test)]); + assert_eq!(g[2].dedup_cluster, "a"); + assert_eq!(audit(&g), Audit::default()); + // A per-row time split leaks the copy into test, and the audit sees it. + let mut naive = g.clone(); + naive[2].split = Some(Split::Test); + naive[2].group_key = "paiml/aprender#9".into(); + assert_eq!(audit(&naive).cross_split_clusters, ["a"]); +} + +#[test] +fn falsify_tsg_003_the_embargo_is_at_least_14_days() { + let d: Vec = (0..5).map(|k| segs(10 * k + 1, 10 * k + 6)).collect(); + let u = [ + unit("t", 1, "2026-08-17T23:59:59Z", &d[0]), // 15 d before val: train + unit("e1", 2, "2026-08-18T00:00:00Z", &d[1]), // 14 d before val: embargoed + unit("v", 3, "2026-09-01T00:00:00Z", &d[2]), + unit("e2", 4, "2026-09-30T00:00:00Z", &d[3]), + unit("x", 5, "2026-10-01T00:00:00Z", &d[4]), + ]; + let g = assign(&u, &[], &plan()).expect("assigns"); + assert_eq!( + splits(&g), + [ + Some(Split::Train), + None, + Some(Split::Val), + None, + Some(Split::Test) + ] + ); + // An embargoed member never makes a straddle. + let mut one = g.clone(); + one[1].group_key = one[0].group_key.clone(); + assert_eq!(audit(&one), Audit::default()); + // A val window no longer than the embargo is refused, as is a bad date. + let short = Plan { + val_from: "2026-09-01".into(), + test_from: "2026-09-15".into(), + }; + assert!(assign(&u, &[], &short).is_err()); + let bad = [unit("t", 1, "yesterday", &d[0])]; + assert!(assign(&bad, &[], &plan()).is_err()); +} + +#[test] +fn falsify_tsg_004_a_perturbed_sealed_item_is_refused_with_its_whole_pr() { + let (s, b) = (segs(1, 6), segs(20, 25)); + let leak = perturb(&s); + let c = segs(40, 45); + let u = [ + unit("leak", 1, "2026-08-01T00:00:00Z", &leak), + unit("b", 1, "2026-08-02T00:00:00Z", &b), + unit("c", 2, "2026-08-03T00:00:00Z", &c), + ]; + let g = assign(&u, &[("sealed-7", &s)], &plan()).expect("assigns"); + let sealed = Some(Split::Sealed); + assert_eq!(splits(&g), [sealed, sealed, Some(Split::Train)]); + assert_eq!(g[0].dedup_cluster, "sealed-7"); +} + +#[test] +fn day_matches_the_civil_calendar() { + let cases = [ + ("1970-01-01", Some(0)), + ("2000-03-01", Some(11_017)), + ("2026-09-25T12:00:00Z", Some(20_721)), + ("2024-02-29", Some(19_782)), + ("2026-13-01", None), + ("26-09-25", None), + ]; + for (s, want) in cases { + assert_eq!(day(s), want, "{s}"); + } +} diff --git a/crates/aprender-review-experiment/src/stats.rs b/crates/aprender-review-experiment/src/stats.rs new file mode 100644 index 0000000000..4a56319b4b --- /dev/null +++ b/crates/aprender-review-experiment/src/stats.rs @@ -0,0 +1,619 @@ +//! PRM-001 (PROMETHEUS) frozen analysis code (spec v3 §2.7, §3, §5.3; was REX-001). +//! +//! This file is part of the pre-registration bundle (`prereg.rs`): its sha256 is +//! folded into the prereg sha that every receipt carries. Changing a single byte +//! here after REX-00 is a new spec version (R-1), and data gathered before it is +//! `exploratory`. +//! +//! Pre-registered choices, stated once so nobody re-derives them: +//! - Wilson score interval, z = 1.959963984540054 (two-sided 95 %). +//! - McNemar EXACT, paired, one-sided: under H0 the discordant count `b` +//! (challenger right, champion wrong) is Binomial(b + c, 1/2); p = P(X >= b). +//! - Bootstrap: percentile CI over items resampled with replacement, 10 000 +//! resamples, SplitMix64 seeded (the generator is written out here so its +//! stream can never change under a dependency bump). One-sided bootstrap p = +//! fraction of resamples whose difference is < 0. +//! - v2 H4: the bootstrap statistic is `p(apr) − min(p(Haiku), p(agy))` with +//! the min computed inside each resample (`bootstrap_vs_min_voter`). The +//! decision is the Holm-adjusted one-sided p; the 95 % CI is report-only. +//! - v2 H5: recall on class R only; class-P recall and precision are +//! descriptive (mutant confound). +//! - v3 H7 (a gate outside the Holm family, δ-free): Cohen's κ on per-item +//! errors (`error_kappa`); holds iff κ(apr, each voter) ≤ the max voter–voter +//! κ on the same items (`h7_holds`). +//! - Holm step-down over the family H1–H6 at overall alpha = 0.05. The count +//! and threshold tests (H1, H2, H6) enter the family as p = 0 when rejected +//! and p = 1 when not: they are deterministic rules, not sampling statistics. +//! - Percentiles: nearest-rank (`ceil(q * n)`-th order statistic). +//! - H6 noise = 2 x sample sd (n - 1) of the alone runs; rejected when +//! mean(co-located) - mean(alone) > noise. + +/// z for a two-sided 95 % interval. +pub const Z95: f64 = 1.959_963_984_540_054; +/// Overall family-wise alpha for H1–H6 (spec §3). +pub const ALPHA: f64 = 0.05; +/// Bootstrap resamples (spec §3 H4). +pub const BOOTSTRAP_RESAMPLES: usize = 10_000; + +/// Wilson score interval for `k` successes in `n` trials. `None` when `n == 0`. +#[must_use] +pub fn wilson(k: u64, n: u64, z: f64) -> Option<(f64, f64)> { + if n == 0 || k > n { + return None; + } + let n_f = n as f64; + let p = k as f64 / n_f; + let z2 = z * z; + let denom = 1.0 + z2 / n_f; + let centre = (p + z2 / (2.0 * n_f)) / denom; + let half = z * ((p * (1.0 - p) / n_f) + z2 / (4.0 * n_f * n_f)).sqrt() / denom; + Some(((centre - half).max(0.0), (centre + half).min(1.0))) +} + +/// Half-width of the Wilson interval — the §2.2 sample-size rule reads this. +#[must_use] +pub fn wilson_half_width(k: u64, n: u64) -> Option { + wilson(k, n, Z95).map(|(lo, hi)| (hi - lo) / 2.0) +} + +fn ln_choose(n: u64, k: u64) -> f64 { + // Sum of logs is exact enough for n <= a few thousand and has no overflow. + let k = k.min(n - k); + (0..k) + .map(|i| ((n - i) as f64).ln() - ((i + 1) as f64).ln()) + .sum() +} + +/// McNemar exact, paired, one-sided. `b` = pairs where the challenger is right +/// and the champion wrong; `c` = the reverse. Returns P(X >= b), X ~ Bin(b+c, ½). +/// With no discordant pairs there is no evidence: p = 1. +#[must_use] +pub fn mcnemar_exact_one_sided(b: u64, c: u64) -> f64 { + let n = b + c; + if n == 0 { + return 1.0; + } + let ln_half_n = (n as f64) * 0.5_f64.ln(); + let p: f64 = (b..=n).map(|x| (ln_choose(n, x) + ln_half_n).exp()).sum(); + p.min(1.0) +} + +/// SplitMix64 — the pre-registered bootstrap generator. +#[derive(Debug, Clone)] +pub struct SplitMix64(u64); + +impl SplitMix64 { + #[must_use] + pub fn new(seed: u64) -> Self { + Self(seed) + } + + pub fn next_u64(&mut self) -> u64 { + self.0 = self.0.wrapping_add(0x9E37_79B9_7F4A_7C15); + let mut z = self.0; + z = (z ^ (z >> 30)).wrapping_mul(0xBF58_476D_1CE4_E5B9); + z = (z ^ (z >> 27)).wrapping_mul(0x94D0_49BB_1331_11EB); + z ^ (z >> 31) + } + + /// Uniform index in `0..n` (Lemire multiply-shift; n > 0). + pub fn below(&mut self, n: usize) -> usize { + ((u128::from(self.next_u64()) * n as u128) >> 64) as usize + } +} + +/// One paired review item for two lanes, A (the candidate) and B (the baseline). +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct Paired { + /// Ground truth: the item carries a defect (class P or R). + pub defect: bool, + /// Lane A said FAIL (a parsed FAIL; `Unparsed`/`NotRun` are NOT a FAIL). + pub a_fail: bool, + /// Lane B said FAIL. + pub b_fail: bool, +} + +/// Which lane-level rate a bootstrap compares. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Rate { + /// FAIL on defect ÷ all FAIL. + Precision, + /// FAIL on defect ÷ defects. + Recall, +} + +fn rate(items: &[Paired], idx: &[usize], lane_a: bool, which: Rate) -> Option { + let (mut num, mut den) = (0u64, 0u64); + for &i in idx { + let it = items[i]; + let fail = if lane_a { it.a_fail } else { it.b_fail }; + match which { + Rate::Precision => { + if fail { + den += 1; + num += u64::from(it.defect); + } + } + Rate::Recall => { + if it.defect { + den += 1; + num += u64::from(fail); + } + } + } + } + (den > 0).then(|| num as f64 / den as f64) +} + +/// Result of a paired bootstrap of `rate(A) - rate(B)`. +#[derive(Debug, Clone, Copy, PartialEq)] +pub struct BootstrapDiff { + /// Point estimate on the full sample. + pub estimate: f64, + /// Percentile CI at `1 - alpha` two-sided. + pub ci: (f64, f64), + /// One-sided p: fraction of defined resamples with difference < 0. + pub p_below_zero: f64, + /// Resamples where either rate was undefined (empty denominator); excluded. + pub undefined: usize, +} + +/// Paired bootstrap of `rate(A) - rate(B)` over items (spec §3 H4/H5). +/// `None` when the point estimate is undefined or no resample is defined. +#[must_use] +pub fn paired_bootstrap_diff( + items: &[Paired], + which: Rate, + resamples: usize, + seed: u64, + alpha: f64, +) -> Option { + let all: Vec = (0..items.len()).collect(); + let estimate = rate(items, &all, true, which)? - rate(items, &all, false, which)?; + let mut rng = SplitMix64::new(seed); + let mut diffs = Vec::with_capacity(resamples); + let mut undefined = 0usize; + let mut idx = vec![0usize; items.len()]; + for _ in 0..resamples { + for slot in &mut idx { + *slot = rng.below(items.len()); + } + match ( + rate(items, &idx, true, which), + rate(items, &idx, false, which), + ) { + (Some(a), Some(b)) => diffs.push(a - b), + _ => undefined += 1, + } + } + if diffs.is_empty() { + return None; + } + diffs.sort_by(f64::total_cmp); + let below = diffs.iter().filter(|d| **d < 0.0).count(); + Some(BootstrapDiff { + estimate, + ci: ( + percentile_sorted(&diffs, alpha / 2.0), + percentile_sorted(&diffs, 1.0 - alpha / 2.0), + ), + p_below_zero: below as f64 / diffs.len() as f64, + undefined, + }) +} + +/// One H4 item scored by apr and the two existing voters (Haiku, agy). +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct VsVoters { + /// Ground truth: the item carries a defect (class P or R). + pub defect: bool, + /// apr said FAIL. + pub apr_fail: bool, + /// The two voting lanes said FAIL, in a fixed order (Haiku, agy). + pub voter_fail: [bool; 2], +} + +fn rate_of(items: &[VsVoters], idx: &[usize], lane: Option, which: Rate) -> Option { + let pick = |it: &VsVoters| lane.map_or(it.apr_fail, |v| it.voter_fail[v]); + let (mut num, mut den) = (0u64, 0u64); + for &i in idx { + let it = &items[i]; + let fail = pick(it); + match which { + Rate::Precision if fail => { + den += 1; + num += u64::from(it.defect); + } + Rate::Recall if it.defect => { + den += 1; + num += u64::from(fail); + } + _ => {} + } + } + (den > 0).then(|| num as f64 / den as f64) +} + +/// `rate(apr) − min(rate(Haiku), rate(agy))` on one index set. +fn diff_vs_min(items: &[VsVoters], idx: &[usize], which: Rate) -> Option { + let a = rate_of(items, idx, None, which)?; + let lo = rate_of(items, idx, Some(0), which)?.min(rate_of(items, idx, Some(1), which)?); + Some(a - lo) +} + +/// H4 (spec v2 §3): paired bootstrap of `p(apr) − min(p(Haiku), p(agy))`. +/// The min is taken INSIDE each resample, so the weaker voter is chosen on +/// that resample and the selection is part of the sampling distribution. The +/// decision is the Holm-adjusted `p_below_zero`; the CI is report-only. +#[must_use] +pub fn bootstrap_vs_min_voter( + items: &[VsVoters], + which: Rate, + resamples: usize, + seed: u64, + alpha: f64, +) -> Option { + let all: Vec = (0..items.len()).collect(); + let estimate = diff_vs_min(items, &all, which)?; + let mut rng = SplitMix64::new(seed); + let mut diffs = Vec::with_capacity(resamples); + let mut undefined = 0usize; + let mut idx = vec![0usize; items.len()]; + for _ in 0..resamples { + for slot in &mut idx { + *slot = rng.below(items.len()); + } + match diff_vs_min(items, &idx, which) { + Some(d) => diffs.push(d), + None => undefined += 1, + } + } + if diffs.is_empty() { + return None; + } + diffs.sort_by(f64::total_cmp); + let below = diffs.iter().filter(|d| **d < 0.0).count(); + Some(BootstrapDiff { + estimate, + ci: ( + percentile_sorted(&diffs, alpha / 2.0), + percentile_sorted(&diffs, 1.0 - alpha / 2.0), + ), + p_below_zero: below as f64 / diffs.len() as f64, + undefined, + }) +} + +/// Cohen's κ between two lanes' per-item ERROR indicators (PRM-001 v3 §2.7, +/// H7): 1 = they miss the same items, 0 = overlap at chance. +/// `None` when the lengths differ, the sample is empty, or chance agreement +/// is 1 (both lanes constant), where κ is undefined. +#[must_use] +pub fn error_kappa(a_err: &[bool], b_err: &[bool]) -> Option { + if a_err.len() != b_err.len() || a_err.is_empty() { + return None; + } + let n = a_err.len() as f64; + let agree = a_err.iter().zip(b_err).filter(|(x, y)| x == y).count() as f64 / n; + let pa = a_err.iter().filter(|x| **x).count() as f64 / n; + let pb = b_err.iter().filter(|x| **x).count() as f64 / n; + let chance = pa * pb + (1.0 - pa) * (1.0 - pb); + (chance < 1.0).then(|| (agree - chance) / (1.0 - chance)) +} + +/// H7 independence gate (PRM-001 v3 §3; outside the Holm family, δ-free): +/// the local lane's κ_err with EVERY voter must not exceed the largest κ_err +/// among voter–voter pairs on the same items — a tie-breaker may not be more +/// correlated with a voter than the voters already are with each other. +/// `None` (undecided, never "holds") when either side is empty or any κ is +/// undefined. +#[must_use] +pub fn h7_holds(qwen_vs_voter: &[Option], voter_vs_voter: &[Option]) -> Option { + if qwen_vs_voter.is_empty() || voter_vs_voter.is_empty() { + return None; + } + let pairs: Option> = voter_vs_voter.iter().copied().collect(); + let ceiling = pairs?.into_iter().fold(f64::NEG_INFINITY, f64::max); + let qwen: Option> = qwen_vs_voter.iter().copied().collect(); + Some(qwen?.into_iter().all(|k| k <= ceiling)) +} + +/// Nearest-rank percentile of an ascending slice. `q` in (0, 1]. +#[must_use] +pub fn percentile_sorted(sorted: &[f64], q: f64) -> f64 { + if sorted.is_empty() { + return f64::NAN; + } + let rank = (q * sorted.len() as f64).ceil().max(1.0) as usize; + sorted[rank.min(sorted.len()) - 1] +} + +/// Nearest-rank percentile of an unsorted sample. +#[must_use] +pub fn percentile(sample: &[f64], q: f64) -> f64 { + let mut s = sample.to_vec(); + s.sort_by(f64::total_cmp); + percentile_sorted(&s, q) +} + +/// Holm step-down. Returns (adjusted p, rejected) in input order. +#[must_use] +pub fn holm(p: &[f64], alpha: f64) -> Vec<(f64, bool)> { + let m = p.len(); + let mut order: Vec = (0..m).collect(); + order.sort_by(|&a, &b| p[a].total_cmp(&p[b]).then(a.cmp(&b))); + let mut out = vec![(1.0, false); m]; + let mut running = 0.0_f64; + let mut still_rejecting = true; + for (rank, &i) in order.iter().enumerate() { + let factor = (m - rank) as f64; + running = running.max((p[i] * factor).min(1.0)); + still_rejecting = still_rejecting && p[i] <= alpha / factor; + out[i] = (running, still_rejecting); + } + out +} + +fn mean(x: &[f64]) -> f64 { + x.iter().sum::() / x.len() as f64 +} + +/// Sample standard deviation (n - 1). `None` for fewer than two values. +#[must_use] +pub fn sample_sd(x: &[f64]) -> Option { + if x.len() < 2 { + return None; + } + let m = mean(x); + Some((x.iter().map(|v| (v - m).powi(2)).sum::() / (x.len() - 1) as f64).sqrt()) +} + +/// H6 interference verdict for one cell. +#[derive(Debug, Clone, Copy, PartialEq)] +pub struct Interference { + /// mean(co-located) − mean(alone), in the workload's unit (seconds). + pub slowdown: f64, + /// 2 × sd(alone). + pub noise: f64, + /// slowdown > noise. + pub rejected: bool, +} + +/// H6: reference workload wall-clock alone vs co-located with the serve. +#[must_use] +pub fn interference(alone: &[f64], colocated: &[f64]) -> Option { + let noise = 2.0 * sample_sd(alone)?; + if colocated.is_empty() { + return None; + } + let slowdown = mean(colocated) - mean(alone); + Some(Interference { + slowdown, + noise, + rejected: slowdown > noise, + }) +} + +/// H1/H2 are counts: any divergent item rejects. Enters Holm as p ∈ {0, 1}. +#[must_use] +pub fn count_rule_p(divergent: u64) -> f64 { + if divergent > 0 { + 0.0 + } else { + 1.0 + } +} + +/// McNemar discordant counts from per-item correctness of (challenger, champion). +#[must_use] +pub fn discordant(pairs: &[(bool, bool)]) -> (u64, u64) { + pairs + .iter() + .fold((0, 0), |(b, c), &(ch, cp)| match (ch, cp) { + (true, false) => (b + 1, c), + (false, true) => (b, c + 1), + _ => (b, c), + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn close(a: f64, b: f64, tol: f64) -> bool { + (a - b).abs() <= tol + } + + #[test] + fn wilson_matches_hand_computed() { + // k=35, n=70, p=.5: centre .5, half = 1.96*sqrt(.25/70 + 3.8415/19600)/1.05488 = .114044 + let (lo, hi) = wilson(35, 70, Z95).expect("defined"); + assert!(close(lo, 0.385_957, 1e-5), "{lo}"); + assert!(close(hi, 0.614_043, 1e-5), "{hi}"); + // §2.2 basis: 70 defect items at p = .5. The spec's "≈ 0.117" is the WALD + // half-width 1.96*sqrt(.25/70); Wilson, the pre-registered interval, gives 0.114. + let hw = wilson_half_width(35, 70).expect("defined"); + assert!(close(hw, 0.114_043, 1e-5), "{hw}"); + assert_eq!(wilson(0, 0, Z95), None); + let (lo0, _) = wilson(0, 10, Z95).expect("defined"); + assert!(close(lo0, 0.0, 1e-12)); + } + + #[test] + fn mcnemar_exact_matches_binomial_tail() { + // b=8, c=2: P(X>=8 | n=10) = (45+10+1)/1024 = 0.0546875 + assert!(close(mcnemar_exact_one_sided(8, 2), 56.0 / 1024.0, 1e-12)); + // b=5, c=0: 1/32 + assert!(close(mcnemar_exact_one_sided(5, 0), 1.0 / 32.0, 1e-12)); + // b=0: whole distribution + assert!(close(mcnemar_exact_one_sided(0, 7), 1.0, 1e-12)); + assert!(close(mcnemar_exact_one_sided(0, 0), 1.0, 0.0)); + assert_eq!( + discordant(&[(true, false), (true, false), (false, true), (true, true)]), + (2, 1) + ); + } + + #[test] + fn holm_matches_hand_computed() { + // p sorted .01 .02 .03 .5 ; thresholds .0125 .0167 .025 .05 -> reject .01 only + let r = holm(&[0.03, 0.01, 0.5, 0.02], 0.05); + assert_eq!( + r.iter().map(|x| x.1).collect::>(), + vec![false, true, false, false] + ); + assert!(close(r[1].0, 0.04, 1e-12)); + assert!(close(r[3].0, 0.06, 1e-12)); + assert!(close(r[0].0, 0.06, 1e-12)); // monotone: max(0.06, 0.03*2) + assert!(close(r[2].0, 0.5, 1e-12)); + // deterministic count tests enter as 0 / 1 + let r = holm(&[count_rule_p(3), count_rule_p(0)], 0.05); + assert_eq!((r[0].1, r[1].1), (true, false)); + } + + #[test] + fn percentile_nearest_rank() { + let s: Vec = (1..=20).map(f64::from).collect(); + assert!(close(percentile(&s, 0.95), 19.0, 0.0)); + assert!(close(percentile(&s, 0.5), 10.0, 0.0)); + assert!(percentile(&[], 0.5).is_nan()); + } + + #[test] + fn splitmix_stream_is_frozen() { + // Reference values of SplitMix64 seeded 0 (Vigna's published sequence). + let mut r = SplitMix64::new(0); + assert_eq!(r.next_u64(), 0xE220_A839_7B1D_CDAF); + assert_eq!(r.next_u64(), 0x6E78_9E6A_A1B9_65F4); + } + + #[test] + fn bootstrap_is_seeded_and_signed() { + // A strictly better lane A: identical except A catches 5 more defects. + let mut items = Vec::new(); + for i in 0..40 { + let defect = i < 20; + let b_fail = defect && i < 10; + let a_fail = defect && i < 15; + items.push(Paired { + defect, + a_fail, + b_fail, + }); + } + let x = paired_bootstrap_diff(&items, Rate::Recall, 2000, 7, ALPHA).expect("defined"); + let y = paired_bootstrap_diff(&items, Rate::Recall, 2000, 7, ALPHA).expect("defined"); + assert_eq!(x, y, "same seed must give the same bootstrap"); + assert!(close(x.estimate, 0.25, 1e-12)); + assert!(x.ci.0 >= 0.0 && x.p_below_zero == 0.0, "{x:?}"); + // Precision: A and B both 1.0 -> diff 0 everywhere. + let p = paired_bootstrap_diff(&items, Rate::Precision, 500, 7, ALPHA).expect("defined"); + assert!(close(p.estimate, 0.0, 0.0) && close(p.ci.0, 0.0, 0.0)); + } + + /// Three lanes, equal point precision 10/12, false positives on + /// disjoint good items: which voter is weaker flips between resamples. + fn tied_voters() -> Vec { + (0..40) + .map(|i| { + let defect = i < 20; + let caught = i < 10; + VsVoters { + defect, + apr_fail: caught || i == 20 || i == 21, + voter_fail: [caught || i == 22 || i == 23, caught || i == 24 || i == 25], + } + }) + .collect() + } + + #[test] + fn falsify_h4_min_is_taken_inside_each_resample() { + let items = tied_voters(); + let inside = + bootstrap_vs_min_voter(&items, Rate::Precision, 2000, 4354, ALPHA).expect("defined"); + // The v1 rule: pick the weaker voter ONCE on the full sample (a tie → + // voter 0), then bootstrap against it. Same seed → same resamples. + let fixed: Vec = items + .iter() + .map(|t| Paired { + defect: t.defect, + a_fail: t.apr_fail, + b_fail: t.voter_fail[0], + }) + .collect(); + let once = + paired_bootstrap_diff(&fixed, Rate::Precision, 2000, 4354, ALPHA).expect("defined"); + assert!(close(inside.estimate, 0.0, 1e-12) && close(once.estimate, 0.0, 1e-12)); + // a − min(v0, v1) ≥ a − v0 on every resample, strictly on those where + // v1 is the weaker: the two rules must give different p. + assert!( + inside.p_below_zero < once.p_below_zero, + "{inside:?} vs {once:?}" + ); + // Voter order is irrelevant under min(). + let swapped: Vec = items + .iter() + .map(|t| VsVoters { + voter_fail: [t.voter_fail[1], t.voter_fail[0]], + ..*t + }) + .collect(); + assert_eq!( + bootstrap_vs_min_voter(&swapped, Rate::Precision, 2000, 4354, ALPHA), + Some(inside) + ); + } + + #[test] + fn error_kappa_matches_hand_computed() { + // agree 3/4, pa = .5, pb = .25, chance = .5 → κ = .5 + let k = error_kappa(&[true, true, false, false], &[true, false, false, false]); + assert!(close(k.expect("defined"), 0.5, 1e-12)); + assert_eq!(error_kappa(&[true, false], &[true, false]), Some(1.0)); + assert_eq!( + error_kappa(&[false, false], &[false, false]), + None, + "chance = 1" + ); + assert_eq!( + error_kappa(&[true], &[true, false]), + None, + "length mismatch" + ); + assert_eq!(error_kappa(&[], &[]), None); + } + + #[test] + fn falsify_h7_is_max_pair_and_undecided_on_gaps() { + let pairs = [Some(0.30), Some(0.10), Some(0.20)]; + assert_eq!( + h7_holds(&[Some(0.30), Some(0.05)], &pairs), + Some(true), + "≤ max pair" + ); + assert_eq!( + h7_holds(&[Some(0.31), Some(0.05)], &pairs), + Some(false), + "one voter above max" + ); + // Not the min, not the mean: 0.25 is above both and still holds. + assert_eq!(h7_holds(&[Some(0.25)], &pairs), Some(true)); + assert_eq!(h7_holds(&[None, Some(0.0)], &pairs), None); + assert_eq!(h7_holds(&[Some(0.0)], &[Some(0.3), None]), None); + assert_eq!(h7_holds(&[], &pairs), None); + assert_eq!(h7_holds(&[Some(0.0)], &[]), None); + } + + #[test] + fn interference_rule() { + let alone = [10.0, 10.2, 9.8, 10.1, 9.9]; + let sd = sample_sd(&alone).expect("n>=2"); + assert!(close(sd, 0.158_113_9, 1e-6)); + let quiet = interference(&alone, &[10.1, 10.2, 10.0, 10.2, 10.1]).expect("defined"); + assert!(!quiet.rejected, "{quiet:?}"); + let loud = interference(&alone, &[11.0, 11.2, 10.9, 11.1, 11.0]).expect("defined"); + assert!(loud.rejected, "{loud:?}"); + assert_eq!(interference(&[1.0], &[2.0]), None); + } +} diff --git a/crates/aprender-review-experiment/src/terms.rs b/crates/aprender-review-experiment/src/terms.rs new file mode 100644 index 0000000000..376a71404f --- /dev/null +++ b/crates/aprender-review-experiment/src/terms.rs @@ -0,0 +1,360 @@ +//! Provider-terms tagging for `agent-trace-v1` rows (PRA-001 T12, spec §2.10; +//! contract `agent-trace-terms-v1`). +//! +//! Every captured row carries `provider`, an exact `model_id`, `access_channel` +//! and a `terms_ref` snapshot, so any pool can be rebuilt without a given +//! provider or channel in one call (P-TERMS). This records which terms governed +//! a row; it is not legal advice, and the operator's S-6 ruling governs use. +//! +//! A row whose tags are missing or inconsistent is REFUSED by every rebuild, even +//! one that excludes nothing: an untagged row cannot be shown not to come from an +//! excluded source. + +use serde_json::Value; + +/// Who served the lane (`provider` in §2.2). +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Provider { + Anthropic, + Google, + Local, +} + +impl Provider { + pub const ALL: [Provider; 3] = [Provider::Anthropic, Provider::Google, Provider::Local]; + + #[must_use] + pub fn as_str(self) -> &'static str { + match self { + Provider::Anthropic => "anthropic", + Provider::Google => "google", + Provider::Local => "local", + } + } + + /// The exact wire spelling; anything else is unknown. + #[must_use] + pub fn parse(s: &str) -> Option { + Self::ALL.into_iter().find(|p| p.as_str() == s) + } +} + +/// How the lane was reached (`access_channel` in §2.2); terms differ by channel. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Channel { + AnthropicApi, + ClaudeCodeCli, + Antigravity, + GeminiApi, + Vertex, + AprServe, +} + +impl Channel { + pub const ALL: [Channel; 6] = [ + Channel::AnthropicApi, + Channel::ClaudeCodeCli, + Channel::Antigravity, + Channel::GeminiApi, + Channel::Vertex, + Channel::AprServe, + ]; + + #[must_use] + pub fn as_str(self) -> &'static str { + match self { + Channel::AnthropicApi => "anthropic-api", + Channel::ClaudeCodeCli => "claude-code-cli", + Channel::Antigravity => "antigravity", + Channel::GeminiApi => "gemini-api", + Channel::Vertex => "vertex", + Channel::AprServe => "apr-serve", + } + } + + /// The exact wire spelling; anything else is unknown. + #[must_use] + pub fn parse(s: &str) -> Option { + Self::ALL.into_iter().find(|c| c.as_str() == s) + } + + /// The only provider this channel can serve. + #[must_use] + pub fn provider(self) -> Provider { + match self { + Channel::AnthropicApi | Channel::ClaudeCodeCli => Provider::Anthropic, + Channel::Antigravity | Channel::GeminiApi | Channel::Vertex => Provider::Google, + Channel::AprServe => Provider::Local, + } + } +} + +/// The governing-terms snapshot taken at capture. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct TermsRef { + pub url: String, + pub effective_date: String, + pub fetched_at: String, +} + +/// The four terms fields of one row, validated. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Tags { + pub provider: Provider, + pub model_id: String, + pub channel: Channel, + pub terms: TermsRef, +} + +/// One reason a row's terms tags do not hold. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum Gap { + /// The line is not a JSON object. + NotJson, + /// A required field is absent, null or not a string/object. + Missing(&'static str), + UnknownProvider(String), + UnknownChannel(String), + /// The channel cannot serve the stated provider. + ChannelProvider { + channel: Channel, + provider: Provider, + }, + /// A family alias or placeholder, not an exact model id. + AliasModel(String), + /// A local model is named by its weights sha256 and nothing else. + LocalModelNotSha(String), + TermsUrl(String), + TermsEffectiveDate(String), + TermsFetchedAt(String), +} + +/// The validated tags of `row`, or every gap found. +/// +/// # Errors +/// Returns the gaps when any tag is missing, unknown or inconsistent. +pub fn tags(row: &Value) -> Result> { + let mut gaps = Vec::new(); + let text = |key: &'static str, gaps: &mut Vec| { + let s = row.get(key).and_then(Value::as_str); + if s.is_none() { + gaps.push(Gap::Missing(key)); + } + s + }; + let provider = text("provider", &mut gaps).and_then(|s| { + Provider::parse(s).or_else(|| { + gaps.push(Gap::UnknownProvider(s.to_string())); + None + }) + }); + let channel = text("access_channel", &mut gaps).and_then(|s| { + Channel::parse(s).or_else(|| { + gaps.push(Gap::UnknownChannel(s.to_string())); + None + }) + }); + let model_id = text("model_id", &mut gaps); + if let (Some(p), Some(c)) = (provider, channel) { + if c.provider() != p { + gaps.push(Gap::ChannelProvider { + channel: c, + provider: p, + }); + } + } + if let Some(m) = model_id { + if let Some(gap) = model_gap(m, provider) { + gaps.push(gap); + } + } + let terms = terms_ref(row.get("terms_ref"), &mut gaps); + match (provider, model_id, channel, terms) { + (Some(provider), Some(m), Some(channel), Some(terms)) if gaps.is_empty() => Ok(Tags { + provider, + model_id: m.to_string(), + channel, + terms, + }), + _ => Err(gaps), + } +} + +/// An exact id carries a version (a digit); a local id is a weights sha256. +fn model_gap(m: &str, provider: Option) -> Option { + let placeholder = m.is_empty() || m.eq_ignore_ascii_case("unknown"); + if provider == Some(Provider::Local) && !placeholder { + let hex = m.strip_prefix("sha256:").unwrap_or(m); + let is_sha = hex.len() == 64 && hex.bytes().all(|b| matches!(b, b'0'..=b'9' | b'a'..=b'f')); + return (!is_sha).then(|| Gap::LocalModelNotSha(m.to_string())); + } + let versioned = m.bytes().any(|b| b.is_ascii_digit()) && !m.contains(char::is_whitespace); + (placeholder || !versioned).then(|| Gap::AliasModel(m.to_string())) +} + +fn terms_ref(v: Option<&Value>, gaps: &mut Vec) -> Option { + let Some(obj) = v.and_then(Value::as_object) else { + gaps.push(Gap::Missing("terms_ref")); + return None; + }; + let field = |key: &'static str, gaps: &mut Vec| { + let s = obj.get(key).and_then(Value::as_str).map(str::to_string); + if s.is_none() { + gaps.push(Gap::Missing(key)); + } + s + }; + let url = field("url", gaps); + let effective_date = field("effective_date", gaps); + let fetched_at = field("fetched_at", gaps); + let before = gaps.len(); + if let Some(u) = &url { + if !u.starts_with("https://") || u.len() == "https://".len() { + gaps.push(Gap::TermsUrl(u.clone())); + } + } + if let Some(d) = &effective_date { + if !is_date(d) { + gaps.push(Gap::TermsEffectiveDate(d.clone())); + } + } + if let Some(t) = &fetched_at { + if !is_utc_timestamp(t) { + gaps.push(Gap::TermsFetchedAt(t.clone())); + } + } + Some(TermsRef { + url: url?, + effective_date: effective_date?, + fetched_at: fetched_at?, + }) + .filter(|_| gaps.len() == before) +} + +/// `YYYY-MM-DD` with a month 01-12 and a day 01-31. +fn is_date(s: &str) -> bool { + let b = s.as_bytes(); + let digits = |r: std::ops::Range| { + b[r.clone()] + .iter() + .all(u8::is_ascii_digit) + .then(|| num(&b[r])) + }; + if b.len() != 10 || b[4] != b'-' || b[7] != b'-' { + return false; + } + matches!( + (digits(0..4), digits(5..7), digits(8..10)), + (Some(_), Some(1..=12), Some(1..=31)) + ) +} + +/// RFC 3339 in UTC: `YYYY-MM-DDTHH:MM:SS[.frac]Z`. +fn is_utc_timestamp(s: &str) -> bool { + let Some(rest) = s.strip_suffix('Z') else { + return false; + }; + let (date, time) = match rest.split_once('T') { + Some(p) => p, + None => return false, + }; + let (hms, frac) = time.split_once('.').unwrap_or((time, "0")); + let t = hms.as_bytes(); + let two = |i: usize| { + t[i..i + 2] + .iter() + .all(u8::is_ascii_digit) + .then(|| num(&t[i..i + 2])) + }; + is_date(date) + && t.len() == 8 + && t[2] == b':' + && t[5] == b':' + && matches!( + (two(0), two(3), two(6)), + (Some(0..=23), Some(0..=59), Some(0..=60)) + ) + && !frac.is_empty() + && frac.bytes().all(|c| c.is_ascii_digit()) +} + +fn num(digits: &[u8]) -> u32 { + digits.iter().fold(0, |n, d| n * 10 + u32::from(d - b'0')) +} + +/// What a pool rebuild leaves out. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct Exclude { + pub providers: Vec, + pub channels: Vec, +} + +impl Exclude { + /// The public-release default (§2.11): gold-and-local only, so every hosted + /// provider is excluded unless the operator overrides it at a §8 STOP. + #[must_use] + pub fn public_release() -> Self { + Self { + providers: vec![Provider::Anthropic, Provider::Google], + channels: Vec::new(), + } + } + + /// Parse `--exclude-provider

` / `--exclude-channel `, each repeatable. + /// + /// # Errors + /// An unknown flag, a missing value, or an unknown provider or channel. + pub fn from_args(args: &[&str]) -> Result { + let mut ex = Self::default(); + let mut it = args.iter(); + while let Some(&flag) = it.next() { + let value = it.next().ok_or_else(|| format!("{flag}: missing value"))?; + match flag { + "--exclude-provider" => ex.providers.push( + Provider::parse(value).ok_or_else(|| format!("unknown provider: {value}"))?, + ), + "--exclude-channel" => ex.channels.push( + Channel::parse(value).ok_or_else(|| format!("unknown channel: {value}"))?, + ), + other => return Err(format!("unknown argument: {other}")), + } + } + Ok(ex) + } + + fn excludes(&self, t: &Tags) -> bool { + self.providers.contains(&t.provider) || self.channels.contains(&t.channel) + } +} + +/// A rebuilt pool: rows kept, rows left out by the filter, rows refused for gaps. +#[derive(Debug, Default)] +pub struct Rebuild<'a> { + pub kept: Vec<&'a str>, + pub excluded: Vec<&'a str>, + pub refused: Vec<(&'a str, Vec)>, +} + +/// Rebuild a pool from a JSONL batch, leaving out every excluded source. +/// Rows are passed through byte-for-byte, in input order. +#[must_use] +pub fn rebuild<'a>(rows: &'a str, ex: &Exclude) -> Rebuild<'a> { + let mut r = Rebuild::default(); + for line in rows.lines().filter(|l| !l.trim().is_empty()) { + let checked = serde_json::from_str::(line) + .ok() + .filter(Value::is_object) + .ok_or_else(|| vec![Gap::NotJson]) + .and_then(|v| tags(&v)); + match checked { + Ok(t) if ex.excludes(&t) => r.excluded.push(line), + Ok(_) => r.kept.push(line), + Err(gaps) => r.refused.push((line, gaps)), + } + } + r +} + +#[cfg(test)] +#[path = "terms_tests.rs"] +mod tests; diff --git a/crates/aprender-review-experiment/src/terms_tests.rs b/crates/aprender-review-experiment/src/terms_tests.rs new file mode 100644 index 0000000000..e338fc2bca --- /dev/null +++ b/crates/aprender-review-experiment/src/terms_tests.rs @@ -0,0 +1,352 @@ +use super::*; + +const TERMS: &str = r#"{"url":"https://www.anthropic.com/legal/commercial-terms","effective_date":"2025-06-17","fetched_at":"2026-09-25T17:00:00Z"}"#; +const LOCAL_SHA: &str = "3f1a9c0b5e2d4f6a8b7c9d0e1f2a3b4c5d6e7f8091a2b3c4d5e6f708192a3b4c"; + +fn row(id: &str, provider: &str, model: &str, channel: &str, terms: &str) -> String { + format!( + r#"{{"schema":"agent-trace-v1","trace_id":"{id}","provider":"{provider}","model_id":"{model}","access_channel":"{channel}","terms_ref":{terms}}}"# + ) +} + +fn terms(url: &str, effective: &str, fetched: &str) -> String { + format!(r#"{{"url":"{url}","effective_date":"{effective}","fetched_at":"{fetched}"}}"#) +} + +/// One row per channel, all correctly tagged. +fn mixed_batch() -> String { + [ + row( + "t1", + "anthropic", + "claude-sonnet-5", + "claude-code-cli", + TERMS, + ), + row( + "t2", + "anthropic", + "claude-haiku-4-5-20251001", + "anthropic-api", + TERMS, + ), + row("t3", "google", "gemini-3.1-pro-high", "antigravity", TERMS), + row("t4", "google", "gemini-3.1-pro", "gemini-api", TERMS), + row("t5", "google", "gemini-3.1-pro", "vertex", TERMS), + row("t6", "local", LOCAL_SHA, "apr-serve", TERMS), + ] + .join("\n") +} + +fn ids(rows: &[&str]) -> Vec { + rows.iter() + .map(|r| { + let v: Value = serde_json::from_str(r).expect("row is JSON"); + v["trace_id"].as_str().expect("trace_id").to_string() + }) + .collect() +} + +#[test] +fn falsify_terms_001_pool_rebuild_excluding_a_provider() { + let batch = mixed_batch(); + let ex = Exclude::from_args(&["--exclude-provider", "google"]).expect("valid args"); + let r = rebuild(&batch, &ex); + assert!(r.refused.is_empty(), "every row is tagged: {:?}", r.refused); + assert_eq!(ids(&r.kept), ["t1", "t2", "t6"]); + assert_eq!(ids(&r.excluded), ["t3", "t4", "t5"]); + for kept in &r.kept { + let t = tags(&serde_json::from_str(kept).expect("json")).expect("tagged"); + assert_ne!( + t.provider, + Provider::Google, + "a google row survived: {kept}" + ); + assert!( + batch.lines().any(|l| l == *kept), + "kept row rewritten: {kept}" + ); + } +} + +#[test] +fn falsify_terms_002_channel_exclusion_is_narrower_than_provider() { + let batch = mixed_batch(); + let ex = Exclude::from_args(&["--exclude-channel", "antigravity"]).expect("valid args"); + let r = rebuild(&batch, &ex); + assert_eq!(ids(&r.excluded), ["t3"]); + assert_eq!(ids(&r.kept), ["t1", "t2", "t4", "t5", "t6"]); +} + +#[test] +fn falsify_terms_003_identity_case_table() { + let https = "https://x.example/terms"; + let at = "2026-09-25T17:00:00Z"; + let prefixed = format!("sha256:{LOCAL_SHA}"); + let upper = LOCAL_SHA.to_uppercase(); + // (trace_id, provider, model_id, channel, terms_ref, must be tagged?) + let cases: Vec<(&str, &str, &str, &str, String, bool)> = vec![ + ( + "ok-a", + "anthropic", + "claude-opus-5-5", + "claude-code-cli", + TERMS.into(), + true, + ), + ( + "ok-g", + "google", + "gemini-3.1-pro-high", + "antigravity", + TERMS.into(), + true, + ), + ("ok-l", "local", LOCAL_SHA, "apr-serve", TERMS.into(), true), + ("ok-l2", "local", &prefixed, "apr-serve", TERMS.into(), true), + ( + "alias", + "anthropic", + "claude-sonnet", + "claude-code-cli", + TERMS.into(), + false, + ), + ( + "bare", + "anthropic", + "sonnet", + "claude-code-cli", + TERMS.into(), + false, + ), + ( + "gem-alias", + "google", + "gemini-pro", + "gemini-api", + TERMS.into(), + false, + ), + ( + "unk-model", + "anthropic", + "unknown", + "claude-code-cli", + TERMS.into(), + false, + ), + ( + "empty-model", + "anthropic", + "", + "claude-code-cli", + TERMS.into(), + false, + ), + ( + "unk-prov", + "openai", + "gpt-5", + "anthropic-api", + TERMS.into(), + false, + ), + ( + "unk-chan", + "anthropic", + "claude-opus-5-5", + "web", + TERMS.into(), + false, + ), + ( + "mismatch", + "google", + "gemini-3.1-pro", + "claude-code-cli", + TERMS.into(), + false, + ), + ( + "local-alias", + "local", + "qwen3.5-4b", + "apr-serve", + TERMS.into(), + false, + ), + ( + "local-short", + "local", + "sha256:3f1a9c", + "apr-serve", + TERMS.into(), + false, + ), + ( + "local-upper", + "local", + &upper, + "apr-serve", + TERMS.into(), + false, + ), + ( + "no-terms", + "anthropic", + "claude-opus-5-5", + "claude-code-cli", + "null".into(), + false, + ), + ( + "http", + "anthropic", + "claude-opus-5-5", + "claude-code-cli", + terms("http://x.example/terms", "2025-06-17", at), + false, + ), + ( + "bad-date", + "anthropic", + "claude-opus-5-5", + "claude-code-cli", + terms(https, "June 17", at), + false, + ), + ( + "bad-month", + "anthropic", + "claude-opus-5-5", + "claude-code-cli", + terms(https, "2025-13-17", at), + false, + ), + ( + "bad-fetch", + "anthropic", + "claude-opus-5-5", + "claude-code-cli", + terms(https, "2025-06-17", "yesterday"), + false, + ), + ( + "fetch-no-zone", + "anthropic", + "claude-opus-5-5", + "claude-code-cli", + terms(https, "2025-06-17", "2026-09-25T17:00:00"), + false, + ), + // Terms announced ahead of their effective date are still a valid snapshot. + ( + "fetch-before-effective", + "anthropic", + "claude-opus-5-5", + "claude-code-cli", + terms(https, "2026-10-01", at), + true, + ), + ]; + for (id, p, m, c, t, want) in &cases { + let v: Value = serde_json::from_str(&row(id, p, m, c, t)).expect("case is JSON"); + let got = tags(&v); + assert_eq!(got.is_ok(), *want, "case {id} -> {got:?}"); + if let Err(gaps) = got { + assert!(!gaps.is_empty(), "case {id}: a refusal names its gaps"); + } + } + let missing: Value = serde_json::from_str( + r#"{"schema":"agent-trace-v1","trace_id":"missing","model_id":"claude-opus-5-5","access_channel":"claude-code-cli"}"#, + ) + .expect("json"); + let gaps = tags(&missing).expect_err("no provider, no terms_ref"); + assert!(gaps.contains(&Gap::Missing("provider")), "{gaps:?}"); + assert!(gaps.contains(&Gap::Missing("terms_ref")), "{gaps:?}"); +} + +#[test] +fn falsify_terms_004_untagged_row_is_refused_never_kept() { + let batch = format!( + "{}\n{}\n\n", + row( + "good", + "anthropic", + "claude-opus-5-5", + "claude-code-cli", + TERMS + ), + row("alias", "google", "gemini-pro", "gemini-api", TERMS), + ); + // Excluding anthropic must not let an untagged row into the pool. + let ex = Exclude::from_args(&["--exclude-provider", "anthropic"]).expect("valid"); + let r = rebuild(&batch, &ex); + assert!(r.kept.is_empty(), "kept: {:?}", r.kept); + assert_eq!(ids(&r.excluded), ["good"]); + assert_eq!(r.refused.len(), 1); + assert!(!r.refused[0].1.is_empty(), "a refusal names its gaps"); + // With nothing excluded the untagged row is still refused. + let r = rebuild(&batch, &Exclude::default()); + assert_eq!(ids(&r.kept), ["good"]); + assert_eq!(r.refused.len(), 1); + // A line that is not JSON is refused too, never dropped silently. + let r = rebuild("not json\n", &Exclude::default()); + assert!(r.kept.is_empty()); + assert_eq!(r.refused.len(), 1); +} + +#[test] +fn exclude_args_reject_unknown_values_and_flags() { + assert!(Exclude::from_args(&["--exclude-provider", "openai"]).is_err()); + assert!(Exclude::from_args(&["--exclude-channel", "web"]).is_err()); + assert!(Exclude::from_args(&["--exclude-provider"]).is_err()); + assert!(Exclude::from_args(&["--include-provider", "google"]).is_err()); + let ex = Exclude::from_args(&[ + "--exclude-provider", + "anthropic", + "--exclude-channel", + "vertex", + "--exclude-provider", + "google", + ]) + .expect("valid"); + assert_eq!(ex.providers, [Provider::Anthropic, Provider::Google]); + assert_eq!(ex.channels, [Channel::Vertex]); + assert_eq!(Exclude::from_args(&[]).expect("empty"), Exclude::default()); +} + +#[test] +fn public_release_keeps_only_local_rows() { + let batch = mixed_batch(); + let r = rebuild(&batch, &Exclude::public_release()); + assert_eq!(ids(&r.kept), ["t6"]); + assert_eq!(r.excluded.len(), 5); +} + +#[test] +fn every_channel_maps_to_its_provider() { + let table = [ + ("anthropic-api", Provider::Anthropic), + ("claude-code-cli", Provider::Anthropic), + ("antigravity", Provider::Google), + ("gemini-api", Provider::Google), + ("vertex", Provider::Google), + ("apr-serve", Provider::Local), + ]; + for (name, p) in table { + let c = Channel::parse(name).expect("known channel"); + assert_eq!(c.provider(), p, "{name}"); + assert_eq!(c.as_str(), name); + } + assert_eq!(table.len(), Channel::ALL.len()); + for (name, p) in [ + ("anthropic", Provider::Anthropic), + ("google", Provider::Google), + ("local", Provider::Local), + ] { + assert_eq!(Provider::parse(name), Some(p)); + } + assert_eq!(Provider::parse("Anthropic"), None); +} diff --git a/crates/aprender-review-experiment/src/workload.rs b/crates/aprender-review-experiment/src/workload.rs new file mode 100644 index 0000000000..8ed4244b5e --- /dev/null +++ b/crates/aprender-review-experiment/src/workload.rs @@ -0,0 +1,200 @@ +//! Review workload profile (PRM-001 v3 PRM-S2, §0.2 and §2.4; contract +//! `workload-profile-v1`, FALSIFY-WLP-001..004). +//! +//! The profile reads `agent-trace-v1` rows and reports, per lane, the +//! distribution of `tokens.input` and `tokens.output`. It decides §0.2's claim +//! that review is prefill-heavy (2k–32k tokens in, a short verdict out) and +//! sets the replay strata weights over distinct diffs. + +use std::collections::BTreeMap; + +use serde_json::{json, Value}; + +use crate::stats::percentile; + +/// §2.4 replay strata upper bounds, in input tokens. +pub const STRATA: [u64; 4] = [2000, 8000, 16000, 32000]; + +/// Per-lane token quantiles over the measured rows. +#[derive(Debug, Clone, PartialEq)] +pub struct LaneProfile { + pub lane: String, + pub n: usize, + pub input_p50: f64, + pub input_p95: f64, + pub output_p50: f64, + pub output_p95: f64, +} + +impl LaneProfile { + /// §0.2 "a diff of 2k–32k tokens goes in". + #[must_use] + pub fn in_band(&self) -> bool { + self.input_p50 >= STRATA[0] as f64 && self.input_p95 <= STRATA[3] as f64 + } + + /// §0.2 "prefill-heavy": the long output is shorter than the typical input. + #[must_use] + pub fn prefill_heavy(&self) -> bool { + self.output_p95 < self.input_p50 + } +} + +/// The profile of one set of trace rows. +#[derive(Debug, Clone, PartialEq)] +pub struct Profile { + pub lanes: Vec, + /// Rows that could not be measured (capture failed, unparsed, no tokens). + pub excluded: usize, + /// Distinct `diff_sha256` work units with a measurement. + pub units: usize, + /// Units over the top stratum; not weighted. + pub over_32k: usize, + /// Weight per stratum bound, over the in-range units. + pub strata: Vec<(u64, f64)>, + /// §0.2: `None` when no lane reached `min_n`. + pub claim: Option, + /// Why the claim is not confirmed (S-code lines), empty when it is. + pub refusals: Vec, +} + +/// Build the profile of JSONL `rows`; lanes with fewer than `min_n` measured +/// rows are reported but do not decide the claim. +#[must_use] +pub fn profile(rows: &str, min_n: usize) -> Profile { + let mut by_lane: BTreeMap, Vec)> = BTreeMap::new(); + let mut size: BTreeMap = BTreeMap::new(); + let mut excluded = 0; + for line in rows.lines().filter(|l| !l.trim().is_empty()) { + let Some((lane, diff, input, output)) = measured(line) else { + excluded += 1; + continue; + }; + let e = by_lane.entry(lane).or_default(); + e.0.push(input as f64); + e.1.push(output as f64); + let s = size.entry(diff).or_insert(0); + *s = (*s).max(input); + } + let lanes: Vec = by_lane + .into_iter() + .map(|(lane, (i, o))| LaneProfile { + lane, + n: i.len(), + input_p50: percentile(&i, 0.5), + input_p95: percentile(&i, 0.95), + output_p50: percentile(&o, 0.5), + output_p95: percentile(&o, 0.95), + }) + .collect(); + let (claim, refusals) = decide(&lanes, min_n); + let over_32k = size.values().filter(|&&s| s > STRATA[3]).count(); + let in_range = size.len() - over_32k; + let strata = STRATA + .iter() + .enumerate() + .map(|(k, &b)| { + let lo = if k == 0 { 0 } else { STRATA[k - 1] }; + let c = size.values().filter(|&&s| s > lo && s <= b).count(); + let w = if in_range == 0 { + 0.0 + } else { + c as f64 / in_range as f64 + }; + (b, w) + }) + .collect(); + Profile { + lanes, + excluded, + units: size.len(), + over_32k, + strata, + claim, + refusals, + } +} + +/// `(lane, diff_sha256, input, output)` of a measured row; `None` for a failed +/// capture, a line that is not a row, or a row without positive input and an +/// output count. +fn measured(line: &str) -> Option<(String, String, u64, u64)> { + let v: Value = serde_json::from_str(line).ok()?; + if v["trace_status"].as_str() == Some("capture_failed") { + return None; + } + let input = v["tokens"]["input"].as_u64().filter(|&n| n > 0)?; + let output = v["tokens"]["output"].as_u64()?; + Some(( + v["lane"].as_str()?.to_string(), + v["diff_sha256"].as_str()?.to_string(), + input, + output, + )) +} + +fn decide(lanes: &[LaneProfile], min_n: usize) -> (Option, Vec) { + let deciding: Vec<&LaneProfile> = lanes.iter().filter(|l| l.n >= min_n).collect(); + if deciding.is_empty() { + return ( + None, + vec![format!( + "S-14 §0.2 undecided: no lane has min_n = {min_n} measured rows" + )], + ); + } + let mut refusals = Vec::new(); + for l in deciding { + if !l.in_band() { + refusals.push(format!( + "§0.2 falsified by {}: input p50 {} / p95 {} outside 2000–32000", + l.lane, l.input_p50, l.input_p95 + )); + } + if !l.prefill_heavy() { + refusals.push(format!( + "§0.2 falsified by {}: output p95 {} ≥ input p50 {}", + l.lane, l.output_p95, l.input_p50 + )); + } + } + (Some(refusals.is_empty()), refusals) +} + +/// The receipt's `workload` block (§7 template) plus the claim, strata and +/// exclusions it was decided on. +#[must_use] +#[allow(clippy::disallowed_methods)] // `json!` expands to an `unwrap` of an infallible `to_value` +pub fn render(p: &Profile) -> Value { + let pair = |f: fn(&LaneProfile) -> [f64; 2]| -> serde_json::Map { + p.lanes + .iter() + .map(|l| (l.lane.clone(), json!(f(l)))) + .collect() + }; + let strata: serde_json::Map = p + .strata + .iter() + .map(|(b, w)| (format!("{}k", b / 1000), json!(w))) + .collect(); + let n: serde_json::Map = p + .lanes + .iter() + .map(|l| (l.lane.clone(), json!(l.n))) + .collect(); + json!({"workload": { + "input_tokens_p50_p95": pair(|l| [l.input_p50, l.input_p95]), + "output_tokens_p50_p95": pair(|l| [l.output_p50, l.output_p95]), + "n": n, + "claim_0_2": p.claim, + "refusals": p.refusals, + "strata": strata, + "units": p.units, + "over_32k": p.over_32k, + "excluded": p.excluded, + }}) +} + +#[cfg(test)] +#[path = "workload_tests.rs"] +mod tests; diff --git a/crates/aprender-review-experiment/src/workload_tests.rs b/crates/aprender-review-experiment/src/workload_tests.rs new file mode 100644 index 0000000000..eeeb470be6 --- /dev/null +++ b/crates/aprender-review-experiment/src/workload_tests.rs @@ -0,0 +1,170 @@ +// `json!` expands to an `unwrap` of an infallible `to_value`. +#![allow(clippy::disallowed_methods)] + +use super::*; + +fn row(lane: &str, diff: &str, input: u64, output: u64) -> String { + json!({"lane": lane, "diff_sha256": diff, "trace_status": "ok", + "tokens": {"input": input, "output": output}}) + .to_string() +} + +fn jsonl(rows: &[String]) -> String { + rows.iter().map(|r| format!("{r}\n")).collect() +} + +fn lane<'a>(p: &'a Profile, name: &str) -> &'a LaneProfile { + p.lanes + .iter() + .find(|l| l.lane == name) + .unwrap_or_else(|| panic!("lane {name} missing from {:?}", p.lanes)) +} + +/// 20 rows of one lane: inputs 1000, 2000, …, 20000 and outputs 10, 20, …, 200. +fn ramp(name: &str) -> Vec { + (1..=20u64) + .map(|i| row(name, &format!("d{i}"), 1000 * i, 10 * i)) + .collect() +} + +/// FALSIFY-WLP-001: quantiles are nearest rank on measured rows; a failed +/// capture, an unparseable line, a zero input and a missing output are +/// excluded and counted, and move nothing. +#[test] +fn falsify_wlp_001_quantiles_are_nearest_rank_on_measured_rows_only() { + let clean = ramp("qwen"); + let p = profile(&jsonl(&clean), 1); + let q = lane(&p, "qwen"); + assert_eq!(q.n, 20); + // nearest rank: p50 = rank 10, p95 = rank 19 + assert_eq!((q.input_p50, q.input_p95), (10_000.0, 19_000.0)); + assert_eq!((q.output_p50, q.output_p95), (100.0, 190.0)); + assert_eq!(p.excluded, 0); + + let mut dirty = clean.clone(); + dirty.push( + json!({"lane": "qwen", "diff_sha256": "x1", "trace_status": "capture_failed", + "tokens": {"input": 1, "output": 1}}) + .to_string(), + ); + dirty.push("{not json".into()); + dirty.push(row("qwen", "x2", 0, 0)); + dirty.push(json!({"lane": "qwen", "diff_sha256": "x3", "tokens": {"input": 5}}).to_string()); + dirty.push(json!({"lane": "qwen", "diff_sha256": "x4"}).to_string()); + let d = profile(&jsonl(&dirty), 1); + assert_eq!(d.excluded, 5); + assert_eq!(lane(&d, "qwen"), q, "an excluded row moved the quantiles"); + assert_eq!(d.units, p.units); +} + +/// FALSIFY-WLP-002: §0.2 is decided per lane on lanes with min_n rows. +#[test] +fn falsify_wlp_002_claim_is_decided_per_lane_and_refused_when_thin() { + // ramp: input p50 10000 ≥ 2000, p95 19000 ≤ 32000, output p95 190 < 10000 + let mut rows = ramp("qwen"); + rows.extend(ramp("sonnet-5")); + let p = profile(&jsonl(&rows), 20); + assert!(lane(&p, "qwen").in_band() && lane(&p, "qwen").prefill_heavy()); + assert_eq!(p.claim, Some(true)); + assert!(p.refusals.is_empty(), "{:?}", p.refusals); + + // a decode-heavy lane: output p95 ≥ input p50 falsifies, naming the lane + let mut chatty = rows.clone(); + chatty.extend((1..=20u64).map(|i| row("gemini", &format!("d{i}"), 4000, 5000))); + let c = profile(&jsonl(&chatty), 20); + assert!(lane(&c, "gemini").in_band() && !lane(&c, "gemini").prefill_heavy()); + assert_eq!(c.claim, Some(false)); + assert!( + c.refusals.iter().any(|r| r.contains("gemini")), + "{:?}", + c.refusals + ); + // the edge: output p95 == input p50 is not prefill-heavy + let edge = profile(&jsonl(&[row("e", "d", 3000, 3000)]), 1); + assert!(!lane(&edge, "e").prefill_heavy()); + assert_eq!(edge.claim, Some(false)); + + // out of band, both ends; the bounds are inclusive + for (input, band) in [(1999, false), (2000, true), (32_000, true), (32_001, false)] { + let b = profile(&jsonl(&[row("b", "d", input, 10)]), 1); + assert_eq!(lane(&b, "b").in_band(), band, "input {input}"); + assert_eq!(b.claim, Some(band), "input {input}"); + } + + // a thin decode-heavy lane does not decide + let mut thin = rows.clone(); + thin.push(row("haiku", "d1", 3000, 9000)); + let t = profile(&jsonl(&thin), 20); + assert_eq!(lane(&t, "haiku").n, 1); + assert_eq!(t.claim, Some(true)); + + // no lane at min_n: undecided, refused, never confirmed + let u = profile(&jsonl(&rows), 21); + assert_eq!(u.claim, None); + assert!( + u.refusals.iter().any(|r| r.contains("min_n")), + "{:?}", + u.refusals + ); + let e = profile("", 1); + assert_eq!(e.claim, None); + assert!(e.lanes.is_empty()); +} + +/// FALSIFY-WLP-003: strata weights are over distinct diffs at their largest +/// lane measurement. +#[test] +fn falsify_wlp_003_strata_weights_are_over_distinct_diffs() { + let rows = [ + // one diff, three lanes: sized by its largest measurement, 8001 → 16k + row("qwen", "a", 7000, 10), + row("sonnet-5", "a", 8001, 10), + row("gemini", "a", 7500, 10), + // inclusive bounds + row("qwen", "b", 2000, 10), + row("qwen", "c", 2001, 10), + row("qwen", "d", 32_000, 10), + // over the top stratum: counted, not weighted + row("qwen", "e", 40_000, 10), + row("sonnet-5", "e", 1000, 10), + ]; + let p = profile(&jsonl(&rows), 1); + assert_eq!(p.units, 5); + assert_eq!(p.over_32k, 1); + assert_eq!( + p.strata, + vec![(2000, 0.25), (8000, 0.25), (16000, 0.25), (32000, 0.25)] + ); + let sum: f64 = p.strata.iter().map(|s| s.1).sum(); + assert!((sum - 1.0).abs() < 1e-12); + + // a lane count does not weight a diff + let mut many = rows.to_vec(); + many.extend((0..10).map(|i| row(&format!("l{i}"), "b", 2000, 10))); + assert_eq!(profile(&jsonl(&many), 1).strata, p.strata); + + // nothing in range: no weights, not NaN + let o = profile(&jsonl(&[row("q", "z", 50_000, 10)]), 1); + assert_eq!(o.over_32k, 1); + assert!(o.strata.iter().all(|s| s.1 == 0.0), "{:?}", o.strata); +} + +/// FALSIFY-WLP-004: the rendered block matches the v3 §7 receipt template. +#[test] +fn falsify_wlp_004_receipt_block_matches_the_v3_template() { + let p = profile(&jsonl(&ramp("qwen")), 20); + let v = render(&p); + assert_eq!( + v["workload"]["input_tokens_p50_p95"]["qwen"], + json!([10_000.0, 19_000.0]) + ); + assert_eq!( + v["workload"]["output_tokens_p50_p95"]["qwen"], + json!([100.0, 190.0]) + ); + assert_eq!(v["workload"]["claim_0_2"], json!(true)); + assert_eq!(v["workload"]["excluded"], json!(0)); + assert_eq!(v["workload"]["strata"]["16k"], json!(0.4)); + let u = render(&profile(&jsonl(&ramp("qwen")), 99)); + assert_eq!(u["workload"]["claim_0_2"], Value::Null); +} diff --git a/crates/aprender-serve/src/api/chat_logprobs.rs b/crates/aprender-serve/src/api/chat_logprobs.rs new file mode 100644 index 0000000000..053ff822d6 --- /dev/null +++ b/crates/aprender-serve/src/api/chat_logprobs.rs @@ -0,0 +1,91 @@ +//! OpenAI `logprobs` / `top_logprobs` on `/v1/chat/completions` (#4026). +//! +//! Only the session engine records per-step logprobs, so only the Qwen3.5 arm +//! answers them. Every other arm, and the streaming path, refuses the request +//! by name: a reply without the logprobs it was asked for would read as a +//! model that has none. +//! +//! Each `content` entry is OpenAI's shape (`token`, `logprob`, `bytes`, +//! `top_logprobs`) plus three fields PRM C11 (`sparse-logits-v1`) needs and +//! OpenAI does not carry: `token_id`, the top ids, and `logsumexp_full`, the +//! full-vocab log-sum-exp at T = 1.0 from which the retained mass and the +//! residual tail follow. All values are from the forward's logits, before any +//! repetition penalty or sampler. + +use super::ChatCompletionRequest; +use crate::gguf::StepLogprobs; + +/// OpenAI's ceiling on `top_logprobs`. +pub(crate) const MAX_TOP_LOGPROBS: usize = 20; + +/// `Ok(None)` when the request asks for no logprobs, `Ok(Some(n))` for `n` +/// alternatives per token, `Err` for a request that cannot be answered as +/// asked. +pub(crate) fn requested_top_logprobs(req: &ChatCompletionRequest) -> Result, String> { + let wanted = req.logprobs.unwrap_or(false); + match (wanted, req.top_logprobs) { + (false, None) => Ok(None), + (false, Some(_)) => Err("`top_logprobs` requires `logprobs: true` (#4026)".to_string()), + (true, Some(n)) if n > MAX_TOP_LOGPROBS => Err(format!( + "`top_logprobs` is {n}; at most {MAX_TOP_LOGPROBS} are supported (#4026)" + )), + (true, _) if req.stream => Err( + "`logprobs` on a streamed chat completion is not recorded yet; send `stream: false` \ + (#4026)" + .to_string(), + ), + (true, n) => Ok(Some(n.unwrap_or(0))), + } +} + +/// The refusal every arm but the session engine returns for `logprobs: true`. +pub(crate) fn unsupported_backend_reason(architecture: Option<&str>) -> String { + format!( + "`logprobs` are recorded only by the Qwen3.5 session engine; this model ({}) is served \ + by a backend that cannot record them, so the request is refused rather than answered \ + without them (#4026)", + architecture.unwrap_or("unknown architecture") + ) +} + +/// The `choices[].logprobs` object for the generated `steps`. +/// +/// `steps` must already be trimmed to the ids the reply reports; `n_top` of +/// each step's recorded alternatives are emitted. +pub(crate) fn chat_logprobs_json( + steps: &[StepLogprobs], + n_top: usize, + decode: &dyn Fn(u32) -> String, +) -> serde_json::Value { + let content: Vec = steps + .iter() + .map(|s| { + let top: Vec = s + .top + .iter() + .take(n_top) + .map(|t| { + serde_json::json!({ + "token": decode(t.token_id), + "token_id": t.token_id, + "logprob": t.logprob, + "bytes": serde_json::Value::Null, + }) + }) + .collect(); + serde_json::json!({ + "token": decode(s.chosen), + "token_id": s.chosen, + "logprob": s.chosen_logprob, + "bytes": serde_json::Value::Null, + "top_logprobs": top, + "logsumexp_full": s.logsumexp_full, + }) + }) + .collect(); + serde_json::json!({ "content": content }) +} + +#[cfg(test)] +#[path = "chat_logprobs_tests.rs"] +mod tests; diff --git a/crates/aprender-serve/src/api/chat_logprobs_tests.rs b/crates/aprender-serve/src/api/chat_logprobs_tests.rs new file mode 100644 index 0000000000..199c2d37e4 --- /dev/null +++ b/crates/aprender-serve/src/api/chat_logprobs_tests.rs @@ -0,0 +1,105 @@ +use super::*; +use crate::gguf::TopLogprob; + +fn req(logprobs: Option, top: Option, stream: bool) -> ChatCompletionRequest { + ChatCompletionRequest { + logprobs, + top_logprobs: top, + stream, + ..Default::default() + } +} + +#[test] +fn requested_top_logprobs_case_table() { + // (logprobs, top_logprobs, stream) -> Ok(want) | Err(substring) + let table: &[( + Option, + Option, + bool, + Result, &str>, + )] = &[ + (None, None, false, Ok(None)), + (Some(false), None, false, Ok(None)), + (None, None, true, Ok(None)), + (Some(true), None, false, Ok(Some(0))), + (Some(true), Some(0), false, Ok(Some(0))), + (Some(true), Some(20), false, Ok(Some(20))), + (Some(true), Some(21), false, Err("at most 20")), + (None, Some(5), false, Err("requires `logprobs: true`")), + ( + Some(false), + Some(5), + false, + Err("requires `logprobs: true`"), + ), + (Some(true), Some(5), true, Err("stream")), + ]; + for (lp, top, stream, want) in table { + let got = requested_top_logprobs(&req(*lp, *top, *stream)); + match (want, &got) { + (Ok(w), Ok(g)) => assert_eq!(w, g, "{lp:?} {top:?} {stream}"), + (Err(s), Err(e)) => assert!(e.contains(s), "{lp:?} {top:?} {stream}: {e}"), + _ => panic!("{lp:?} {top:?} {stream}: want {want:?}, got {got:?}"), + } + } +} + +#[test] +fn refusal_names_the_architecture() { + assert!(unsupported_backend_reason(Some("llama")).contains("(llama)")); + assert!(unsupported_backend_reason(None).contains("unknown architecture")); +} + +fn step(chosen: u32, top: &[(u32, f32)], chosen_logprob: f32, lse: f32) -> StepLogprobs { + StepLogprobs { + step: 0, + chosen, + top: top + .iter() + .map(|&(token_id, logprob)| TopLogprob { + token_id, + logit: logprob + lse, + logprob, + }) + .collect(), + chosen_logprob, + logsumexp_full: lse, + } +} + +/// Every field a C11 row needs is present, `n_top` truncates the +/// alternatives only, and a sampled token outside the top-K keeps its own +/// logprob instead of borrowing the top entry's. +#[test] +fn chat_logprobs_json_carries_ids_logprobs_and_lse() { + let steps = [ + step(7, &[(7, -0.1), (3, -2.5), (9, -3.0)], -0.1, 12.5), + step(42, &[(1, -0.2), (2, -1.9), (4, -2.2)], -6.0, 11.0), + ]; + let v = chat_logprobs_json(&steps, 2, &|id| format!("t{id}")); + let c = v["content"].as_array().expect("content"); + assert_eq!(c.len(), 2); + assert_eq!(c[0]["token"], "t7"); + assert_eq!(c[0]["token_id"], 7); + assert_eq!(c[0]["logsumexp_full"], 12.5); + assert_eq!(c[0]["top_logprobs"].as_array().expect("top").len(), 2); + assert_eq!(c[0]["top_logprobs"][1]["token_id"], 3); + assert_eq!(c[0]["top_logprobs"][1]["token"], "t3"); + assert_eq!(c[1]["token_id"], 42); + assert_eq!(c[1]["logprob"], -6.0); + assert!(c[1]["top_logprobs"] + .as_array() + .expect("top") + .iter() + .all(|t| t["token_id"] != 42)); + let none = chat_logprobs_json(&steps, 0, &|id| id.to_string()); + assert!(none["content"][0]["top_logprobs"] + .as_array() + .expect("top") + .is_empty()); + assert_eq!( + none["content"][0]["logprob"].as_f64(), + Some(f64::from(-0.1_f32)) + ); +} diff --git a/crates/aprender-serve/src/api/cuda_chat_backend.rs b/crates/aprender-serve/src/api/cuda_chat_backend.rs index 6f229a0de7..3fe61f9253 100644 --- a/crates/aprender-serve/src/api/cuda_chat_backend.rs +++ b/crates/aprender-serve/src/api/cuda_chat_backend.rs @@ -942,11 +942,26 @@ pub async fn openai_chat_completions_handler( return fail_response(&state, StatusCode::BAD_REQUEST, reason); } + // #4026: a logprobs request that cannot be answered as asked is refused here. + let wants_logprobs = match super::chat_logprobs::requested_top_logprobs(&request) { + Ok(n) => n.is_some(), + Err(reason) => return fail_response(&state, StatusCode::BAD_REQUEST, reason), + }; + // #3571: a Qwen3.5 hybrid is answered from its resident session or not at all. if let Some(r) = try_qwen35_backend(&state, &request, &request_id, start, &cancel).await { return r; } + // #4026: no arm below records logprobs. + if wants_logprobs { + return fail_response( + &state, + StatusCode::BAD_REQUEST, + super::chat_logprobs::unsupported_backend_reason(state.model_architecture().as_deref()), + ); + } + if let Some(r) = try_qwen3_moe_backend(&state, &request, &request_id, start, &cancel) { return r; } diff --git a/crates/aprender-serve/src/api/mod.rs b/crates/aprender-serve/src/api/mod.rs index f2747b2335..9817ad7504 100644 --- a/crates/aprender-serve/src/api/mod.rs +++ b/crates/aprender-serve/src/api/mod.rs @@ -51,6 +51,7 @@ use crate::{ // aprender#2376(3): request-scoped cancellation for the generate handlers. mod cancel_scope; +mod chat_logprobs; pub(crate) use cancel_scope::cancel_on_disconnect; pub use cancel_scope::request_cancel_token; diff --git a/crates/aprender-serve/src/api/mod_create_demo.rs b/crates/aprender-serve/src/api/mod_create_demo.rs index 0036602a7f..a886a2e1ea 100644 --- a/crates/aprender-serve/src/api/mod_create_demo.rs +++ b/crates/aprender-serve/src/api/mod_create_demo.rs @@ -122,6 +122,15 @@ pub struct ChatCompletionRequest { /// Stream responses #[serde(default)] pub stream: bool, + /// OpenAI `logprobs`: return per-token log-probabilities (#4026). + /// + /// Only the session engine records them. A backend that cannot refuses the + /// request by name rather than answering without them. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub logprobs: Option, + /// OpenAI `top_logprobs`: how many alternatives per token, 0..=20. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub top_logprobs: Option, /// OpenAI `stream_options`, accepted so a client that sends /// `{"include_usage": true}` is not rejected as malformed. /// diff --git a/crates/aprender-serve/src/api/openai_handlers.rs b/crates/aprender-serve/src/api/openai_handlers.rs index 81eb375e1d..a27962c23e 100644 --- a/crates/aprender-serve/src/api/openai_handlers.rs +++ b/crates/aprender-serve/src/api/openai_handlers.rs @@ -450,6 +450,8 @@ mod pmat821_chat_handler_threading_tests { ignore_eos: None, n: crate::api::ChoiceCount::ONE, stream: false, + logprobs: None, + top_logprobs: None, stop: None, user: None, tools: None, @@ -677,6 +679,42 @@ pub(crate) fn build_chat_response( timings: Option, used_gpu: Option, ) -> Response { + Json(chat_response_body( + request_id, + model, + text, + prompt_tokens, + completion_tokens, + max_tokens, + stops, + trace_level, + latency, + tools, + tool_choice, + timings, + used_gpu, + )) + .into_response() +} + +/// The body [`build_chat_response`] serializes, for an arm that adds to it +/// (the Qwen3.5 arm's `logprobs`, #4026). +#[allow(clippy::too_many_arguments)] +pub(crate) fn chat_response_body( + request_id: String, + model: String, + text: String, + prompt_tokens: usize, + completion_tokens: usize, + max_tokens: usize, + stops: Option<&[String]>, + trace_level: Option<&str>, + latency: Duration, + tools: Option<&[super::OpenAiTool]>, + tool_choice: Option, + timings: Option, + used_gpu: Option, +) -> ChatCompletionResponse { let (brick_trace, step_trace, layer_trace) = build_trace_data( trace_level, latency.as_micros() as u64, @@ -701,7 +739,7 @@ pub(crate) fn build_chat_response( ), }; - Json(ChatCompletionResponse { + ChatCompletionResponse { used_gpu, id: request_id, object: "chat.completion".to_string(), @@ -721,8 +759,7 @@ pub(crate) fn build_chat_response( step_trace, layer_trace, timings, - }) - .into_response() + } } /// Serialize a value to an SSE event, returning `None` if serialization fails. diff --git a/crates/aprender-serve/src/api/qwen35_chat_backend.rs b/crates/aprender-serve/src/api/qwen35_chat_backend.rs index 01ae9ca528..e4fd3f8e6a 100644 --- a/crates/aprender-serve/src/api/qwen35_chat_backend.rs +++ b/crates/aprender-serve/src/api/qwen35_chat_backend.rs @@ -144,7 +144,16 @@ async fn try_qwen35_backend( let stop_tokens = stop_tokens_unless_ignore_eos(request, state.model_eos_token_id()); // The context-bounded budget, not the request's number: what is decoded and what // `finish_reason` is judged against are the same count. - let gen_config = gen_config_from_request(request, budget, stop_tokens.clone(), cancel.clone()); + let mut gen_config = + gen_config_from_request(request, budget, stop_tokens.clone(), cancel.clone()); + // #4026: validated by the handler before any arm ran. + let top_logprobs = crate::api::chat_logprobs::requested_top_logprobs(request) + .ok() + .flatten(); + if let Some(n) = top_logprobs { + // At least one alternative is recorded: `logprobs_top_k` 0 records nothing. + gen_config.logprobs_top_k = n.max(1); + } if request.stream { let (tx, rx) = tokio::sync::mpsc::channel::>(64); @@ -209,6 +218,30 @@ async fn try_qwen35_backend( let duration = start.elapsed(); state.metrics.record_success(completion_tokens, duration); + if let Some(n) = top_logprobs { + // One record per generated id; the popped stop token's record goes too. + let steps = &turn.steps[..completion_tokens.min(turn.steps.len())]; + let decode = |id: u32| decode_mapped.model.decode(&[id]); + let mut body = serde_json::to_value(chat_response_body( + request_id.to_string(), + request.model.clone(), + response_text, + prompt_token_count, + completion_tokens, + budget, + request.stop.as_deref(), + None, + duration, + request.tools.as_deref(), + request_tool_choice(request), + None, + None, + )) + .unwrap_or_default(); + body["choices"][0]["logprobs"] = + crate::api::chat_logprobs::chat_logprobs_json(steps, n, &decode); + return Some(Json(body).into_response()); + } Some(build_chat_response( request_id.to_string(), request.model.clone(), diff --git a/crates/aprender-serve/src/gguf/logprobs.rs b/crates/aprender-serve/src/gguf/logprobs.rs index 1359b34a2f..b241d3d1e6 100644 --- a/crates/aprender-serve/src/gguf/logprobs.rs +++ b/crates/aprender-serve/src/gguf/logprobs.rs @@ -34,3 +34,147 @@ pub fn logprob_of(logits: &[f32], token_id: u32) -> f32 { .ln(); logits[token_id as usize] - max_logit - log_sum_exp } + +/// One of the most likely next tokens at a step (#4026). +#[derive(Debug, Clone, PartialEq, serde::Serialize)] +pub struct TopLogprob { + /// Token ID + pub token_id: u32, + /// The raw logit the forward produced for it + pub logit: f32, + /// ln(softmax(logits)[token_id]) + pub logprob: f32, +} + +/// The distribution one generated step was chosen from (#4026): the `K` most +/// likely tokens, before any repetition penalty or sampling, and the token the +/// engine then chose. `step` 0 is the token after the prompt. +#[derive(Debug, Clone, PartialEq, serde::Serialize)] +pub struct StepLogprobs { + /// 0-based index of the generated token this step produced + pub step: usize, + /// The token the engine chose at this step + pub chosen: u32, + /// The `K` most likely tokens, most likely first + pub top: Vec, + /// ln(softmax(logits)[chosen]) on the same forward logits as `top`, so it + /// is defined when a sampled `chosen` is outside the top `K` (#4026, PRM C11) + pub chosen_logprob: f32, + /// Full-vocab log-sum-exp of those logits at T = 1.0: `top` plus this gives + /// the retained mass and the residual tail `sparse-logits-v1` stores + pub logsumexp_full: f32, +} + +/// Full-vocab log-sum-exp of `logits`, NaNs skipped, accumulated in f64 so a +/// 150k-entry vocab does not lose the tail to f32 rounding (#4026). +#[must_use] +pub fn logsumexp_full(logits: &[f32]) -> f32 { + let max = logits + .iter() + .copied() + .filter(|x| !x.is_nan()) + .fold(f32::NEG_INFINITY, f32::max); + if !max.is_finite() { + return max; + } + let sum: f64 = logits + .iter() + .filter(|x| !x.is_nan()) + .map(|&x| f64::from(x - max).exp()) + .sum(); + (f64::from(max) + sum.ln()) as f32 +} + +/// The `k` most likely tokens in `logits`, most likely first (#4026). +/// +/// Ties go to the lower token id, as [`crate::gguf::ops::argmax`] breaks them, +/// so with no penalty the first entry IS the greedy choice. NaN logits are +/// never ranked. `logprob` is `logit` minus [`logsumexp_full`]. +#[must_use] +pub fn top_k_logprobs(logits: &[f32], k: usize) -> Vec { + if k == 0 { + return Vec::new(); + } + // The step record's `logsumexp_full`, so Σexp(top) and the tail a reader + // derives from the two agree exactly (PRM C11). + let lse = logsumexp_full(logits); + let mut ranked: Vec<(u32, f32)> = logits + .iter() + .enumerate() + .filter(|(_, x)| !x.is_nan()) + .map(|(i, &x)| (i as u32, x)) + .collect(); + let k = k.min(ranked.len()); + let by_rank = |a: &(u32, f32), b: &(u32, f32)| b.1.total_cmp(&a.1).then(a.0.cmp(&b.0)); + if k < ranked.len() { + ranked.select_nth_unstable_by(k, by_rank); + ranked.truncate(k); + } + ranked.sort_by(by_rank); + ranked + .into_iter() + .map(|(token_id, logit)| TopLogprob { + token_id, + logit, + logprob: logit - lse, + }) + .collect() +} + +#[cfg(test)] +mod tests_4026 { + use super::*; + + fn ids(top: &[TopLogprob]) -> Vec { + top.iter().map(|t| t.token_id).collect() + } + + #[test] + fn top_k_logprobs_case_table() { + let logits = [0.5, 2.0, -1.0, 2.0, 1.0]; + // (k, expected ids): ties go to the lower id, as ops::argmax. + let table: &[(usize, &[u32])] = &[ + (0, &[]), + (1, &[1]), + (2, &[1, 3]), + (3, &[1, 3, 4]), + (5, &[1, 3, 4, 0, 2]), + (99, &[1, 3, 4, 0, 2]), + ]; + for &(k, want) in table { + assert_eq!(ids(&top_k_logprobs(&logits, k)), want, "k={k}"); + } + assert_eq!( + top_k_logprobs(&logits, 1)[0].token_id, + crate::gguf::ops::argmax(&logits), + "top-1 is the greedy choice" + ); + } + + #[test] + fn top_k_logprobs_agree_with_logprob_of_and_sum_to_one() { + let logits = [0.5_f32, 2.0, -1.0, 2.0, 1.0]; + let all = top_k_logprobs(&logits, logits.len()); + for t in &all { + assert!((t.logprob - logprob_of(&logits, t.token_id)).abs() < 1e-6); + assert_eq!(t.logit, logits[t.token_id as usize]); + } + let total: f32 = all.iter().map(|t| t.logprob.exp()).sum(); + assert!((total - 1.0).abs() < 1e-5, "sum={total}"); + } + + #[test] + fn top_k_logprobs_never_rank_nan() { + let logits = [f32::NAN, 1.0, f32::NAN, 0.0]; + let top = top_k_logprobs(&logits, 4); + assert_eq!(ids(&top), vec![1, 3]); + assert!(top.iter().all(|t| t.logprob.is_finite())); + } + + #[test] + fn top_k_logprobs_survive_huge_logits() { + let top = top_k_logprobs(&[1.0e30, 0.0], 2); + assert_eq!(top[0].logprob, 0.0); + assert!(top[1].logprob.is_finite() && top[1].logprob <= -1.0e29); + } +} diff --git a/crates/aprender-serve/src/gguf/runtime.rs b/crates/aprender-serve/src/gguf/runtime.rs index a63dbb708c..1121fc7134 100644 --- a/crates/aprender-serve/src/gguf/runtime.rs +++ b/crates/aprender-serve/src/gguf/runtime.rs @@ -41,6 +41,11 @@ pub struct QuantizedGenerateConfig { pub trace: bool, /// Return per-token log probabilities (realizr#191, F-QUALITY-01) pub logprobs: bool, + /// Record the `K` most likely next tokens at every generated step + /// (`apr run --logprobs K`, #4026). 0 records nothing. Only the one + /// engine ([`crate::session::Session`]) records; a path without it + /// refuses a non-zero `K` by name rather than returning an empty list. + pub logprobs_top_k: usize, /// Cooperative cancellation signal, polled once per decode step. /// /// aprender#2376(3): without this, an abandoned HTTP request kept decoding to @@ -63,6 +68,7 @@ impl Default for QuantizedGenerateConfig { stop_tokens: Vec::new(), trace: false, logprobs: false, + logprobs_top_k: 0, cancel: CancelToken::never(), } } diff --git a/crates/aprender-serve/src/infer/inference_result.rs b/crates/aprender-serve/src/infer/inference_result.rs index b024fef522..fd3588e96f 100644 --- a/crates/aprender-serve/src/infer/inference_result.rs +++ b/crates/aprender-serve/src/infer/inference_result.rs @@ -261,6 +261,34 @@ fn mock_run_report(config: &InferenceConfig, result: &InferenceResult) -> run_re } } +/// #4026: `--logprobs K` is recorded by the one engine only. The two GGUF +/// routes known before the run not to go through it are refused here, by +/// name, before any forward runs; a route that turns out not to record (the +/// wgpu decode) is refused by the caller when no steps come back. +fn refuse_logprobs_off_engine( + gen_config: &crate::gguf::QuantizedGenerateConfig, + architecture: &str, +) -> Result<()> { + if gen_config.logprobs_top_k == 0 { + return Ok(()); + } + let route = if gen_config.trace { + "--trace (the instrumented decode loops, which keep no logits)" + } else if crate::gguf::moe_forward_handles(architecture) { + "the qwen3_moe decode (not on the session engine)" + } else { + return Ok(()); + }; + Err(RealizarError::UnsupportedOperation { + operation: "logprobs".to_string(), + reason: format!( + "--logprobs {} is not recorded by {route}; only the session engine records \ + per-step logprobs (#4026) — refused rather than reported empty", + gen_config.logprobs_top_k + ), + }) +} + /// Run GGUF model inference /// /// PMAT-236: Accepts `PreparedTokens` (compile-time enforced chat template). @@ -337,6 +365,8 @@ fn run_gguf_inference( // (M32c.2.2.2.1.2's run_qwen3_moe_generate). The dense path goes through // run_gguf_generate as before. This replaces M32c.2.1's // gguf_gpu_generate.rs short-circuit with an actual forward pass. + refuse_logprobs_off_engine(&gen_config, &model.config.architecture)?; + let _ = crate::session::take_last_turn_steps(); // #4026: never inherit an earlier turn's steps let infer_start = Instant::now(); let _ = take_generation_start(); // #3981: never inherit a mark from an earlier run let canonical_arch = crate::tensor_names::normalize_architecture(&model.config.architecture); @@ -2322,3 +2352,36 @@ mod throughput_3981_tests { } } + +#[cfg(test)] +mod tests_4026_refuse_off_engine { + use super::refuse_logprobs_off_engine; + use crate::gguf::QuantizedGenerateConfig; + + fn cfg(k: usize, trace: bool) -> QuantizedGenerateConfig { + QuantizedGenerateConfig { + logprobs_top_k: k, + trace, + ..Default::default() + } + } + + #[test] + fn logprobs_refusal_case_table() { + // (k, trace, arch, refused) + let table = [ + (0, true, "qwen3_moe", false), + (3, false, "qwen2", false), + (3, false, "qwen35", false), + (3, true, "qwen2", true), + (3, false, "qwen3_moe", true), + ]; + for (k, trace, arch, refused) in table { + let got = refuse_logprobs_off_engine(&cfg(k, trace), arch); + assert_eq!(got.is_err(), refused, "k={k} trace={trace} arch={arch}"); + if let Err(e) = got { + assert!(e.to_string().contains("--logprobs 3"), "{e}"); + } + } + } +} diff --git a/crates/aprender-serve/src/infer/mod.rs b/crates/aprender-serve/src/infer/mod.rs index a419bf248a..dca5046297 100644 --- a/crates/aprender-serve/src/infer/mod.rs +++ b/crates/aprender-serve/src/infer/mod.rs @@ -234,6 +234,11 @@ pub struct InferenceConfig { /// removes the empty `` prefill so the model reasons, and is refused by name on a /// template with no thinking mode ([`crate::chat_template::apply_thinking_mode`]). pub thinking: Option, + /// #4026: `apr run --logprobs K` — record the `K` most likely next tokens at + /// every generated step. 0 records nothing. Only the one engine records + /// ([`crate::session::take_last_turn_steps`]); a run served by any other + /// path is refused by name by the caller, never reported with an empty list. + pub logprobs_top_k: usize, } /// The top-k a SAMPLED generation uses when the caller names none (#3754). @@ -288,9 +293,17 @@ impl InferenceConfig { use_mock_backend: false, force_chat_template: false, thinking: None, + logprobs_top_k: 0, } } + /// #4026: record the `k` most likely next tokens at every generated step. + #[must_use] + pub fn with_logprobs_top_k(mut self, k: usize) -> Self { + self.logprobs_top_k = k; + self + } + /// Set the text prompt #[must_use] pub fn with_prompt(mut self, prompt: impl Into) -> Self { @@ -425,6 +438,7 @@ impl InferenceConfig { gen_config.seed = self.seed; gen_config.repeat_penalty = self.repeat_penalty; gen_config.repeat_last_n = self.repeat_last_n; + gen_config.logprobs_top_k = self.logprobs_top_k; } } diff --git a/crates/aprender-serve/src/infer/tests_clean_model_03.rs b/crates/aprender-serve/src/infer/tests_clean_model_03.rs index d4e7834878..a1219cb5be 100644 --- a/crates/aprender-serve/src/infer/tests_clean_model_03.rs +++ b/crates/aprender-serve/src/infer/tests_clean_model_03.rs @@ -194,6 +194,7 @@ fn main() { use_mock_backend: false, force_chat_template: false, thinking: None, + logprobs_top_k: 0, }; let debug = format!("{:?}", config); diff --git a/crates/aprender-serve/src/infer/tests_inference_config_02.rs b/crates/aprender-serve/src/infer/tests_inference_config_02.rs index ec5df64d58..989a5f3b9b 100644 --- a/crates/aprender-serve/src/infer/tests_inference_config_02.rs +++ b/crates/aprender-serve/src/infer/tests_inference_config_02.rs @@ -324,6 +324,7 @@ use_mock_backend: false, force_chat_template: false, thinking: None, + logprobs_top_k: 0, }; let debug_str = format!("{:?}", config); assert!(debug_str.contains("trace_verbose")); @@ -355,6 +356,7 @@ use_mock_backend: false, force_chat_template: false, thinking: None, + logprobs_top_k: 0, }; let cloned = config.clone(); assert_eq!(cloned.trace_verbose, config.trace_verbose); diff --git a/crates/aprender-serve/src/infer/tests_run_inference.rs b/crates/aprender-serve/src/infer/tests_run_inference.rs index 487fc413ec..9c006371c5 100644 --- a/crates/aprender-serve/src/infer/tests_run_inference.rs +++ b/crates/aprender-serve/src/infer/tests_run_inference.rs @@ -366,6 +366,7 @@ use_mock_backend: false, force_chat_template: false, thinking: None, + logprobs_top_k: 0, }; let debug = format!("{:?}", config); @@ -401,6 +402,7 @@ use_mock_backend: false, force_chat_template: false, thinking: None, + logprobs_top_k: 0, }; let cloned = original.clone(); diff --git a/crates/aprender-serve/src/safetensors_infer_session_tests.rs b/crates/aprender-serve/src/safetensors_infer_session_tests.rs index 31cdc42466..da0249377c 100644 --- a/crates/aprender-serve/src/safetensors_infer_session_tests.rs +++ b/crates/aprender-serve/src/safetensors_infer_session_tests.rs @@ -76,6 +76,7 @@ fn safetensors_cpu_session_generate_leaves_a_witness_entry() { stop_tokens: vec![0], trace: false, logprobs: false, + logprobs_top_k: 0, cancel: crate::generate::CancelToken::never(), }; let turn = session diff --git a/crates/aprender-serve/src/session.rs b/crates/aprender-serve/src/session.rs index 9f6892ddc3..570f26fd9c 100644 --- a/crates/aprender-serve/src/session.rs +++ b/crates/aprender-serve/src/session.rs @@ -38,7 +38,9 @@ use std::collections::VecDeque; use std::sync::Mutex; use crate::error::{RealizarError, Result}; -use crate::gguf::{OwnedQuantizedModel, QuantizedGenerateConfig}; +use crate::gguf::{ + logsumexp_full, top_k_logprobs, OwnedQuantizedModel, QuantizedGenerateConfig, StepLogprobs, +}; /// One architecture's forward on one backend: the only per-arch code a verb /// reaches, and only through a [`Session`]. @@ -102,7 +104,7 @@ pub trait ArchForward { } /// What one [`Session::generate`] call did. -#[derive(Debug, Clone, PartialEq, Eq)] +#[derive(Debug, Clone, PartialEq)] pub struct Turn { /// The prompt followed by the generated tokens. pub tokens: Vec, @@ -115,6 +117,24 @@ pub struct Turn { /// `max_tokens` and before a stop token. It is the one reason a reply is /// shorter than asked for that the caller did not choose. pub context_capped: bool, + /// One record per generated token when `config.logprobs_top_k` > 0 + /// (#4026), else empty. + pub steps: Vec, +} + +// #4026: the steps of the last turn a session generated on this thread, for +// `apr run`, whose dispatch drops the `Turn` inside per-backend helpers. Same +// idiom as `infer`'s generation-start mark: cleared before a dispatch, taken +// after it. `None` after a dispatch means the engine did not serve it. +std::thread_local! { + static LAST_TURN_STEPS: std::cell::RefCell>> = + const { std::cell::RefCell::new(None) }; +} + +/// Take the steps [`Session::generate`] last recorded on this thread (#4026). +/// `None` when no turn with `logprobs_top_k` > 0 ran since the last take. +pub fn take_last_turn_steps() -> Option> { + LAST_TURN_STEPS.with(std::cell::RefCell::take) } /// A loaded model plus its decode state: the one engine (#4263). @@ -229,13 +249,21 @@ impl Session { /// return it and how many leading tokens were already held. A greedy /// choice with no repetition penalty goes through /// [`ArchForward::forward_greedy`] when the backend has it. + /// + /// With `config.logprobs_top_k` > 0 the choice always comes from host + /// logits (the device argmax never shows them), and one [`StepLogprobs`] + /// is pushed to `steps`, ranked on the logits the FORWARD produced — + /// before the repetition penalty — so it records what the model said, + /// not what the sampler was handed (#4026). fn advance_and_choose( &mut self, tokens: &[u32], config: &QuantizedGenerateConfig, rng: &mut rand::rngs::StdRng, + steps: &mut Vec, ) -> Result<(u32, usize)> { - if is_greedy(config) && !penalty_active(config) { + let recording = config.logprobs_top_k > 0; + if !recording && is_greedy(config) && !penalty_active(config) { let start = if self.extends(tokens) { self.processed.len() } else { @@ -255,13 +283,27 @@ impl Session { } } let (mut logits, reused) = self.advance_to(tokens)?; + // The forward's logits, before the penalty edits them in place: the + // chosen token's logprob is read from these, not the penalised row. + let raw = recording.then(|| logits.clone()); OwnedQuantizedModel::apply_repeat_penalty( &mut logits, tokens, config.repeat_penalty, config.repeat_last_n, ); - Ok((choose_token(&logits, config, rng), reused)) + let chosen = choose_token(&logits, config, rng); + if let Some(raw) = raw { + let lse = logsumexp_full(&raw); + steps.push(StepLogprobs { + step: steps.len(), + chosen, + top: top_k_logprobs(&raw, config.logprobs_top_k), + chosen_logprob: raw.get(chosen as usize).map_or(f32::NAN, |&x| x - lse), + logsumexp_full: lse, + }); + } + Ok((chosen, reused)) } fn reserve(&mut self, positions: usize) -> Result<()> { @@ -322,7 +364,8 @@ impl Session { self.reserve(prompt.len() + budget)?; let mut rng = rand::rngs::StdRng::seed_from_u64(config.seed); - let (mut next, reused) = self.advance_and_choose(prompt, config, &mut rng)?; + let mut steps = Vec::new(); + let (mut next, reused) = self.advance_and_choose(prompt, config, &mut rng, &mut steps)?; let mut tokens = prompt.to_vec(); let mut context_capped = false; for generated in 1..=budget { @@ -338,13 +381,22 @@ impl Session { context_capped = context_limited; break; } - next = self.advance_and_choose(&tokens, config, &mut rng)?.0; + next = self + .advance_and_choose(&tokens, config, &mut rng, &mut steps)? + .0; + } + // A step past the last kept token (a cancel before it was pushed) is + // not part of the turn. + steps.truncate(tokens.len() - prompt.len()); + if config.logprobs_top_k > 0 { + LAST_TURN_STEPS.with(|c| c.replace(Some(steps.clone()))); } Ok(Turn { tokens, reused, used_gpu: self.on_gpu(), context_capped, + steps, }) } diff --git a/crates/aprender-serve/src/session_tests.rs b/crates/aprender-serve/src/session_tests.rs index 59b84cb463..858e6d044f 100644 --- a/crates/aprender-serve/src/session_tests.rs +++ b/crates/aprender-serve/src/session_tests.rs @@ -234,3 +234,116 @@ fn plain_greedy_takes_the_device_argmax_and_a_penalty_or_sampling_does_not() { "a repeat penalty needs the logits" ); } + +/// #4026: `logprobs_top_k` on a greedy config. +fn greedy_logprobs(max_tokens: usize, k: usize) -> QuantizedGenerateConfig { + QuantizedGenerateConfig { + logprobs_top_k: k, + ..greedy(max_tokens) + } +} + +#[test] +fn logprobs_record_one_step_per_generated_token_whose_top1_is_the_choice() { + let _ = take_last_turn_steps(); + let mut s = Session::new(Scripted::new(3, 100)); + let turn = s + .generate(&[7801, 7802], &greedy_logprobs(3, 2), &mut |_| true) + .expect("turn"); + assert_eq!(turn.tokens, vec![7801, 7802, 3, 3, 3]); + assert_eq!(turn.steps.len(), 3, "one record per generated token"); + for (i, step) in turn.steps.iter().enumerate() { + assert_eq!(step.step, i); + assert_eq!(step.chosen, turn.tokens[2 + i]); + assert_eq!(step.top.len(), 2); + assert_eq!( + step.top[0].token_id, step.chosen, + "greedy: top-1 is the choice" + ); + assert_eq!(step.top[0].logit, 1.0); + assert!(step.top[0].logprob > step.top[1].logprob); + } + assert_eq!(take_last_turn_steps(), Some(turn.steps.clone())); + assert_eq!(take_last_turn_steps(), None, "a take clears it"); +} + +#[test] +fn logprobs_zero_records_nothing_and_publishes_nothing() { + let _ = take_last_turn_steps(); + let mut s = Session::new(Scripted::new(3, 100)); + let turn = s + .generate(&[7811], &greedy(2), &mut |_| true) + .expect("turn"); + assert!(turn.steps.is_empty()); + assert_eq!(take_last_turn_steps(), None); +} + +#[test] +fn logprobs_bypass_the_device_argmax_because_it_never_shows_the_logits() { + let mut s = Session::new(DeviceGreedy { + inner: Scripted::new(3, 100), + greedy_answer: 5, + greedy_calls: 0, + }); + let turn = s + .generate(&[7821], &greedy_logprobs(2, 1), &mut |_| true) + .expect("turn"); + assert_eq!(s.engine().greedy_calls, 0); + assert_eq!(turn.tokens, vec![7821, 3, 3], "chosen from the host logits"); + assert_eq!(turn.steps.len(), 2); +} + +#[test] +fn logprobs_rank_what_the_model_said_not_what_the_penalty_left() { + // The prompt holds 3. A positive penalty only shrinks 3's logit toward + // the others' 0.0, never below it, so a negative one is used to flip its + // sign: 1.0 / -1.0 = -1.0, and the argmax falls to token 0. The record + // must still rank 3 first — it is taken before the penalty. + let mut cfg = greedy_logprobs(1, 2); + cfg.repeat_penalty = -1.0; + let mut s = Session::new(Scripted::new(3, 100)); + let turn = s.generate(&[3], &cfg, &mut |_| true).expect("turn"); + assert_eq!(turn.steps[0].chosen, 0, "the penalty moved the choice"); + assert_eq!(turn.steps[0].top[0].token_id, 3, "the record did not move"); + // PRM C11: the chosen token's logprob and the log-sum-exp come from the + // same unpenalised logits as `top` (3 → 1.0, the other 7 of the 8 → 0.0). + let lse = (1.0_f64.exp() + 7.0).ln(); + let s0 = &turn.steps[0]; + assert!( + (f64::from(s0.logsumexp_full) - lse).abs() < 1e-5, + "{}", + s0.logsumexp_full + ); + assert!( + (f64::from(s0.chosen_logprob) + lse).abs() < 1e-5, + "{}", + s0.chosen_logprob + ); + assert!((f64::from(s0.top[0].logprob) - (1.0 - lse)).abs() < 1e-5); +} + +#[test] +fn logprobs_follow_the_logits_a_planted_change_moves_the_record() { + // Negative control: the same prompt against a forward whose argmax was + // moved must record a different top-1, or the record is not reading them. + let run = |next| { + let mut s = Session::new(Scripted::new(next, 100)); + s.generate(&[7831], &greedy_logprobs(1, 1), &mut |_| true) + .expect("turn") + .steps[0] + .top[0] + .token_id + }; + assert_eq!(run(3), 3); + assert_eq!(run(6), 6); +} + +#[test] +fn logprobs_stop_on_a_stop_token_with_its_own_record() { + let mut cfg = greedy_logprobs(4, 1); + cfg.stop_tokens = vec![3]; + let mut s = Session::new(Scripted::new(3, 100)); + let turn = s.generate(&[7841], &cfg, &mut |_| true).expect("turn"); + assert_eq!(turn.tokens, vec![7841, 3]); + assert_eq!(turn.steps.len(), 1); +} diff --git a/docs/audits/prm-001/analysis-plan.md b/docs/audits/prm-001/analysis-plan.md new file mode 100644 index 0000000000..c4bd0e97e7 --- /dev/null +++ b/docs/audits/prm-001/analysis-plan.md @@ -0,0 +1,57 @@ +# PRM-001 analysis plan v2 (pre-registered for spec PRM-001 v3, 2026-09-25) + +Frozen together with spec `docs/specifications/PRM-001-prometheus.md` §2–§5, +`crates/aprender-review-experiment/src/stats.rs` and prompt v1 by `docs/audits/prm-001/prereg.lock` +(`rex-prereg-v2`). Any byte change to one of those files is a new spec version (R-1); data +gathered under the old version is `exploratory`. Plan v1 (`docs/audits/rex-001/analysis-plan.md`) +and its lock are superseded, frozen and never edited. + +## Units + +- **Item**: one corpus diff (class P, R or G; stratum S/M/L; split dev/test). +- **Arm**: (lane, model) — `apr-4b` (primary), `apr-9b` (control), `haiku`, `agy`. +- **Receipt**: one `review-experiment-receipt-v1` row per (item, cell, arm). +- **Correct**: a parsed `FAIL` on a P/R item, a parsed `PASS` on a G item. + `Unparsed`, `NotRun{…}`, `Refused` and `ContextOverflow` are never correct and never a + FAIL (R-6); they are counted and reported per state. + +## Tests (code: `stats.rs`) + +| H | Statistic | Function | Rejection rule | +|---|---|---|---| +| H1 invariance | test items whose VERDICT differs from reference cell C2 (v2; byte divergence is descriptive) | `verdict_divergence`, `count_rule_p` | count > 0 | +| H2 determinism | divergent items on the 10 % same-cell re-run | `count_rule_p` | count > 0 | +| H3 size | McNemar exact, one-sided, `b` = 9B right & 4B wrong | `discordant`, `mcnemar_exact_one_sided` | Holm-adjusted p ≤ 0.05 | +| H4 usefulness | precision(4B) − min(precision(haiku), precision(agy)), min computed INSIDE each resample; paired bootstrap, 10 000 resamples, seed 4354 | `vs_voters_parsed`, `bootstrap_vs_min_voter(Rate::Precision)` | holds iff the Holm-adjusted one-sided p (share of resamples < 0) ≤ 0.05; the 95 % CI is report-only | +| H5 recall | recall(4B) − recall(Claude lane) on class-R items only, paired bootstrap as H4 | `paired_parsed_class(.., Class::R)`, `paired_bootstrap_diff(Rate::Recall)` | as H4. Class-P recall and precision are descriptive | +| H6 interference | mean(co-located) − mean(alone), 5 + 5 runs | `interference` | slowdown > 2 × sd(alone) | +| H7 independence (a gate, outside the family) | κ_err(apr, each voter) vs max κ_err over voter–voter pairs, same items | `error_pairs`, `error_kappa`, `h7_holds` | holds iff every apr–voter κ ≤ the max voter–voter κ; undefined κ is undecided, never holds | + +Family H1–H6, Holm step-down (`holm`), α = 0.05. H1, H2 and H6 are deterministic rules and +enter the family as p = 0 (rejected) or p = 1 (not rejected). + +For H4 (v2) there is no fixed comparison lane: each resample subtracts whichever voter has the +lower precision ON THAT RESAMPLE. Items where apr or either voter is not a parsed verdict are +dropped and counted. For H5 the comparison is the Claude lane (Haiku 4.5), pairwise-dropped. + +## Descriptive + +Per cell: p50/p95 review wall-clock (nearest rank, `percentile`) with a bootstrap CI of the +p95 (same seed), TTFT, prefill and decode tok/s, peak anon memory, parse rate, and the ratio +to llama.cpp `d1d3c3396` on the same cell. Recall/precision/false-refute/localization with +Wilson 95 % intervals (`wilson`). + + +Sensitivity: every per-cell number is reported twice — all runs (primary) and runs with +loadavg1 < 0.5 × physical cores (secondary). Decisions read the primary column only. + +## Sample-size rule (§2.2) + +After PRM-05 (was REX-05), project the test-split Wilson half-width on recall at the pilot's point estimate +(`wilson_half_width`). If > 0.12, the corpus grows; hypotheses do not change. + +## Seeds + +- Run order per cell: SplitMix64 seed 4354 (the epic number), Fisher–Yates. +- Determinism re-run subset: the first 10 % of the seed-4354 order of test items. +- Bootstrap: seed 4354. diff --git a/docs/audits/prm-001/c11-4026-live/RECEIPT.md b/docs/audits/prm-001/c11-4026-live/RECEIPT.md new file mode 100644 index 0000000000..5b768d6b27 --- /dev/null +++ b/docs/audits/prm-001/c11-4026-live/RECEIPT.md @@ -0,0 +1,64 @@ +# #4026 live control: `apr serve` chat `top_logprobs` compared with a llama.cpp oracle + +**Verdict: RED on the pre-registered magnitude bar.** The feature reports apr's distribution, but apr's Q4_K CPU logits differ from llama.cpp's by more than 0.25 nats on 25 of 29 steps. + +- The report is faithful. Rank and alignment agree with the oracle, the refusals work, and the planted cases fail. +- The magnitude gap belongs to forward parity (apr vs llama.cpp on the same GGUF), not to the logprobs report. That attribution is an inference and is untested here; see *What this does not prove*. + +## Setup (mechanism proven, not assumed) + +| | | +|---|---| +| apr | `apr 0.69.3 (6f909ca30)`, a release build of rex/001 HEAD. The `apr serve` log says `gpu-layers: requested=none resolved=0 total=24 (backend=cpu)` | +| oracle | llama.cpp `llama-server` build 11050 (commit 60b06ab9a), `-ngl 0`, `CUDA_VISIBLE_DEVICES=""` | +| model | `Qwen3.5-0.8B-Q4_K_M.gguf`, sha256 `bd258782e35f7f458f8aced1adc053e6e92e89bc735ba3be89d38a06121dc517` | +| oracle prompt | The GGUF's own jinja template through `/apply-template` with `enable_thinking:false`, which is the mode apr served. It is tokenized by `/tokenize` with `parse_special:true` and sent to `/completion` as **ids** | +| teacher forcing | For step *i*, the oracle gets the prompt ids plus apr's first *i* generated ids, with `n_predict:1`, `n_probs:20`, `post_sampling_probs:false` (raw full-vocab softmax) and `cache_prompt:false` | +| request | `temperature:0`, `max_tokens:8`, `logprobs:true`, `top_logprobs:20` | + +Prompt identity: apr's `usage.prompt_tokens` equals the oracle's token count on all 6 prompts (24/23/17/17/22/17). + +- **This is a count, not an id diff.** apr exposes no chat-path prompt ids, so an id-level comparison was not possible. +- The first run was also a count mismatch (apr 24, oracle 22), caused by the oracle template's thinking-on default. That is how the thinking mode was pinned. + +## Result: 6 prompts, 29 teacher-forced steps + +| prompt | steps | top-1 agrees | top-20 overlap | max \|Δlogprob\| over shared ids | +|---|---|---|---|---| +| p1 capital of France | 1 | **0/1** | 14 | **1.65** | +| p2 three primes | 7 | 7/7 | 19–20 | 0.37–0.81 | +| p3 primary color | 8 | 8/8 | 17–20 | 0.19–0.33 | +| p4 hello → Spanish | 2 | 2/2 | 19–20 | 0.38–0.56 | +| p5 12 × 12 | 8 | 8/8 | 18–20 | 0.37–1.13 | +| p6 say blue | 3 | 3/3 | 19–20 | 0.25–0.40 | + +Pre-registered bar (set in `c4026_live.sh` before any data): top-1 equal, overlap ≥ 16, max |Δ| ≤ 0.25, chosen |Δ| ≤ 0.25, and Σexp(top-20) ≤ 1. +- Overlap ≥ 16 and top-1 each pass on 28 of 29. Both misses are p1 step 0 (overlap 14). +- **|Δ| ≤ 0.25 fails on 25 of 29 steps.** The four within it are all p3 steps (0.19–0.24). + +p1 step 0 is the top-1 flip: +- apr: `" Paris"` at −0.506, then `"Paris"` at −1.183. +- oracle: `"Paris"` at −0.390, then `" Paris"` at −1.651. + +The step-0 prefill logits are not the culprit, because the other five step-0s look like decode steps (0.31–0.40). One flip is an anecdote, not a named cause. + +## Controls + +| control | expected | observed | +|---|---|---| +| positive: the comparator on the oracle against itself (p2, p3, p5), `bash positive_control.sh` | pass, Δ = 0 | pass, max Δ 0 on 7/8/8 steps (`p{2,3,5}.posctl.json`). The same comparator on a 0.3-nat plant in one alternative fails (Δ 0.30) | +| planted: the oracle shifted one step late (p1) | fail | fail | +| planted: p1's record against p2's oracle | fail | fail | +| `top_logprobs:21` / `top_logprobs` without `logprobs` / `stream` with logprobs | 400, named | 400 / 400 / 400 | + +## Findings (for the cop; only the cop files issues) + +1. **Forward parity, apr vs llama.cpp, Qwen3.5-0.8B Q4_K_M on CPU.** Top-20 logprobs differ by 0.19–1.65 nats, and one top-1 flips (p1). Attribution is open: apr's Q4_K/Q8_K dot products, accumulation order, or the SSM/linear-attention kernels. It belongs to the parity contract, not #4026. +2. **`POST /tokenize` on apr serve splits `<|im_end|>`** while it parses `<|im_start|>` (248045) as one special. Given this prompt, it splits `<|im_end|>` into 6 literal pieces (28 ids, where the chat path and the oracle have 24). Evidence: `p1.apr_tok.json`. + +## What this does not prove + +- That the 0.25 bar is the right bar for #4026. It tests forward parity and the logprobs report together. A report-only oracle, meaning the same session's full logits dumped independently, is not in this receipt. +- GPU (the lambda 4090 is excluded; gx10 was not run), other quants, streaming, and `/v1/completions`, which are not implemented yet. + +Reproduce: `bash c4026_live.sh `, then `bash positive_control.sh ` (offline; it extracts the comparator from `c4026_live.sh`, so it tests the exact pre-registered bar). Both use bash, curl and jq only. diff --git a/docs/audits/prm-001/c11-4026-live/c4026_live.sh b/docs/audits/prm-001/c11-4026-live/c4026_live.sh new file mode 100644 index 0000000000..84b6c5c901 --- /dev/null +++ b/docs/audits/prm-001/c11-4026-live/c4026_live.sh @@ -0,0 +1,94 @@ +#!/usr/bin/env bash +# #4026 live control: apr serve chat top_logprobs vs a llama.cpp oracle on the +# official template, teacher-forced per step on the ids apr generated. CPU only. +# Usage: c4026_live.sh APR_BIN MODEL OUT_DIR +set -euo pipefail +APR="$1"; MODEL="$2"; OUT="$3" +LLAMA="$HOME/src/llama.cpp/build/bin/llama-server" +PA=18432; PL=18431; K=20; N=8 +export CUDA_VISIBLE_DEVICES="" +mkdir -p "$OUT" +pids=() +cleanup() { for p in "${pids[@]}"; do kill "$p" 2>/dev/null || true; done; } +trap cleanup EXIT + +"$LLAMA" -m "$MODEL" --port "$PL" -ngl 0 --jinja -t 8 -c 4096 >"$OUT/llama.log" 2>&1 & +pids+=("$!") +"$APR" serve run "$MODEL" --port "$PA" --no-gpu >"$OUT/apr.log" 2>&1 & +pids+=("$!") +echo "pids ${pids[*]}" >"$OUT/pids" +for u in "http://127.0.0.1:$PL/health" "http://127.0.0.1:$PA/health"; do + for _ in $(seq 1 180); do curl -sf "$u" >/dev/null && break; sleep 1; done + curl -sf "$u" >/dev/null || { echo "RED: $u never healthy"; exit 2; } +done + +run_prompt() { # $1 tag, $2 user text + local tag="$1" text="$2" + local msgs; msgs=$(jq -nc --arg t "$text" '[{role:"user",content:$t}]') + curl -sf "http://127.0.0.1:$PA/v1/chat/completions" -H 'content-type: application/json' \ + -d "$(jq -nc --argjson m "$msgs" --argjson k "$K" --argjson n "$N" \ + '{model:"q",messages:$m,max_tokens:$n,temperature:0,logprobs:true,top_logprobs:$k}')" \ + >"$OUT/$tag.apr.json" + # The oracle's prompt: the GGUF's own jinja template (thinking as apr's default). + local think; think=$(jq -r '.choices[0].message.content' "$OUT/$tag.apr.json" | grep -c '' || true) + curl -sf "http://127.0.0.1:$PL/apply-template" -H 'content-type: application/json' \ + -d "$(jq -nc --argjson m "$msgs" '{messages:$m, chat_template_kwargs:{enable_thinking:false}}')" >"$OUT/$tag.tmpl.json" + curl -sf "http://127.0.0.1:$PL/tokenize" -H 'content-type: application/json' \ + -d "$(jq -c '{content:.prompt, add_special:false, parse_special:true}' "$OUT/$tag.tmpl.json")" \ + >"$OUT/$tag.ids.json" + local n_prompt_llama n_prompt_apr + n_prompt_llama=$(jq '.tokens|length' "$OUT/$tag.ids.json") + n_prompt_apr=$(jq '.usage.prompt_tokens' "$OUT/$tag.apr.json") + echo "$tag prompt_tokens apr=$n_prompt_apr llama=$n_prompt_llama think_tag=$think" + local gen; gen=$(jq -c '[.choices[0].logprobs.content[].token_id]' "$OUT/$tag.apr.json") + local steps; steps=$(jq 'length' <<<"$gen") + : >"$OUT/$tag.oracle.jsonl" + for i in $(seq 0 $((steps - 1))); do + local prompt; prompt=$(jq -c --argjson g "$gen" --argjson i "$i" '.tokens + $g[:$i]' "$OUT/$tag.ids.json") + curl -sf "http://127.0.0.1:$PL/completion" -H 'content-type: application/json' \ + -d "$(jq -nc --argjson p "$prompt" --argjson k "$K" \ + '{prompt:$p,n_predict:1,n_probs:$k,temperature:0,post_sampling_probs:false,cache_prompt:false}')" \ + | jq -c '.completion_probabilities[0]' >>"$OUT/$tag.oracle.jsonl" + done +} + +# compare APR_JSON ORACLE_JSONL SHIFT -> one line per step + verdict line +compare() { + jq -n --slurpfile a "$1" --slurpfile o "$2" --argjson s "$3" ' + [$a[0].choices[0].logprobs.content, $o] as [$ac, $oc] + | [range(0; ($ac|length) - $s) as $i + | ($ac[$i]) as $x | ($oc[$i + $s]) as $y + | ($y.top_logprobs | map({key: (.id|tostring), value: .logprob}) | from_entries) as $om + | ($x.top_logprobs | map(select($om[(.token_id|tostring)] != null) + | (.logprob - $om[(.token_id|tostring)]) | fabs)) as $d + | {step: $i, + top1_apr: $x.top_logprobs[0].token_id, top1_oracle: $y.top_logprobs[0].id, + overlap: ($d|length), + max_abs_dlogprob: ($d|max // null), + chosen_dlogprob: (($x.logprob - ($om[($x.token_id|tostring)] // -1e9))|fabs), + apr_mass: ([$x.top_logprobs[].logprob | exp] | add)}] + | {steps: ., pass: (length > 0 and all(.[]; + .top1_apr == .top1_oracle and .overlap >= 16 + and .max_abs_dlogprob <= 0.25 and .chosen_dlogprob <= 0.25 and .apr_mass <= 1.0001))}' +} + +run_prompt p1 "What is the capital of France? Answer in one word." +run_prompt p2 "Write the first three prime numbers, separated by commas." +run_prompt p3 "Name a primary color." +run_prompt p4 "Translate hello into Spanish." +run_prompt p5 "What is 12 times 12?" +run_prompt p6 "Say the word blue." +for t in p3 p4 p5 p6; do compare "$OUT/$t.apr.json" "$OUT/$t.oracle.jsonl" 0 >"$OUT/$t.cmp.json"; done +for t in p1 p2; do compare "$OUT/$t.apr.json" "$OUT/$t.oracle.jsonl" 0 >"$OUT/$t.cmp.json"; done +# Planted: the oracle one step late. It must fail, or the comparator cannot. +compare "$OUT/p1.apr.json" "$OUT/p1.oracle.jsonl" 1 >"$OUT/p1.planted_shift.json" +# Planted: p1's record against p2's oracle. +compare "$OUT/p1.apr.json" "$OUT/p2.oracle.jsonl" 0 >"$OUT/p1_vs_p2.planted.json" +# Refusal controls: must be 400 and named. +code() { curl -s -o "$OUT/$1.body" -w '%{http_code}' "http://127.0.0.1:$PA/v1/chat/completions" \ + -H 'content-type: application/json' -d "$2"; } +r1=$(code r_k21 '{"model":"q","messages":[{"role":"user","content":"hi"}],"logprobs":true,"top_logprobs":21}') +r2=$(code r_nolp '{"model":"q","messages":[{"role":"user","content":"hi"}],"top_logprobs":5}') +r3=$(code r_stream '{"model":"q","messages":[{"role":"user","content":"hi"}],"logprobs":true,"top_logprobs":5,"stream":true}') +echo "p1 pass=$(jq .pass "$OUT/p1.cmp.json") p2 pass=$(jq .pass "$OUT/p2.cmp.json") planted_shift pass=$(jq .pass "$OUT/p1.planted_shift.json") planted_cross pass=$(jq .pass "$OUT/p1_vs_p2.planted.json") refusals=$r1/$r2/$r3" +grep -iE 'gpu|cuda|cpu' "$OUT/apr.log" | head -5 || true diff --git a/docs/audits/prm-001/c11-4026-live/p1.apr.json b/docs/audits/prm-001/c11-4026-live/p1.apr.json new file mode 100644 index 0000000000..8f272789be --- /dev/null +++ b/docs/audits/prm-001/c11-4026-live/p1.apr.json @@ -0,0 +1 @@ +{"id":"chatcmpl-q4k-1790386249053","object":"chat.completion","created":1790386250,"model":"q","choices":[{"index":0,"message":{"role":"assistant","content":"Paris"},"finish_reason":"stop","logprobs":{"content":[{"token":" Paris","token_id":11751,"logprob":-0.5058727264404297,"bytes":null,"top_logprobs":[{"token":" Paris","token_id":11751,"logprob":-0.5058727264404297,"bytes":null},{"token":"Paris","token_id":57590,"logprob":-1.1826248168945312,"bytes":null},{"token":" PARIS","token_id":217707,"logprob":-3.248992919921875,"bytes":null},{"token":"巴黎","token_id":109705,"logprob":-5.060816764831543,"bytes":null},{"token":" Versailles","token_id":227953,"logprob":-5.29096794128418,"bytes":null},{"token":" paris","token_id":39509,"logprob":-5.338647842407227,"bytes":null},{"token":"<|im_end|>","token_id":248046,"logprob":-5.550654411315918,"bytes":null},{"token":"L","token_id":43,"logprob":-5.580808639526367,"bytes":null},{"token":" La","token_id":4770,"logprob":-5.87992000579834,"bytes":null},{"token":"PAR","token_id":16079,"logprob":-5.95156192779541,"bytes":null},{"token":"Par","token_id":4132,"logprob":-6.211648941040039,"bytes":null},{"token":" L","token_id":436,"logprob":-6.274188995361328,"bytes":null},{"token":"La","token_id":8482,"logprob":-6.596256256103516,"bytes":null},{"token":" London","token_id":6924,"logprob":-6.829401016235352,"bytes":null},{"token":" **","token_id":2972,"logprob":-7.020146369934082,"bytes":null},{"token":"The","token_id":760,"logprob":-7.260797500610352,"bytes":null},{"token":" París","token_id":206626,"logprob":-7.267195701599121,"bytes":null},{"token":" Lyon","token_id":53347,"logprob":-7.4127044677734375,"bytes":null},{"token":" Pari","token_id":163512,"logprob":-7.414087295532227,"bytes":null},{"token":" باريس","token_id":200360,"logprob":-7.585988998413086,"bytes":null}],"logsumexp_full":19.630521774291992}]}}],"usage":{"prompt_tokens":24,"completion_tokens":1,"total_tokens":25}} \ No newline at end of file diff --git a/docs/audits/prm-001/c11-4026-live/p1.apr_tok.json b/docs/audits/prm-001/c11-4026-live/p1.apr_tok.json new file mode 100644 index 0000000000..a4e1471af6 --- /dev/null +++ b/docs/audits/prm-001/c11-4026-live/p1.apr_tok.json @@ -0,0 +1 @@ +{"token_ids":[248045,846,198,3710,369,279,6511,314,9338,30,21134,303,799,3299,15294,92811,76,6018,91,29,198,248045,74455,198,248068,271,248069,271],"num_tokens":28} \ No newline at end of file diff --git a/docs/audits/prm-001/c11-4026-live/p1.cmp.json b/docs/audits/prm-001/c11-4026-live/p1.cmp.json new file mode 100644 index 0000000000..b1a0f0d81a --- /dev/null +++ b/docs/audits/prm-001/c11-4026-live/p1.cmp.json @@ -0,0 +1,14 @@ +{ + "steps": [ + { + "step": 0, + "top1_apr": 11751, + "top1_oracle": 57590, + "overlap": 14, + "max_abs_dlogprob": 1.647125244140625, + "chosen_dlogprob": 1.1446846723556519, + "apr_mass": 0.9878417476958157 + } + ], + "pass": false +} diff --git a/docs/audits/prm-001/c11-4026-live/p1.ids.json b/docs/audits/prm-001/c11-4026-live/p1.ids.json new file mode 100644 index 0000000000..c3bf4533fb --- /dev/null +++ b/docs/audits/prm-001/c11-4026-live/p1.ids.json @@ -0,0 +1 @@ +{"tokens":[248045,846,198,3710,369,279,6511,314,9338,30,21134,303,799,3299,13,248046,198,248045,74455,198,248068,271,248069,271]} \ No newline at end of file diff --git a/docs/audits/prm-001/c11-4026-live/p1.oracle.jsonl b/docs/audits/prm-001/c11-4026-live/p1.oracle.jsonl new file mode 100644 index 0000000000..3cfc81a30c --- /dev/null +++ b/docs/audits/prm-001/c11-4026-live/p1.oracle.jsonl @@ -0,0 +1 @@ +{"id":57590,"token":"Paris","bytes":[80,97,114,105,115],"logprob":-0.39019858837127686,"top_logprobs":[{"id":57590,"token":"Paris","bytes":[80,97,114,105,115],"logprob":-0.39019858837127686},{"id":11751,"token":" Paris","bytes":[32,80,97,114,105,115],"logprob":-1.6505573987960815},{"id":217707,"token":" PARIS","bytes":[32,80,65,82,73,83],"logprob":-3.24702525138855},{"id":43,"token":"L","bytes":[76],"logprob":-3.9858767986297607},{"id":16079,"token":"PAR","bytes":[80,65,82],"logprob":-4.304436683654785},{"id":4132,"token":"Par","bytes":[80,97,114],"logprob":-4.947832107543945},{"id":109705,"token":"巴黎","bytes":[229,183,180,233,187,142],"logprob":-4.952825546264648},{"id":39509,"token":" paris","bytes":[32,112,97,114,105,115],"logprob":-5.240074157714844},{"id":248046,"token":"","bytes":[],"logprob":-5.292830467224121},{"id":227953,"token":" Versailles","bytes":[32,86,101,114,115,97,105,108,108,101,115],"logprob":-5.32531213760376},{"id":8482,"token":"La","bytes":[76,97],"logprob":-5.423247337341309},{"id":760,"token":"The","bytes":[84,104,101],"logprob":-6.230172157287598},{"id":91149,"token":"Berlin","bytes":[66,101,114,108,105,110],"logprob":-6.674399375915527},{"id":4770,"token":" La","bytes":[32,76,97],"logprob":-6.724493980407715},{"id":38261,"token":"London","bytes":[76,111,110,100,111,110],"logprob":-6.804826736450195},{"id":436,"token":" L","bytes":[32,76],"logprob":-7.236334800720215},{"id":81060,"token":"Vers","bytes":[86,101,114,115],"logprob":-7.272490501403809},{"id":47358,"token":"France","bytes":[70,114,97,110,99,101],"logprob":-7.356118202209473},{"id":1537,"token":"LO","bytes":[76,79],"logprob":-7.429507255554199},{"id":10427,"token":"Ang","bytes":[65,110,103],"logprob":-7.513894081115723}]} diff --git a/docs/audits/prm-001/c11-4026-live/p1.planted_shift.json b/docs/audits/prm-001/c11-4026-live/p1.planted_shift.json new file mode 100644 index 0000000000..50b049e5a7 --- /dev/null +++ b/docs/audits/prm-001/c11-4026-live/p1.planted_shift.json @@ -0,0 +1,4 @@ +{ + "steps": [], + "pass": false +} diff --git a/docs/audits/prm-001/c11-4026-live/p1_vs_p2.planted.json b/docs/audits/prm-001/c11-4026-live/p1_vs_p2.planted.json new file mode 100644 index 0000000000..8520cd47e0 --- /dev/null +++ b/docs/audits/prm-001/c11-4026-live/p1_vs_p2.planted.json @@ -0,0 +1,14 @@ +{ + "steps": [ + { + "step": 0, + "top1_apr": 11751, + "top1_oracle": 17, + "overlap": 1, + "max_abs_dlogprob": 5.084360599517822, + "chosen_dlogprob": 999999999.4941273, + "apr_mass": 0.9878417476958157 + } + ], + "pass": false +} diff --git a/docs/audits/prm-001/c11-4026-live/p2.apr.json b/docs/audits/prm-001/c11-4026-live/p2.apr.json new file mode 100644 index 0000000000..ce14b617f6 --- /dev/null +++ b/docs/audits/prm-001/c11-4026-live/p2.apr.json @@ -0,0 +1 @@ +{"id":"chatcmpl-q4k-1790386250690","object":"chat.completion","created":1790386252,"model":"q","choices":[{"index":0,"message":{"role":"assistant","content":"2, 3, 5"},"finish_reason":"stop","logprobs":{"content":[{"token":"2","token_id":17,"logprob":-1.0003395080566406,"bytes":null,"top_logprobs":[{"token":"2","token_id":17,"logprob":-1.0003395080566406,"bytes":null},{"token":"1","token_id":16,"logprob":-1.006631851196289,"bytes":null},{"token":"The","token_id":760,"logprob":-1.9230270385742188,"bytes":null},{"token":"3","token_id":18,"logprob":-2.9555835723876953,"bytes":null},{"token":"5","token_id":20,"logprob":-4.111040115356445,"bytes":null},{"token":"7","token_id":22,"logprob":-4.443422317504883,"bytes":null},{"token":"0","token_id":15,"logprob":-5.042646408081055,"bytes":null},{"token":"4","token_id":19,"logprob":-5.305843353271484,"bytes":null},{"token":"First","token_id":5170,"logprob":-5.539482116699219,"bytes":null},{"token":"","token_id":248068,"logprob":-6.017667770385742,"bytes":null},{"token":"Here","token_id":8160,"logprob":-6.145254135131836,"bytes":null},{"token":"To","token_id":1206,"logprob":-6.272462844848633,"bytes":null},{"token":"As","token_id":2053,"logprob":-6.274328231811523,"bytes":null},{"token":"Let","token_id":9764,"logprob":-6.368865966796875,"bytes":null},{"token":"6","token_id":21,"logprob":-6.80714225769043,"bytes":null},{"token":"9","token_id":24,"logprob":-7.019098281860352,"bytes":null},{"token":"A","token_id":32,"logprob":-7.135217666625977,"bytes":null},{"token":"There","token_id":3733,"logprob":-7.153751373291016,"bytes":null},{"token":"Based","token_id":27775,"logprob":-7.36669921875,"bytes":null},{"token":"Two","token_id":11280,"logprob":-7.561948776245117,"bytes":null}],"logsumexp_full":23.3278751373291},{"token":",","token_id":11,"logprob":-0.0069255828857421875,"bytes":null,"top_logprobs":[{"token":",","token_id":11,"logprob":-0.0069255828857421875,"bytes":null},{"token":"\n\n","token_id":271,"logprob":-6.036037445068359,"bytes":null},{"token":"\n","token_id":198,"logprob":-6.635034561157227,"bytes":null},{"token":"<|im_end|>","token_id":248046,"logprob":-6.782379150390625,"bytes":null},{"token":"3","token_id":18,"logprob":-7.909076690673828,"bytes":null},{"token":"1","token_id":16,"logprob":-8.372833251953125,"bytes":null},{"token":"7","token_id":22,"logprob":-8.502805709838867,"bytes":null},{"token":"9","token_id":24,"logprob":-8.621204376220703,"bytes":null},{"token":"5","token_id":20,"logprob":-8.624427795410156,"bytes":null},{"token":".","token_id":13,"logprob":-8.789379119873047,"bytes":null},{"token":" and","token_id":321,"logprob":-9.13653564453125,"bytes":null},{"token":"0","token_id":15,"logprob":-9.161332130432129,"bytes":null},{"token":" ","token_id":220,"logprob":-9.302308082580566,"bytes":null},{"token":"4","token_id":19,"logprob":-9.773656845092773,"bytes":null},{"token":";","token_id":26,"logprob":-9.862285614013672,"bytes":null},{"token":" \n","token_id":2228,"logprob":-10.131279945373535,"bytes":null},{"token":"6","token_id":21,"logprob":-10.263653755187988,"bytes":null},{"token":"2","token_id":17,"logprob":-10.381702423095703,"bytes":null},{"token":" ,","token_id":1116,"logprob":-10.468790054321289,"bytes":null},{"token":"<|endoftext|>","token_id":248044,"logprob":-10.809652328491211,"bytes":null}],"logsumexp_full":24.667327880859375},{"token":" ","token_id":220,"logprob":-0.0001926422119140625,"bytes":null,"top_logprobs":[{"token":" ","token_id":220,"logprob":-0.0001926422119140625,"bytes":null},{"token":" **","token_id":2972,"logprob":-9.420595169067383,"bytes":null},{"token":"3","token_id":18,"logprob":-9.615476608276367,"bytes":null},{"token":"�","token_id":373,"logprob":-10.430107116699219,"bytes":null},{"token":" *","token_id":348,"logprob":-13.782861709594727,"bytes":null},{"token":" and","token_id":321,"logprob":-14.073637008666992,"bytes":null},{"token":" ...","token_id":2423,"logprob":-14.13840389251709,"bytes":null},{"token":"\n","token_id":198,"logprob":-14.477970123291016,"bytes":null},{"token":" three","token_id":2250,"logprob":-14.70163345336914,"bytes":null},{"token":" up","token_id":685,"logprob":-14.959277153015137,"bytes":null},{"token":" ...,","token_id":58317,"logprob":-15.001193046569824,"bytes":null},{"token":" ","token_id":3966,"logprob":-15.039016723632812,"bytes":null},{"token":" not","token_id":524,"logprob":-15.04159164428711,"bytes":null},{"token":"�","token_id":15428,"logprob":-15.080004692077637,"bytes":null},{"token":"7","token_id":22,"logprob":-15.109621047973633,"bytes":null},{"token":" ?","token_id":907,"logprob":-15.16732406616211,"bytes":null},{"token":" (","token_id":318,"logprob":-15.233417510986328,"bytes":null},{"token":" -","token_id":471,"logprob":-15.260169982910156,"bytes":null},{"token":" Pr","token_id":2264,"logprob":-15.263830184936523,"bytes":null},{"token":" {","token_id":313,"logprob":-15.314935684204102,"bytes":null}],"logsumexp_full":26.513147354125977},{"token":"3","token_id":18,"logprob":-0.0020751953125,"bytes":null,"top_logprobs":[{"token":"3","token_id":18,"logprob":-0.0020751953125,"bytes":null},{"token":"7","token_id":22,"logprob":-6.530633926391602,"bytes":null},{"token":"5","token_id":20,"logprob":-8.276357650756836,"bytes":null},{"token":"1","token_id":16,"logprob":-8.760847091674805,"bytes":null},{"token":"4","token_id":19,"logprob":-9.263555526733398,"bytes":null},{"token":"2","token_id":17,"logprob":-9.374788284301758,"bytes":null},{"token":"6","token_id":21,"logprob":-11.49905776977539,"bytes":null},{"token":"9","token_id":24,"logprob":-11.832252502441406,"bytes":null},{"token":"8","token_id":23,"logprob":-12.334306716918945,"bytes":null},{"token":"0","token_id":15,"logprob":-14.488407135009766,"bytes":null},{"token":"�","token_id":15428,"logprob":-14.556663513183594,"bytes":null},{"token":" ","token_id":220,"logprob":-14.722291946411133,"bytes":null},{"token":"³","token_id":41776,"logprob":-16.42536735534668,"bytes":null},{"token":"�","token_id":12256,"logprob":-16.54682159423828,"bytes":null},{"token":"�","token_id":151,"logprob":-17.272876739501953,"bytes":null},{"token":"�","token_id":373,"logprob":-17.418846130371094,"bytes":null},{"token":"<|endoftext|>","token_id":248044,"logprob":-18.131458282470703,"bytes":null},{"token":"�","token_id":46701,"logprob":-18.363929748535156,"bytes":null},{"token":"�","token_id":26064,"logprob":-18.372337341308594,"bytes":null},{"token":"３","token_id":32405,"logprob":-18.37651824951172,"bytes":null}],"logsumexp_full":29.821035385131836},{"token":",","token_id":11,"logprob":-0.000064849853515625,"bytes":null,"top_logprobs":[{"token":",","token_id":11,"logprob":-0.000064849853515625,"bytes":null},{"token":"<|im_end|>","token_id":248046,"logprob":-11.033636093139648,"bytes":null},{"token":"5","token_id":20,"logprob":-11.451168060302734,"bytes":null},{"token":"\n\n","token_id":271,"logprob":-11.916438102722168,"bytes":null},{"token":" and","token_id":321,"logprob":-12.22714900970459,"bytes":null},{"token":".","token_id":13,"logprob":-12.556270599365234,"bytes":null},{"token":"4","token_id":19,"logprob":-12.562644958496094,"bytes":null},{"token":"7","token_id":22,"logprob":-12.621861457824707,"bytes":null},{"token":" ,","token_id":1116,"logprob":-12.687139511108398,"bytes":null},{"token":";","token_id":26,"logprob":-12.813016891479492,"bytes":null},{"token":" ","token_id":220,"logprob":-13.073434829711914,"bytes":null},{"token":"\n","token_id":198,"logprob":-13.810102462768555,"bytes":null},{"token":"<|endoftext|>","token_id":248044,"logprob":-13.939496040344238,"bytes":null},{"token":" &","token_id":594,"logprob":-14.170111656188965,"bytes":null},{"token":"1","token_id":16,"logprob":-14.419580459594727,"bytes":null},{"token":" (","token_id":318,"logprob":-14.625482559204102,"bytes":null},{"token":",**","token_id":67608,"logprob":-14.935676574707031,"bytes":null},{"token":"9","token_id":24,"logprob":-15.070652961730957,"bytes":null},{"token":"**","token_id":332,"logprob":-15.246892929077148,"bytes":null},{"token":"�","token_id":373,"logprob":-15.254152297973633,"bytes":null}],"logsumexp_full":27.681629180908203},{"token":" ","token_id":220,"logprob":-0.0063686370849609375,"bytes":null,"top_logprobs":[{"token":" ","token_id":220,"logprob":-0.0063686370849609375,"bytes":null},{"token":" and","token_id":321,"logprob":-5.137779235839844,"bytes":null},{"token":" **","token_id":2972,"logprob":-8.099506378173828,"bytes":null},{"token":" &","token_id":594,"logprob":-10.031352996826172,"bytes":null},{"token":" *","token_id":348,"logprob":-10.52109146118164,"bytes":null},{"token":"�","token_id":373,"logprob":-10.994016647338867,"bytes":null},{"token":" p","token_id":280,"logprob":-11.18138313293457,"bytes":null},{"token":" (","token_id":318,"logprob":-11.2079439163208,"bytes":null},{"token":" or","token_id":466,"logprob":-12.168962478637695,"bytes":null},{"token":" ...","token_id":2423,"logprob":-12.405776977539062,"bytes":null},{"token":" \\","token_id":1088,"logprob":-12.662872314453125,"bytes":null},{"token":" \"","token_id":328,"logprob":-12.769586563110352,"bytes":null},{"token":"<|im_end|>","token_id":248046,"logprob":-12.907818794250488,"bytes":null},{"token":" -","token_id":471,"logprob":-13.173730850219727,"bytes":null},{"token":" Pr","token_id":2264,"logprob":-13.50839614868164,"bytes":null},{"token":"5","token_id":20,"logprob":-13.535835266113281,"bytes":null},{"token":" primes","token_id":47770,"logprob":-13.560518264770508,"bytes":null},{"token":" ?","token_id":907,"logprob":-13.641548156738281,"bytes":null},{"token":" /","token_id":593,"logprob":-13.683661460876465,"bytes":null},{"token":"\n\n","token_id":271,"logprob":-13.720613479614258,"bytes":null}],"logsumexp_full":25.288715362548828},{"token":"5","token_id":20,"logprob":-0.0006732940673828125,"bytes":null,"top_logprobs":[{"token":"5","token_id":20,"logprob":-0.0006732940673828125,"bytes":null},{"token":"7","token_id":22,"logprob":-8.260547637939453,"bytes":null},{"token":"4","token_id":19,"logprob":-8.55605697631836,"bytes":null},{"token":"1","token_id":16,"logprob":-8.866174697875977,"bytes":null},{"token":"2","token_id":17,"logprob":-10.20539665222168,"bytes":null},{"token":"3","token_id":18,"logprob":-10.55875015258789,"bytes":null},{"token":"6","token_id":21,"logprob":-12.253608703613281,"bytes":null},{"token":"0","token_id":15,"logprob":-12.784646987915039,"bytes":null},{"token":"9","token_id":24,"logprob":-13.428323745727539,"bytes":null},{"token":" ","token_id":220,"logprob":-13.487794876098633,"bytes":null},{"token":"�","token_id":373,"logprob":-13.591716766357422,"bytes":null},{"token":"8","token_id":23,"logprob":-13.881246566772461,"bytes":null},{"token":"<|im_end|>","token_id":248046,"logprob":-14.284442901611328,"bytes":null},{"token":" and","token_id":321,"logprob":-14.525642395019531,"bytes":null},{"token":"½","token_id":25229,"logprob":-14.620704650878906,"bytes":null},{"token":"�","token_id":151,"logprob":-14.7728271484375,"bytes":null},{"token":"........................","token_id":54352,"logprob":-14.871286392211914,"bytes":null},{"token":"�","token_id":12256,"logprob":-14.889202117919922,"bytes":null},{"token":"�","token_id":50382,"logprob":-15.045711517333984,"bytes":null},{"token":"�","token_id":15428,"logprob":-15.326576232910156,"bytes":null}],"logsumexp_full":27.617919921875}]}}],"usage":{"prompt_tokens":23,"completion_tokens":7,"total_tokens":30}} \ No newline at end of file diff --git a/docs/audits/prm-001/c11-4026-live/p2.cmp.json b/docs/audits/prm-001/c11-4026-live/p2.cmp.json new file mode 100644 index 0000000000..8dfa6a4de9 --- /dev/null +++ b/docs/audits/prm-001/c11-4026-live/p2.cmp.json @@ -0,0 +1,68 @@ +{ + "steps": [ + { + "step": 0, + "top1_apr": 17, + "top1_oracle": 17, + "overlap": 20, + "max_abs_dlogprob": 0.3666815757751465, + "chosen_dlogprob": 0.1445298194885254, + "apr_mass": 0.9897040243500008 + }, + { + "step": 1, + "top1_apr": 11, + "top1_oracle": 11, + "overlap": 20, + "max_abs_dlogprob": 0.4305438995361328, + "chosen_dlogprob": 0.0013122064992785454, + "apr_mass": 0.9998172977692664 + }, + { + "step": 2, + "top1_apr": 220, + "top1_oracle": 220, + "overlap": 19, + "max_abs_dlogprob": 0.6120805740356445, + "chosen_dlogprob": 9.04603220988065e-05, + "apr_mass": 0.9999910535398526 + }, + { + "step": 3, + "top1_apr": 18, + "top1_oracle": 18, + "overlap": 19, + "max_abs_dlogprob": 0.3667106628417969, + "chosen_dlogprob": 6.8747904151678085e-06, + "apr_mass": 0.9999993427568648 + }, + { + "step": 4, + "top1_apr": 11, + "top1_oracle": 11, + "overlap": 19, + "max_abs_dlogprob": 0.808985710144043, + "chosen_dlogprob": 1.3707762263948098e-05, + "apr_mass": 0.9999964213607592 + }, + { + "step": 5, + "top1_apr": 220, + "top1_oracle": 220, + "overlap": 19, + "max_abs_dlogprob": 0.6074175834655762, + "chosen_dlogprob": 0.0026935243513435125, + "apr_mass": 0.9999683389041605 + }, + { + "step": 6, + "top1_apr": 20, + "top1_oracle": 20, + "overlap": 20, + "max_abs_dlogprob": 0.771885871887207, + "chosen_dlogprob": 0.000273803569143638, + "apr_mass": 0.9999976129809058 + } + ], + "pass": false +} diff --git a/docs/audits/prm-001/c11-4026-live/p2.ids.json b/docs/audits/prm-001/c11-4026-live/p2.ids.json new file mode 100644 index 0000000000..a111029924 --- /dev/null +++ b/docs/audits/prm-001/c11-4026-live/p2.ids.json @@ -0,0 +1 @@ +{"tokens":[248045,846,198,7734,279,1118,2250,9944,4947,11,18101,539,73982,13,248046,198,248045,74455,198,248068,271,248069,271]} \ No newline at end of file diff --git a/docs/audits/prm-001/c11-4026-live/p2.oracle.jsonl b/docs/audits/prm-001/c11-4026-live/p2.oracle.jsonl new file mode 100644 index 0000000000..898611255a --- /dev/null +++ b/docs/audits/prm-001/c11-4026-live/p2.oracle.jsonl @@ -0,0 +1,7 @@ +{"id":17,"token":"2","bytes":[50],"logprob":-0.8558096885681152,"top_logprobs":[{"id":17,"token":"2","bytes":[50],"logprob":-0.8558096885681152},{"id":16,"token":"1","bytes":[49],"logprob":-1.1232353448867798},{"id":760,"token":"The","bytes":[84,104,101],"logprob":-2.1764369010925293},{"id":18,"token":"3","bytes":[51],"logprob":-2.7710108757019043},{"id":20,"token":"5","bytes":[53],"logprob":-3.842771053314209},{"id":22,"token":"7","bytes":[55],"logprob":-4.1483073234558105},{"id":19,"token":"4","bytes":[52],"logprob":-5.159671306610107},{"id":15,"token":"0","bytes":[48],"logprob":-5.1988844871521},{"id":248068,"token":"","bytes":[60,116,104,105,110,107,62],"logprob":-5.721559047698975},{"id":5170,"token":"First","bytes":[70,105,114,115,116],"logprob":-5.852420330047607},{"id":2053,"token":"As","bytes":[65,115],"logprob":-6.441211223602295},{"id":8160,"token":"Here","bytes":[72,101,114,101],"logprob":-6.511935710906982},{"id":9764,"token":"Let","bytes":[76,101,116],"logprob":-6.551112651824951},{"id":1206,"token":"To","bytes":[84,111],"logprob":-6.5843634605407715},{"id":21,"token":"6","bytes":[54],"logprob":-6.708505153656006},{"id":24,"token":"9","bytes":[57],"logprob":-6.875026226043701},{"id":32,"token":"A","bytes":[65],"logprob":-7.424076557159424},{"id":3733,"token":"There","bytes":[84,104,101,114,101],"logprob":-7.494142055511475},{"id":27775,"token":"Based","bytes":[66,97,115,101,100],"logprob":-7.511061191558838},{"id":11280,"token":"Two","bytes":[84,119,111],"logprob":-7.72383451461792}]} +{"id":11,"token":",","bytes":[44],"logprob":-0.005613376386463642,"top_logprobs":[{"id":11,"token":",","bytes":[44],"logprob":-0.005613376386463642},{"id":271,"token":"\n\n","bytes":[10,10],"logprob":-6.123464584350586},{"id":198,"token":"\n","bytes":[10],"logprob":-7.001176834106445},{"id":248046,"token":"","bytes":[],"logprob":-7.212923049926758},{"id":18,"token":"3","bytes":[51],"logprob":-8.070806503295898},{"id":22,"token":"7","bytes":[55],"logprob":-8.370685577392578},{"id":16,"token":"1","bytes":[49],"logprob":-8.451520919799805},{"id":24,"token":"9","bytes":[57],"logprob":-8.786211013793945},{"id":13,"token":".","bytes":[46],"logprob":-8.92701530456543},{"id":20,"token":"5","bytes":[53],"logprob":-9.021446228027344},{"id":321,"token":" and","bytes":[32,97,110,100],"logprob":-9.17351245880127},{"id":15,"token":"0","bytes":[48],"logprob":-9.340082168579102},{"id":220,"token":" ","bytes":[32],"logprob":-9.380533218383789},{"id":26,"token":";","bytes":[59],"logprob":-10.0845947265625},{"id":19,"token":"4","bytes":[52],"logprob":-10.19825267791748},{"id":2228,"token":" \n","bytes":[32,32,10],"logprob":-10.341672897338867},{"id":1116,"token":" ,","bytes":[32,44],"logprob":-10.544290542602539},{"id":17,"token":"2","bytes":[50],"logprob":-10.626684188842773},{"id":21,"token":"6","bytes":[54],"logprob":-10.637521743774414},{"id":248044,"token":"","bytes":[],"logprob":-11.010668754577637}]} +{"id":220,"token":" ","bytes":[32],"logprob":-0.000283102534012869,"top_logprobs":[{"id":220,"token":" ","bytes":[32],"logprob":-0.000283102534012869},{"id":2972,"token":" **","bytes":[32,42,42],"logprob":-8.847826957702637},{"id":18,"token":"3","bytes":[51],"logprob":-9.443541526794434},{"id":373,"token":"","bytes":[226,128],"logprob":-9.978693962097168},{"id":348,"token":" *","bytes":[32,42],"logprob":-13.40364933013916},{"id":2423,"token":" ...","bytes":[32,46,46,46],"logprob":-13.779648780822754},{"id":198,"token":"\n","bytes":[10],"logprob":-13.994604110717773},{"id":321,"token":" and","bytes":[32,97,110,100],"logprob":-14.195822715759277},{"id":22,"token":"7","bytes":[55],"logprob":-14.497540473937988},{"id":2250,"token":" three","bytes":[32,116,104,114,101,101],"logprob":-14.633594512939453},{"id":524,"token":" not","bytes":[32,110,111,116],"logprob":-14.649407386779785},{"id":685,"token":" up","bytes":[32,117,112],"logprob":-14.685708999633789},{"id":58317,"token":" ...,","bytes":[32,46,46,46,44],"logprob":-14.81330394744873},{"id":3966,"token":" ","bytes":[194,160],"logprob":-14.854573249816895},{"id":15428,"token":"","bytes":[226,130],"logprob":-14.89685344696045},{"id":280,"token":" p","bytes":[32,112],"logprob":-14.898183822631836},{"id":907,"token":" ?","bytes":[32,63],"logprob":-14.911726951599121},{"id":318,"token":" (","bytes":[32,40],"logprob":-15.013163566589355},{"id":471,"token":" -","bytes":[32,45],"logprob":-15.053658485412598},{"id":313,"token":" {","bytes":[32,123],"logprob":-15.081854820251465}]} +{"id":18,"token":"3","bytes":[51],"logprob":-0.002082070102915168,"top_logprobs":[{"id":18,"token":"3","bytes":[51],"logprob":-0.002082070102915168},{"id":22,"token":"7","bytes":[55],"logprob":-6.525046348571777},{"id":20,"token":"5","bytes":[53],"logprob":-8.395028114318848},{"id":16,"token":"1","bytes":[49],"logprob":-8.572861671447754},{"id":19,"token":"4","bytes":[52],"logprob":-9.299103736877441},{"id":17,"token":"2","bytes":[50],"logprob":-9.372305870056152},{"id":21,"token":"6","bytes":[54],"logprob":-11.620535850524902},{"id":24,"token":"9","bytes":[57],"logprob":-11.837050437927246},{"id":23,"token":"8","bytes":[56],"logprob":-12.42276668548584},{"id":220,"token":" ","bytes":[32],"logprob":-14.371517181396484},{"id":15,"token":"0","bytes":[48],"logprob":-14.456452369689941},{"id":15428,"token":"","bytes":[226,130],"logprob":-14.575407981872559},{"id":41776,"token":"³","bytes":[194,179],"logprob":-16.389230728149414},{"id":12256,"token":"","bytes":[224,165],"logprob":-16.479965209960938},{"id":373,"token":"","bytes":[226,128],"logprob":-17.052135467529297},{"id":151,"token":"","bytes":[219],"logprob":-17.410802841186523},{"id":248044,"token":"","bytes":[],"logprob":-18.151866912841797},{"id":68086,"token":"","bytes":[226,133],"logprob":-18.24538230895996},{"id":26064,"token":"","bytes":[224,167],"logprob":-18.286691665649414},{"id":46701,"token":"","bytes":[226,145],"logprob":-18.30655288696289}]} +{"id":11,"token":",","bytes":[44],"logprob":-0.0000511420912516769,"top_logprobs":[{"id":11,"token":",","bytes":[44],"logprob":-0.0000511420912516769},{"id":248046,"token":"","bytes":[],"logprob":-11.382871627807617},{"id":20,"token":"5","bytes":[53],"logprob":-11.523244857788086},{"id":271,"token":"\n\n","bytes":[10,10],"logprob":-12.273974418640137},{"id":22,"token":"7","bytes":[55],"logprob":-12.333487510681152},{"id":1116,"token":" ,","bytes":[32,44],"logprob":-12.677053451538086},{"id":13,"token":".","bytes":[46],"logprob":-12.728805541992188},{"id":19,"token":"4","bytes":[52],"logprob":-12.984016418457031},{"id":26,"token":";","bytes":[59],"logprob":-12.997465133666992},{"id":321,"token":" and","bytes":[32,97,110,100],"logprob":-13.036134719848633},{"id":220,"token":" ","bytes":[32],"logprob":-13.07265567779541},{"id":248044,"token":"","bytes":[],"logprob":-14.216419219970703},{"id":198,"token":"\n","bytes":[10],"logprob":-14.286026954650879},{"id":16,"token":"1","bytes":[49],"logprob":-14.432971954345703},{"id":594,"token":" &","bytes":[32,38],"logprob":-14.813390731811523},{"id":318,"token":" (","bytes":[32,40],"logprob":-14.971389770507813},{"id":24,"token":"9","bytes":[57],"logprob":-15.023858070373535},{"id":373,"token":"","bytes":[226,128],"logprob":-15.390326499938965},{"id":67608,"token":",**","bytes":[44,42,42],"logprob":-15.44870376586914},{"id":25,"token":":","bytes":[58],"logprob":-15.490038871765137}]} +{"id":220,"token":" ","bytes":[32],"logprob":-0.003675112733617425,"top_logprobs":[{"id":220,"token":" ","bytes":[32],"logprob":-0.003675112733617425},{"id":321,"token":" and","bytes":[32,97,110,100],"logprob":-5.74519681930542},{"id":2972,"token":" **","bytes":[32,42,42],"logprob":-7.9954352378845215},{"id":594,"token":" &","bytes":[32,38],"logprob":-10.487424850463867},{"id":348,"token":" *","bytes":[32,42],"logprob":-10.515997886657715},{"id":373,"token":"","bytes":[226,128],"logprob":-11.031911849975586},{"id":280,"token":" p","bytes":[32,112],"logprob":-11.425256729125977},{"id":318,"token":" (","bytes":[32,40],"logprob":-11.488652229309082},{"id":2423,"token":" ...","bytes":[32,46,46,46],"logprob":-12.429529190063477},{"id":466,"token":" or","bytes":[32,111,114],"logprob":-12.537545204162598},{"id":1088,"token":" \\","bytes":[32,92],"logprob":-12.84799575805664},{"id":328,"token":" \"","bytes":[32,34],"logprob":-13.017975807189941},{"id":20,"token":"5","bytes":[53],"logprob":-13.118507385253906},{"id":471,"token":" -","bytes":[32,45],"logprob":-13.269417762756348},{"id":248046,"token":"","bytes":[],"logprob":-13.344802856445313},{"id":271,"token":"\n\n","bytes":[10,10],"logprob":-13.532476425170898},{"id":47770,"token":" primes","bytes":[32,112,114,105,109,101,115],"logprob":-13.72369384765625},{"id":2264,"token":" Pr","bytes":[32,80,114],"logprob":-13.795716285705566},{"id":907,"token":" ?","bytes":[32,63],"logprob":-13.803032875061035},{"id":534,"token":" pr","bytes":[32,112,114],"logprob":-14.011653900146484}]} +{"id":20,"token":"5","bytes":[53],"logprob":-0.0003994904982391745,"top_logprobs":[{"id":20,"token":"5","bytes":[53],"logprob":-0.0003994904982391745},{"id":22,"token":"7","bytes":[55],"logprob":-8.731183052062988},{"id":19,"token":"4","bytes":[52],"logprob":-9.11413860321045},{"id":16,"token":"1","bytes":[49],"logprob":-9.470494270324707},{"id":17,"token":"2","bytes":[50],"logprob":-10.665732383728027},{"id":18,"token":"3","bytes":[51],"logprob":-10.992241859436035},{"id":21,"token":"6","bytes":[54],"logprob":-12.596251487731934},{"id":15,"token":"0","bytes":[48],"logprob":-13.346444129943848},{"id":373,"token":"","bytes":[226,128],"logprob":-13.617030143737793},{"id":220,"token":" ","bytes":[32],"logprob":-13.774371147155762},{"id":24,"token":"9","bytes":[57],"logprob":-14.007845878601074},{"id":23,"token":"8","bytes":[56],"logprob":-14.447667121887207},{"id":12256,"token":"","bytes":[224,165],"logprob":-15.055480003356934},{"id":248046,"token":"","bytes":[],"logprob":-15.056328773498535},{"id":25229,"token":"½","bytes":[194,189],"logprob":-15.112027168273926},{"id":50382,"token":"","bytes":[226,129],"logprob":-15.138649940490723},{"id":151,"token":"","bytes":[219],"logprob":-15.147621154785156},{"id":321,"token":" and","bytes":[32,97,110,100],"logprob":-15.288090705871582},{"id":54352,"token":"........................","bytes":[46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46],"logprob":-15.318180084228516},{"id":15428,"token":"","bytes":[226,130],"logprob":-15.645711898803711}]} diff --git a/docs/audits/prm-001/c11-4026-live/p2.oracle_as_apr.json b/docs/audits/prm-001/c11-4026-live/p2.oracle_as_apr.json new file mode 100644 index 0000000000..fbc00b3015 --- /dev/null +++ b/docs/audits/prm-001/c11-4026-live/p2.oracle_as_apr.json @@ -0,0 +1,612 @@ +{ + "choices": [ + { + "logprobs": { + "content": [ + { + "token_id": 17, + "logprob": -0.8558096885681152, + "top_logprobs": [ + { + "token_id": 17, + "logprob": -0.8558096885681152 + }, + { + "token_id": 16, + "logprob": -1.1232353448867798 + }, + { + "token_id": 760, + "logprob": -2.1764369010925293 + }, + { + "token_id": 18, + "logprob": -2.7710108757019043 + }, + { + "token_id": 20, + "logprob": -3.842771053314209 + }, + { + "token_id": 22, + "logprob": -4.1483073234558105 + }, + { + "token_id": 19, + "logprob": -5.159671306610107 + }, + { + "token_id": 15, + "logprob": -5.1988844871521 + }, + { + "token_id": 248068, + "logprob": -5.721559047698975 + }, + { + "token_id": 5170, + "logprob": -5.852420330047607 + }, + { + "token_id": 2053, + "logprob": -6.441211223602295 + }, + { + "token_id": 8160, + "logprob": -6.511935710906982 + }, + { + "token_id": 9764, + "logprob": -6.551112651824951 + }, + { + "token_id": 1206, + "logprob": -6.5843634605407715 + }, + { + "token_id": 21, + "logprob": -6.708505153656006 + }, + { + "token_id": 24, + "logprob": -6.875026226043701 + }, + { + "token_id": 32, + "logprob": -7.424076557159424 + }, + { + "token_id": 3733, + "logprob": -7.494142055511475 + }, + { + "token_id": 27775, + "logprob": -7.511061191558838 + }, + { + "token_id": 11280, + "logprob": -7.72383451461792 + } + ] + }, + { + "token_id": 11, + "logprob": -0.005613376386463642, + "top_logprobs": [ + { + "token_id": 11, + "logprob": -0.005613376386463642 + }, + { + "token_id": 271, + "logprob": -6.123464584350586 + }, + { + "token_id": 198, + "logprob": -7.001176834106445 + }, + { + "token_id": 248046, + "logprob": -7.212923049926758 + }, + { + "token_id": 18, + "logprob": -8.070806503295898 + }, + { + "token_id": 22, + "logprob": -8.370685577392578 + }, + { + "token_id": 16, + "logprob": -8.451520919799805 + }, + { + "token_id": 24, + "logprob": -8.786211013793945 + }, + { + "token_id": 13, + "logprob": -8.92701530456543 + }, + { + "token_id": 20, + "logprob": -9.021446228027344 + }, + { + "token_id": 321, + "logprob": -9.17351245880127 + }, + { + "token_id": 15, + "logprob": -9.340082168579102 + }, + { + "token_id": 220, + "logprob": -9.380533218383789 + }, + { + "token_id": 26, + "logprob": -10.0845947265625 + }, + { + "token_id": 19, + "logprob": -10.19825267791748 + }, + { + "token_id": 2228, + "logprob": -10.341672897338867 + }, + { + "token_id": 1116, + "logprob": -10.544290542602539 + }, + { + "token_id": 17, + "logprob": -10.626684188842773 + }, + { + "token_id": 21, + "logprob": -10.637521743774414 + }, + { + "token_id": 248044, + "logprob": -11.010668754577637 + } + ] + }, + { + "token_id": 220, + "logprob": -0.000283102534012869, + "top_logprobs": [ + { + "token_id": 220, + "logprob": -0.000283102534012869 + }, + { + "token_id": 2972, + "logprob": -8.847826957702637 + }, + { + "token_id": 18, + "logprob": -9.443541526794434 + }, + { + "token_id": 373, + "logprob": -9.978693962097168 + }, + { + "token_id": 348, + "logprob": -13.40364933013916 + }, + { + "token_id": 2423, + "logprob": -13.779648780822754 + }, + { + "token_id": 198, + "logprob": -13.994604110717773 + }, + { + "token_id": 321, + "logprob": -14.195822715759277 + }, + { + "token_id": 22, + "logprob": -14.497540473937988 + }, + { + "token_id": 2250, + "logprob": -14.633594512939453 + }, + { + "token_id": 524, + "logprob": -14.649407386779785 + }, + { + "token_id": 685, + "logprob": -14.685708999633789 + }, + { + "token_id": 58317, + "logprob": -14.81330394744873 + }, + { + "token_id": 3966, + "logprob": -14.854573249816895 + }, + { + "token_id": 15428, + "logprob": -14.89685344696045 + }, + { + "token_id": 280, + "logprob": -14.898183822631836 + }, + { + "token_id": 907, + "logprob": -14.911726951599121 + }, + { + "token_id": 318, + "logprob": -15.013163566589355 + }, + { + "token_id": 471, + "logprob": -15.053658485412598 + }, + { + "token_id": 313, + "logprob": -15.081854820251465 + } + ] + }, + { + "token_id": 18, + "logprob": -0.002082070102915168, + "top_logprobs": [ + { + "token_id": 18, + "logprob": -0.002082070102915168 + }, + { + "token_id": 22, + "logprob": -6.525046348571777 + }, + { + "token_id": 20, + "logprob": -8.395028114318848 + }, + { + "token_id": 16, + "logprob": -8.572861671447754 + }, + { + "token_id": 19, + "logprob": -9.299103736877441 + }, + { + "token_id": 17, + "logprob": -9.372305870056152 + }, + { + "token_id": 21, + "logprob": -11.620535850524902 + }, + { + "token_id": 24, + "logprob": -11.837050437927246 + }, + { + "token_id": 23, + "logprob": -12.42276668548584 + }, + { + "token_id": 220, + "logprob": -14.371517181396484 + }, + { + "token_id": 15, + "logprob": -14.456452369689941 + }, + { + "token_id": 15428, + "logprob": -14.575407981872559 + }, + { + "token_id": 41776, + "logprob": -16.389230728149414 + }, + { + "token_id": 12256, + "logprob": -16.479965209960938 + }, + { + "token_id": 373, + "logprob": -17.052135467529297 + }, + { + "token_id": 151, + "logprob": -17.410802841186523 + }, + { + "token_id": 248044, + "logprob": -18.151866912841797 + }, + { + "token_id": 68086, + "logprob": -18.24538230895996 + }, + { + "token_id": 26064, + "logprob": -18.286691665649414 + }, + { + "token_id": 46701, + "logprob": -18.30655288696289 + } + ] + }, + { + "token_id": 11, + "logprob": -0.0000511420912516769, + "top_logprobs": [ + { + "token_id": 11, + "logprob": -0.0000511420912516769 + }, + { + "token_id": 248046, + "logprob": -11.382871627807617 + }, + { + "token_id": 20, + "logprob": -11.523244857788086 + }, + { + "token_id": 271, + "logprob": -12.273974418640137 + }, + { + "token_id": 22, + "logprob": -12.333487510681152 + }, + { + "token_id": 1116, + "logprob": -12.677053451538086 + }, + { + "token_id": 13, + "logprob": -12.728805541992188 + }, + { + "token_id": 19, + "logprob": -12.984016418457031 + }, + { + "token_id": 26, + "logprob": -12.997465133666992 + }, + { + "token_id": 321, + "logprob": -13.036134719848633 + }, + { + "token_id": 220, + "logprob": -13.07265567779541 + }, + { + "token_id": 248044, + "logprob": -14.216419219970703 + }, + { + "token_id": 198, + "logprob": -14.286026954650879 + }, + { + "token_id": 16, + "logprob": -14.432971954345703 + }, + { + "token_id": 594, + "logprob": -14.813390731811523 + }, + { + "token_id": 318, + "logprob": -14.971389770507813 + }, + { + "token_id": 24, + "logprob": -15.023858070373535 + }, + { + "token_id": 373, + "logprob": -15.390326499938965 + }, + { + "token_id": 67608, + "logprob": -15.44870376586914 + }, + { + "token_id": 25, + "logprob": -15.490038871765137 + } + ] + }, + { + "token_id": 220, + "logprob": -0.003675112733617425, + "top_logprobs": [ + { + "token_id": 220, + "logprob": -0.003675112733617425 + }, + { + "token_id": 321, + "logprob": -5.74519681930542 + }, + { + "token_id": 2972, + "logprob": -7.9954352378845215 + }, + { + "token_id": 594, + "logprob": -10.487424850463867 + }, + { + "token_id": 348, + "logprob": -10.515997886657715 + }, + { + "token_id": 373, + "logprob": -11.031911849975586 + }, + { + "token_id": 280, + "logprob": -11.425256729125977 + }, + { + "token_id": 318, + "logprob": -11.488652229309082 + }, + { + "token_id": 2423, + "logprob": -12.429529190063477 + }, + { + "token_id": 466, + "logprob": -12.537545204162598 + }, + { + "token_id": 1088, + "logprob": -12.84799575805664 + }, + { + "token_id": 328, + "logprob": -13.017975807189941 + }, + { + "token_id": 20, + "logprob": -13.118507385253906 + }, + { + "token_id": 471, + "logprob": -13.269417762756348 + }, + { + "token_id": 248046, + "logprob": -13.344802856445313 + }, + { + "token_id": 271, + "logprob": -13.532476425170898 + }, + { + "token_id": 47770, + "logprob": -13.72369384765625 + }, + { + "token_id": 2264, + "logprob": -13.795716285705566 + }, + { + "token_id": 907, + "logprob": -13.803032875061035 + }, + { + "token_id": 534, + "logprob": -14.011653900146484 + } + ] + }, + { + "token_id": 20, + "logprob": -0.0003994904982391745, + "top_logprobs": [ + { + "token_id": 20, + "logprob": -0.0003994904982391745 + }, + { + "token_id": 22, + "logprob": -8.731183052062988 + }, + { + "token_id": 19, + "logprob": -9.11413860321045 + }, + { + "token_id": 16, + "logprob": -9.470494270324707 + }, + { + "token_id": 17, + "logprob": -10.665732383728027 + }, + { + "token_id": 18, + "logprob": -10.992241859436035 + }, + { + "token_id": 21, + "logprob": -12.596251487731934 + }, + { + "token_id": 15, + "logprob": -13.346444129943848 + }, + { + "token_id": 373, + "logprob": -13.617030143737793 + }, + { + "token_id": 220, + "logprob": -13.774371147155762 + }, + { + "token_id": 24, + "logprob": -14.007845878601074 + }, + { + "token_id": 23, + "logprob": -14.447667121887207 + }, + { + "token_id": 12256, + "logprob": -15.055480003356934 + }, + { + "token_id": 248046, + "logprob": -15.056328773498535 + }, + { + "token_id": 25229, + "logprob": -15.112027168273926 + }, + { + "token_id": 50382, + "logprob": -15.138649940490723 + }, + { + "token_id": 151, + "logprob": -15.147621154785156 + }, + { + "token_id": 321, + "logprob": -15.288090705871582 + }, + { + "token_id": 54352, + "logprob": -15.318180084228516 + }, + { + "token_id": 15428, + "logprob": -15.645711898803711 + } + ] + } + ] + } + } + ] +} diff --git a/docs/audits/prm-001/c11-4026-live/p2.posctl.json b/docs/audits/prm-001/c11-4026-live/p2.posctl.json new file mode 100644 index 0000000000..4b0fc7592f --- /dev/null +++ b/docs/audits/prm-001/c11-4026-live/p2.posctl.json @@ -0,0 +1,68 @@ +{ + "steps": [ + { + "step": 0, + "top1_apr": 17, + "top1_oracle": 17, + "overlap": 20, + "max_abs_dlogprob": 0, + "chosen_dlogprob": 0, + "apr_mass": 0.9911340444251375 + }, + { + "step": 1, + "top1_apr": 11, + "top1_oracle": 11, + "overlap": 20, + "max_abs_dlogprob": 0, + "chosen_dlogprob": 0, + "apr_mass": 0.9998833179728863 + }, + { + "step": 2, + "top1_apr": 220, + "top1_oracle": 220, + "overlap": 20, + "max_abs_dlogprob": 0, + "chosen_dlogprob": 0, + "apr_mass": 0.9999946822338857 + }, + { + "step": 3, + "top1_apr": 18, + "top1_oracle": 18, + "overlap": 20, + "max_abs_dlogprob": 0, + "chosen_dlogprob": 0, + "apr_mass": 1.0000001245443475 + }, + { + "step": 4, + "top1_apr": 11, + "top1_oracle": 11, + "overlap": 20, + "max_abs_dlogprob": 0, + "chosen_dlogprob": 0, + "apr_mass": 0.9999975559249618 + }, + { + "step": 5, + "top1_apr": 220, + "top1_oracle": 220, + "overlap": 20, + "max_abs_dlogprob": 0, + "chosen_dlogprob": 0, + "apr_mass": 0.9999821107833562 + }, + { + "step": 6, + "top1_apr": 20, + "top1_oracle": 20, + "overlap": 20, + "max_abs_dlogprob": 0, + "chosen_dlogprob": 0, + "apr_mass": 1.0000000103717157 + } + ], + "pass": true +} diff --git a/docs/audits/prm-001/c11-4026-live/p3.apr.json b/docs/audits/prm-001/c11-4026-live/p3.apr.json new file mode 100644 index 0000000000..9f2bc8957f --- /dev/null +++ b/docs/audits/prm-001/c11-4026-live/p3.apr.json @@ -0,0 +1 @@ +{"id":"chatcmpl-q4k-1790386253890","object":"chat.completion","created":1790386255,"model":"q","choices":[{"index":0,"message":{"role":"assistant","content":"A primary color is a color that is"},"finish_reason":"length","logprobs":{"content":[{"token":"A","token_id":32,"logprob":-0.9557094573974609,"bytes":null,"top_logprobs":[{"token":"A","token_id":32,"logprob":-0.9557094573974609,"bytes":null},{"token":"**","token_id":332,"logprob":-1.3799381256103516,"bytes":null},{"token":"The","token_id":760,"logprob":-1.6623249053955078,"bytes":null},{"token":"There","token_id":3733,"logprob":-3.487852096557617,"bytes":null},{"token":"One","token_id":3833,"logprob":-3.576862335205078,"bytes":null},{"token":"Primary","token_id":15500,"logprob":-3.7040462493896484,"bytes":null},{"token":"While","token_id":7732,"logprob":-4.382585525512695,"bytes":null},{"token":"Common","token_id":10522,"logprob":-5.221151351928711,"bytes":null},{"token":"In","token_id":623,"logprob":-5.415431976318359,"bytes":null},{"token":"Name","token_id":657,"logprob":-5.4887237548828125,"bytes":null},{"token":"I","token_id":40,"logprob":-5.655656814575195,"bytes":null},{"token":"An","token_id":2014,"logprob":-5.785909652709961,"bytes":null},{"token":"Several","token_id":33613,"logprob":-5.938283920288086,"bytes":null},{"token":"Many","token_id":8181,"logprob":-6.19947624206543,"bytes":null},{"token":"Based","token_id":27775,"logprob":-6.242273330688477,"bytes":null},{"token":"White","token_id":13672,"logprob":-6.247941970825195,"bytes":null},{"token":"Some","token_id":8114,"logprob":-6.40672492980957,"bytes":null},{"token":"Any","token_id":8347,"logprob":-6.411373138427734,"bytes":null},{"token":"Among","token_id":32498,"logprob":-6.436189651489258,"bytes":null},{"token":"Colors","token_id":12728,"logprob":-6.479837417602539,"bytes":null}],"logsumexp_full":23.141611099243164},{"token":" primary","token_id":5839,"logprob":-0.09716224670410156,"bytes":null,"top_logprobs":[{"token":" primary","token_id":5839,"logprob":-0.09716224670410156,"bytes":null},{"token":" **","token_id":2972,"logprob":-2.8211002349853516,"bytes":null},{"token":" common","token_id":4047,"logprob":-4.112998962402344,"bytes":null},{"token":" very","token_id":1546,"logprob":-5.354587554931641,"bytes":null},{"token":" widely","token_id":13177,"logprob":-6.189714431762695,"bytes":null},{"token":" classic","token_id":11088,"logprob":-6.804985046386719,"bytes":null},{"token":" standard","token_id":5129,"logprob":-7.432989120483398,"bytes":null},{"token":" well","token_id":1575,"logprob":-7.558977127075195,"bytes":null},{"token":" universal","token_id":19565,"logprob":-7.631635665893555,"bytes":null},{"token":" typical","token_id":13852,"logprob":-7.730691909790039,"bytes":null},{"token":" true","token_id":804,"logprob":-7.829723358154297,"bytes":null},{"token":" commonly","token_id":16138,"logprob":-7.9703216552734375,"bytes":null},{"token":" variety","token_id":7792,"logprob":-8.148092269897461,"bytes":null},{"token":" few","token_id":2342,"logprob":-8.297765731811523,"bytes":null},{"token":" simple","token_id":4145,"logprob":-8.306324005126953,"bytes":null},{"token":" pure","token_id":10213,"logprob":-8.438831329345703,"bytes":null},{"token":" fundamental","token_id":15346,"logprob":-8.467671394348145,"bytes":null},{"token":" traditional","token_id":8343,"logprob":-8.500631332397461,"bytes":null},{"token":" name","token_id":803,"logprob":-8.648598670959473,"bytes":null},{"token":" clear","token_id":2708,"logprob":-8.671609878540039,"bytes":null}],"logsumexp_full":24.456233978271484},{"token":" color","token_id":1829,"logprob":-0.004276275634765625,"bytes":null,"top_logprobs":[{"token":" color","token_id":1829,"logprob":-0.004276275634765625,"bytes":null},{"token":" (","token_id":318,"logprob":-6.661455154418945,"bytes":null},{"token":" **","token_id":2972,"logprob":-7.349172592163086,"bytes":null},{"token":" light","token_id":3000,"logprob":-7.497528076171875,"bytes":null},{"token":" red","token_id":2438,"logprob":-8.474514961242676,"bytes":null},{"token":" rainbow","token_id":46026,"logprob":-9.020284652709961,"bytes":null},{"token":" human","token_id":3611,"logprob":-9.265497207641602,"bytes":null},{"token":" primary","token_id":5839,"logprob":-9.384819984436035,"bytes":null},{"token":" pigment","token_id":73959,"logprob":-9.408422470092773,"bytes":null},{"token":" paint","token_id":5984,"logprob":-9.714305877685547,"bytes":null},{"token":" water","token_id":2919,"logprob":-9.849031448364258,"bytes":null},{"token":" natural","token_id":5629,"logprob":-10.004255294799805,"bytes":null},{"token":" R","token_id":423,"logprob":-10.040426254272461,"bytes":null},{"token":" colour","token_id":12106,"logprob":-10.291252136230469,"bytes":null},{"token":" object","token_id":1576,"logprob":-10.408336639404297,"bytes":null},{"token":" common","token_id":4047,"logprob":-10.519460678100586,"bytes":null},{"token":",","token_id":11,"logprob":-10.641770362854004,"bytes":null},{"token":" in","token_id":303,"logprob":-10.654306411743164,"bytes":null},{"token":" art","token_id":1880,"logprob":-10.693977355957031,"bytes":null},{"token":" is","token_id":369,"logprob":-10.892050743103027,"bytes":null}],"logsumexp_full":22.899158477783203},{"token":" is","token_id":369,"logprob":-0.2619762420654297,"bytes":null,"top_logprobs":[{"token":" is","token_id":369,"logprob":-0.2619762420654297,"bytes":null},{"token":" in","token_id":303,"logprob":-1.6903209686279297,"bytes":null},{"token":" on","token_id":383,"logprob":-4.778703689575195,"bytes":null},{"token":" (","token_id":318,"logprob":-5.135364532470703,"bytes":null},{"token":" can","token_id":628,"logprob":-5.24024772644043,"bytes":null},{"token":" for","token_id":364,"logprob":-5.303562164306641,"bytes":null},{"token":",","token_id":11,"logprob":-5.424247741699219,"bytes":null},{"token":" of","token_id":314,"logprob":-6.357732772827148,"bytes":null},{"token":" used","token_id":1429,"logprob":-6.4456939697265625,"bytes":null},{"token":" refers","token_id":18675,"logprob":-6.646421432495117,"bytes":null},{"token":" from","token_id":494,"logprob":-6.684572219848633,"bytes":null},{"token":" among","token_id":4082,"logprob":-7.303762435913086,"bytes":null},{"token":" consists","token_id":16661,"logprob":-7.37830924987793,"bytes":null},{"token":" usually","token_id":5802,"logprob":-7.419183731079102,"bytes":null},{"token":" generally","token_id":8524,"logprob":-7.435445785522461,"bytes":null},{"token":" that","token_id":421,"logprob":-7.538309097290039,"bytes":null},{"token":" are","token_id":513,"logprob":-7.608177185058594,"bytes":null},{"token":" typically","token_id":10813,"logprob":-7.617332458496094,"bytes":null},{"token":" name","token_id":803,"logprob":-7.673348426818848,"bytes":null},{"token":" was","token_id":557,"logprob":-7.991828918457031,"bytes":null}],"logsumexp_full":23.128408432006836},{"token":" a","token_id":264,"logprob":-1.0765342712402344,"bytes":null,"top_logprobs":[{"token":" a","token_id":264,"logprob":-1.0765342712402344,"bytes":null},{"token":" one","token_id":799,"logprob":-1.2006092071533203,"bytes":null},{"token":" the","token_id":279,"logprob":-2.7909107208251953,"bytes":null},{"token":" defined","token_id":4364,"logprob":-2.8255882263183594,"bytes":null},{"token":" any","token_id":866,"logprob":-2.8530445098876953,"bytes":null},{"token":" **","token_id":2972,"logprob":-3.156780242919922,"bytes":null},{"token":" generally","token_id":8524,"logprob":-3.8859195709228516,"bytes":null},{"token":" typically","token_id":10813,"logprob":-4.170167922973633,"bytes":null},{"token":" usually","token_id":5802,"logprob":-4.412921905517578,"bytes":null},{"token":" an","token_id":449,"logprob":-4.632009506225586,"bytes":null},{"token":" often","token_id":3428,"logprob":-4.828853607177734,"bytes":null},{"token":" named","token_id":6725,"logprob":-4.960039138793945,"bytes":null},{"token":" determined","token_id":10521,"logprob":-4.985942840576172,"bytes":null},{"token":" always","token_id":2590,"logprob":-5.642122268676758,"bytes":null},{"token":" traditionally","token_id":34075,"logprob":-5.7201128005981445,"bytes":null},{"token":" widely","token_id":13177,"logprob":-5.841684341430664,"bytes":null},{"token":" called","token_id":2512,"logprob":-5.844677925109863,"bytes":null},{"token":" represented","token_id":14807,"logprob":-5.964906692504883,"bytes":null},{"token":" used","token_id":1429,"logprob":-6.017099380493164,"bytes":null},{"token":" simply","token_id":4777,"logprob":-6.030526161193848,"bytes":null}],"logsumexp_full":21.708181381225586},{"token":" color","token_id":1829,"logprob":-0.9037990570068359,"bytes":null,"top_logprobs":[{"token":" color","token_id":1829,"logprob":-0.9037990570068359,"bytes":null},{"token":" specific","token_id":3050,"logprob":-2.7458744049072266,"bytes":null},{"token":" primary","token_id":5839,"logprob":-2.8171939849853516,"bytes":null},{"token":" fundamental","token_id":15346,"logprob":-2.8977222442626953,"bytes":null},{"token":" basic","token_id":6557,"logprob":-2.989988327026367,"bytes":null},{"token":" pigment","token_id":73959,"logprob":-3.0648422241210938,"bytes":null},{"token":" broad","token_id":6979,"logprob":-3.5582027435302734,"bytes":null},{"token":" term","token_id":4496,"logprob":-3.653287887573242,"bytes":null},{"token":" set","token_id":716,"logprob":-3.770282745361328,"bytes":null},{"token":" **","token_id":2972,"logprob":-3.8580474853515625,"bytes":null},{"token":" name","token_id":803,"logprob":-4.163837432861328,"bytes":null},{"token":" hue","token_id":37747,"logprob":-4.385702133178711,"bytes":null},{"token":" category","token_id":5407,"logprob":-4.57133674621582,"bytes":null},{"token":" shade","token_id":26609,"logprob":-4.608417510986328,"bytes":null},{"token":" standard","token_id":5129,"logprob":-4.672388076782227,"bytes":null},{"token":" type","token_id":913,"logprob":-4.71540641784668,"bytes":null},{"token":" pure","token_id":10213,"logprob":-4.891551971435547,"bytes":null},{"token":" group","token_id":1809,"logprob":-5.013057708740234,"bytes":null},{"token":" single","token_id":3074,"logprob":-5.154747009277344,"bytes":null},{"token":" paint","token_id":5984,"logprob":-5.230124473571777,"bytes":null}],"logsumexp_full":21.222383499145508},{"token":" that","token_id":421,"logprob":-0.7661304473876953,"bytes":null,"top_logprobs":[{"token":" that","token_id":421,"logprob":-0.7661304473876953,"bytes":null},{"token":" used","token_id":1429,"logprob":-2.2215442657470703,"bytes":null},{"token":" on","token_id":383,"logprob":-2.893228530883789,"bytes":null},{"token":" in","token_id":303,"logprob":-2.989076614379883,"bytes":null},{"token":" from","token_id":494,"logprob":-3.5175704956054688,"bytes":null},{"token":" chosen","token_id":11543,"logprob":-3.674386978149414,"bytes":null},{"token":" of","token_id":314,"logprob":-3.8426055908203125,"bytes":null},{"token":" found","token_id":1669,"logprob":-4.481485366821289,"bytes":null},{"token":" name","token_id":803,"logprob":-4.573665618896484,"bytes":null},{"token":" added","token_id":3568,"logprob":-4.616769790649414,"bytes":null},{"token":" named","token_id":6725,"logprob":-4.73846435546875,"bytes":null},{"token":" formed","token_id":13717,"logprob":-4.819812774658203,"bytes":null},{"token":" with","token_id":440,"logprob":-4.865320205688477,"bytes":null},{"token":" represented","token_id":14807,"logprob":-4.9525299072265625,"bytes":null},{"token":" present","token_id":2945,"logprob":-5.124289512634277,"bytes":null},{"token":" mixed","token_id":9238,"logprob":-5.15694522857666,"bytes":null},{"token":" whose","token_id":6483,"logprob":-5.160087585449219,"bytes":null},{"token":" mix","token_id":6311,"logprob":-5.292088508605957,"bytes":null},{"token":" pair","token_id":6505,"logprob":-5.293905258178711,"bytes":null},{"token":" defined","token_id":4364,"logprob":-5.319124221801758,"bytes":null}],"logsumexp_full":20.97443199157715},{"token":" is","token_id":369,"logprob":-0.9548187255859375,"bytes":null,"top_logprobs":[{"token":" is","token_id":369,"logprob":-0.9548187255859375,"bytes":null},{"token":" cannot","token_id":4021,"logprob":-2.2097854614257812,"bytes":null},{"token":" has","token_id":682,"logprob":-2.883981704711914,"bytes":null},{"token":",","token_id":11,"logprob":-3.0042896270751953,"bytes":null},{"token":" combines","token_id":31341,"logprob":-3.2171287536621094,"bytes":null},{"token":" exists","token_id":6513,"logprob":-3.3705406188964844,"bytes":null},{"token":" can","token_id":628,"logprob":-3.616985321044922,"bytes":null},{"token":" appears","token_id":7701,"logprob":-3.8516807556152344,"bytes":null},{"token":" must","token_id":1902,"logprob":-3.852071762084961,"bytes":null},{"token":" stands","token_id":12965,"logprob":-4.043643951416016,"bytes":null},{"token":" makes","token_id":3520,"logprob":-4.319679260253906,"bytes":null},{"token":" forms","token_id":7346,"logprob":-4.384073257446289,"bytes":null},{"token":" a","token_id":264,"logprob":-4.397296905517578,"bytes":null},{"token":" occurs","token_id":13273,"logprob":-4.466228485107422,"bytes":null},{"token":" light","token_id":3000,"logprob":-4.628517150878906,"bytes":null},{"token":" the","token_id":279,"logprob":-4.764959335327148,"bytes":null},{"token":" lies","token_id":14995,"logprob":-4.779807090759277,"bytes":null},{"token":" adds","token_id":11039,"logprob":-4.8570661544799805,"bytes":null},{"token":" does","token_id":1503,"logprob":-5.036294937133789,"bytes":null},{"token":" represents","token_id":10551,"logprob":-5.051392555236816,"bytes":null}],"logsumexp_full":20.414379119873047}]}}],"usage":{"prompt_tokens":17,"completion_tokens":8,"total_tokens":25}} \ No newline at end of file diff --git a/docs/audits/prm-001/c11-4026-live/p3.cmp.json b/docs/audits/prm-001/c11-4026-live/p3.cmp.json new file mode 100644 index 0000000000..5ea500c7ad --- /dev/null +++ b/docs/audits/prm-001/c11-4026-live/p3.cmp.json @@ -0,0 +1,77 @@ +{ + "steps": [ + { + "step": 0, + "top1_apr": 32, + "top1_oracle": 32, + "overlap": 18, + "max_abs_dlogprob": 0.30834031105041504, + "chosen_dlogprob": 0.11582398414611816, + "apr_mass": 0.9570037367672385 + }, + { + "step": 1, + "top1_apr": 5839, + "top1_oracle": 5839, + "overlap": 19, + "max_abs_dlogprob": 0.26138973236083984, + "chosen_dlogprob": 0.007406465709209442, + "apr_mass": 0.9957348750925334 + }, + { + "step": 2, + "top1_apr": 1829, + "top1_oracle": 1829, + "overlap": 19, + "max_abs_dlogprob": 0.3337841033935547, + "chosen_dlogprob": 1.9443687051534653e-05, + "apr_mass": 0.9991819372269393 + }, + { + "step": 3, + "top1_apr": 369, + "top1_oracle": 369, + "overlap": 19, + "max_abs_dlogprob": 0.32706785202026367, + "chosen_dlogprob": 0.02656775712966919, + "apr_mass": 0.993642985307549 + }, + { + "step": 4, + "top1_apr": 264, + "top1_oracle": 264, + "overlap": 17, + "max_abs_dlogprob": 0.19165945053100586, + "chosen_dlogprob": 0.0014759302139282227, + "apr_mass": 0.962362714201912 + }, + { + "step": 5, + "top1_apr": 1829, + "top1_oracle": 1829, + "overlap": 19, + "max_abs_dlogprob": 0.23700666427612305, + "chosen_dlogprob": 0.030861258506774902, + "apr_mass": 0.871546881695331 + }, + { + "step": 6, + "top1_apr": 421, + "top1_oracle": 421, + "overlap": 19, + "max_abs_dlogprob": 0.20099973678588867, + "chosen_dlogprob": 0.05885547399520874, + "apr_mass": 0.8509725983302818 + }, + { + "step": 7, + "top1_apr": 369, + "top1_oracle": 369, + "overlap": 20, + "max_abs_dlogprob": 0.20266485214233398, + "chosen_dlogprob": 0.014675676822662354, + "apr_mass": 0.8583507145161019 + } + ], + "pass": false +} diff --git a/docs/audits/prm-001/c11-4026-live/p3.ids.json b/docs/audits/prm-001/c11-4026-live/p3.ids.json new file mode 100644 index 0000000000..68cf8139fc --- /dev/null +++ b/docs/audits/prm-001/c11-4026-live/p3.ids.json @@ -0,0 +1 @@ +{"tokens":[248045,846,198,657,264,5839,1829,13,248046,198,248045,74455,198,248068,271,248069,271]} \ No newline at end of file diff --git a/docs/audits/prm-001/c11-4026-live/p3.oracle.jsonl b/docs/audits/prm-001/c11-4026-live/p3.oracle.jsonl new file mode 100644 index 0000000000..e06429f420 --- /dev/null +++ b/docs/audits/prm-001/c11-4026-live/p3.oracle.jsonl @@ -0,0 +1,8 @@ +{"id":32,"token":"A","bytes":[65],"logprob":-1.071533441543579,"top_logprobs":[{"id":32,"token":"A","bytes":[65],"logprob":-1.071533441543579},{"id":332,"token":"**","bytes":[42,42],"logprob":-1.1200430393218994},{"id":760,"token":"The","bytes":[84,104,101],"logprob":-1.8397066593170166},{"id":3733,"token":"There","bytes":[84,104,101,114,101],"logprob":-3.579895257949829},{"id":15500,"token":"Primary","bytes":[80,114,105,109,97,114,121],"logprob":-3.630493640899658},{"id":3833,"token":"One","bytes":[79,110,101],"logprob":-3.885202646255493},{"id":7732,"token":"While","bytes":[87,104,105,108,101],"logprob":-4.172399997711182},{"id":657,"token":"Name","bytes":[78,97,109,101],"logprob":-5.336533069610596},{"id":623,"token":"In","bytes":[73,110],"logprob":-5.352776050567627},{"id":10522,"token":"Common","bytes":[67,111,109,109,111,110],"logprob":-5.444912433624268},{"id":40,"token":"I","bytes":[73],"logprob":-5.463242053985596},{"id":2014,"token":"An","bytes":[65,110],"logprob":-5.874866962432861},{"id":27775,"token":"Based","bytes":[66,97,115,101,100],"logprob":-5.935917377471924},{"id":13672,"token":"White","bytes":[87,104,105,116,101],"logprob":-6.026829242706299},{"id":14004,"token":"Black","bytes":[66,108,97,99,107],"logprob":-6.098835468292236},{"id":33613,"token":"Several","bytes":[83,101,118,101,114,97,108],"logprob":-6.1494140625},{"id":8114,"token":"Some","bytes":[83,111,109,101],"logprob":-6.365744113922119},{"id":10025,"token":"Blue","bytes":[66,108,117,101],"logprob":-6.449535846710205},{"id":8181,"token":"Many","bytes":[77,97,110,121],"logprob":-6.457029819488525},{"id":8347,"token":"Any","bytes":[65,110,121],"logprob":-6.559885501861572}]} +{"id":5839,"token":" primary","bytes":[32,112,114,105,109,97,114,121],"logprob":-0.08975578099489212,"top_logprobs":[{"id":5839,"token":" primary","bytes":[32,112,114,105,109,97,114,121],"logprob":-0.08975578099489212},{"id":2972,"token":" **","bytes":[32,42,42],"logprob":-2.9284169673919678},{"id":4047,"token":" common","bytes":[32,99,111,109,109,111,110],"logprob":-4.1467719078063965},{"id":1546,"token":" very","bytes":[32,118,101,114,121],"logprob":-5.481477737426758},{"id":13177,"token":" widely","bytes":[32,119,105,100,101,108,121],"logprob":-6.161518096923828},{"id":11088,"token":" classic","bytes":[32,99,108,97,115,115,105,99],"logprob":-6.93243932723999},{"id":5129,"token":" standard","bytes":[32,115,116,97,110,100,97,114,100],"logprob":-7.340293884277344},{"id":1575,"token":" well","bytes":[32,119,101,108,108],"logprob":-7.500030517578125},{"id":19565,"token":" universal","bytes":[32,117,110,105,118,101,114,115,97,108],"logprob":-7.520591735839844},{"id":804,"token":" true","bytes":[32,116,114,117,101],"logprob":-7.710727214813232},{"id":13852,"token":" typical","bytes":[32,116,121,112,105,99,97,108],"logprob":-7.743495941162109},{"id":16138,"token":" commonly","bytes":[32,99,111,109,109,111,110,108,121],"logprob":-7.855956554412842},{"id":7792,"token":" variety","bytes":[32,118,97,114,105,101,116,121],"logprob":-8.20311164855957},{"id":15346,"token":" fundamental","bytes":[32,102,117,110,100,97,109,101,110,116,97,108],"logprob":-8.206281661987305},{"id":8343,"token":" traditional","bytes":[32,116,114,97,100,105,116,105,111,110,97,108],"logprob":-8.419899940490723},{"id":2342,"token":" few","bytes":[32,102,101,119],"logprob":-8.421042442321777},{"id":4145,"token":" simple","bytes":[32,115,105,109,112,108,101],"logprob":-8.445547103881836},{"id":10213,"token":" pure","bytes":[32,112,117,114,101],"logprob":-8.46473217010498},{"id":803,"token":" name","bytes":[32,110,97,109,101],"logprob":-8.495408058166504},{"id":58391,"token":" universally","bytes":[32,117,110,105,118,101,114,115,97,108,108,121],"logprob":-8.577983856201172}]} +{"id":1829,"token":" color","bytes":[32,99,111,108,111,114],"logprob":-0.00425683194771409,"top_logprobs":[{"id":1829,"token":" color","bytes":[32,99,111,108,111,114],"logprob":-0.00425683194771409},{"id":318,"token":" (","bytes":[32,40],"logprob":-6.518619060516357},{"id":2972,"token":" **","bytes":[32,42,42],"logprob":-7.577304363250732},{"id":3000,"token":" light","bytes":[32,108,105,103,104,116],"logprob":-7.603311061859131},{"id":2438,"token":" red","bytes":[32,114,101,100],"logprob":-8.677526473999023},{"id":46026,"token":" rainbow","bytes":[32,114,97,105,110,98,111,119],"logprob":-8.686500549316406},{"id":3611,"token":" human","bytes":[32,104,117,109,97,110],"logprob":-9.06228256225586},{"id":5839,"token":" primary","bytes":[32,112,114,105,109,97,114,121],"logprob":-9.334321975708008},{"id":73959,"token":" pigment","bytes":[32,112,105,103,109,101,110,116],"logprob":-9.409225463867188},{"id":5984,"token":" paint","bytes":[32,112,97,105,110,116],"logprob":-9.688760757446289},{"id":423,"token":" R","bytes":[32,82],"logprob":-9.926101684570313},{"id":2919,"token":" water","bytes":[32,119,97,116,101,114],"logprob":-9.952198028564453},{"id":5629,"token":" natural","bytes":[32,110,97,116,117,114,97,108],"logprob":-9.977592468261719},{"id":12106,"token":" colour","bytes":[32,99,111,108,111,117,114],"logprob":-10.244378089904785},{"id":303,"token":" in","bytes":[32,105,110],"logprob":-10.457124710083008},{"id":4047,"token":" common","bytes":[32,99,111,109,109,111,110],"logprob":-10.47756576538086},{"id":11,"token":",","bytes":[44],"logprob":-10.517362594604492},{"id":1576,"token":" object","bytes":[32,111,98,106,101,99,116],"logprob":-10.613184928894043},{"id":37747,"token":" hue","bytes":[32,104,117,101],"logprob":-10.78063678741455},{"id":1880,"token":" art","bytes":[32,97,114,116],"logprob":-10.880351066589355}]} +{"id":369,"token":" is","bytes":[32,105,115],"logprob":-0.2885439991950989,"top_logprobs":[{"id":369,"token":" is","bytes":[32,105,115],"logprob":-0.2885439991950989},{"id":303,"token":" in","bytes":[32,105,110],"logprob":-1.6184141635894775},{"id":383,"token":" on","bytes":[32,111,110],"logprob":-4.787460803985596},{"id":364,"token":" for","bytes":[32,102,111,114],"logprob":-4.980970859527588},{"id":318,"token":" (","bytes":[32,40],"logprob":-5.045620441436768},{"id":628,"token":" can","bytes":[32,99,97,110],"logprob":-5.076292514801025},{"id":11,"token":",","bytes":[44],"logprob":-5.334011554718018},{"id":1429,"token":" used","bytes":[32,117,115,101,100],"logprob":-6.118626117706299},{"id":314,"token":" of","bytes":[32,111,102],"logprob":-6.244320392608643},{"id":18675,"token":" refers","bytes":[32,114,101,102,101,114,115],"logprob":-6.406418323516846},{"id":494,"token":" from","bytes":[32,102,114,111,109],"logprob":-6.544653415679932},{"id":4082,"token":" among","bytes":[32,97,109,111,110,103],"logprob":-7.073530673980713},{"id":5802,"token":" usually","bytes":[32,117,115,117,97,108,108,121],"logprob":-7.227379322052002},{"id":8524,"token":" generally","bytes":[32,103,101,110,101,114,97,108,108,121],"logprob":-7.394110202789307},{"id":421,"token":" that","bytes":[32,116,104,97,116],"logprob":-7.399153709411621},{"id":16661,"token":" consists","bytes":[32,99,111,110,115,105,115,116,115],"logprob":-7.4406609535217285},{"id":13402,"token":" depends","bytes":[32,100,101,112,101,110,100,115],"logprob":-7.5282063484191895},{"id":10813,"token":" typically","bytes":[32,116,121,112,105,99,97,108,108,121],"logprob":-7.541653156280518},{"id":803,"token":" name","bytes":[32,110,97,109,101],"logprob":-7.551076889038086},{"id":513,"token":" are","bytes":[32,97,114,101],"logprob":-7.634177207946777}]} +{"id":264,"token":" a","bytes":[32,97],"logprob":-1.0750583410263062,"top_logprobs":[{"id":264,"token":" a","bytes":[32,97],"logprob":-1.0750583410263062},{"id":799,"token":" one","bytes":[32,111,110,101],"logprob":-1.2597602605819702},{"id":279,"token":" the","bytes":[32,116,104,101],"logprob":-2.750683307647705},{"id":866,"token":" any","bytes":[32,97,110,121],"logprob":-2.751688241958618},{"id":4364,"token":" defined","bytes":[32,100,101,102,105,110,101,100],"logprob":-2.805025339126587},{"id":2972,"token":" **","bytes":[32,42,42],"logprob":-3.0838208198547363},{"id":8524,"token":" generally","bytes":[32,103,101,110,101,114,97,108,108,121],"logprob":-3.846597909927368},{"id":10813,"token":" typically","bytes":[32,116,121,112,105,99,97,108,108,121],"logprob":-4.157089710235596},{"id":5802,"token":" usually","bytes":[32,117,115,117,97,108,108,121],"logprob":-4.448530673980713},{"id":449,"token":" an","bytes":[32,97,110],"logprob":-4.686405658721924},{"id":3428,"token":" often","bytes":[32,111,102,116,101,110],"logprob":-4.8459553718566895},{"id":10521,"token":" determined","bytes":[32,100,101,116,101,114,109,105,110,101,100],"logprob":-4.877230167388916},{"id":6725,"token":" named","bytes":[32,110,97,109,101,100],"logprob":-4.9115891456604},{"id":2590,"token":" always","bytes":[32,97,108,119,97,121,115],"logprob":-5.450462818145752},{"id":2512,"token":" called","bytes":[32,99,97,108,108,101,100],"logprob":-5.684446811676025},{"id":34075,"token":" traditionally","bytes":[32,116,114,97,100,105,116,105,111,110,97,108,108,121],"logprob":-5.744984149932861},{"id":524,"token":" not","bytes":[32,110,111,116],"logprob":-5.778433322906494},{"id":1379,"token":" most","bytes":[32,109,111,115,116],"logprob":-5.780783176422119},{"id":25,"token":":","bytes":[58],"logprob":-5.872226238250732},{"id":14807,"token":" represented","bytes":[32,114,101,112,114,101,115,101,110,116,101,100],"logprob":-5.9709248542785645}]} +{"id":1829,"token":" color","bytes":[32,99,111,108,111,114],"logprob":-0.872937798500061,"top_logprobs":[{"id":1829,"token":" color","bytes":[32,99,111,108,111,114],"logprob":-0.872937798500061},{"id":3050,"token":" specific","bytes":[32,115,112,101,99,105,102,105,99],"logprob":-2.6793839931488037},{"id":6557,"token":" basic","bytes":[32,98,97,115,105,99],"logprob":-2.8913114070892334},{"id":5839,"token":" primary","bytes":[32,112,114,105,109,97,114,121],"logprob":-2.928636312484741},{"id":15346,"token":" fundamental","bytes":[32,102,117,110,100,97,109,101,110,116,97,108],"logprob":-2.989755153656006},{"id":73959,"token":" pigment","bytes":[32,112,105,103,109,101,110,116],"logprob":-3.0805070400238037},{"id":6979,"token":" broad","bytes":[32,98,114,111,97,100],"logprob":-3.426826238632202},{"id":2972,"token":" **","bytes":[32,42,42],"logprob":-3.6996495723724365},{"id":4496,"token":" term","bytes":[32,116,101,114,109],"logprob":-3.780423879623413},{"id":716,"token":" set","bytes":[32,115,101,116],"logprob":-4.007289409637451},{"id":803,"token":" name","bytes":[32,110,97,109,101],"logprob":-4.208184719085693},{"id":37747,"token":" hue","bytes":[32,104,117,101],"logprob":-4.316415309906006},{"id":913,"token":" type","bytes":[32,116,121,112,101],"logprob":-4.601918697357178},{"id":10213,"token":" pure","bytes":[32,112,117,114,101],"logprob":-4.6612067222595215},{"id":26609,"token":" shade","bytes":[32,115,104,97,100,101],"logprob":-4.698535442352295},{"id":5129,"token":" standard","bytes":[32,115,116,97,110,100,97,114,100],"logprob":-4.758004665374756},{"id":5407,"token":" category","bytes":[32,99,97,116,101,103,111,114,121],"logprob":-4.791621685028076},{"id":5629,"token":" natural","bytes":[32,110,97,116,117,114,97,108],"logprob":-5.102899074554443},{"id":1809,"token":" group","bytes":[32,103,114,111,117,112],"logprob":-5.113021373748779},{"id":3074,"token":" single","bytes":[32,115,105,110,103,108,101],"logprob":-5.1606669425964355}]} +{"id":421,"token":" that","bytes":[32,116,104,97,116],"logprob":-0.824985921382904,"top_logprobs":[{"id":421,"token":" that","bytes":[32,116,104,97,116],"logprob":-0.824985921382904},{"id":1429,"token":" used","bytes":[32,117,115,101,100],"logprob":-2.105898380279541},{"id":383,"token":" on","bytes":[32,111,110],"logprob":-2.839155673980713},{"id":303,"token":" in","bytes":[32,105,110],"logprob":-3.0154099464416504},{"id":494,"token":" from","bytes":[32,102,114,111,109],"logprob":-3.5576729774475098},{"id":11543,"token":" chosen","bytes":[32,99,104,111,115,101,110],"logprob":-3.6659398078918457},{"id":314,"token":" of","bytes":[32,111,102],"logprob":-3.7362847328186035},{"id":1669,"token":" found","bytes":[32,102,111,117,110,100],"logprob":-4.40273904800415},{"id":3568,"token":" added","bytes":[32,97,100,100,101,100],"logprob":-4.596193790435791},{"id":6725,"token":" named","bytes":[32,110,97,109,101,100],"logprob":-4.643208026885986},{"id":803,"token":" name","bytes":[32,110,97,109,101],"logprob":-4.664583683013916},{"id":13717,"token":" formed","bytes":[32,102,111,114,109,101,100],"logprob":-4.792622089385986},{"id":440,"token":" with","bytes":[32,119,105,116,104],"logprob":-4.794378757476807},{"id":14807,"token":" represented","bytes":[32,114,101,112,114,101,115,101,110,116,101,100],"logprob":-4.843843936920166},{"id":2945,"token":" present","bytes":[32,112,114,101,115,101,110,116],"logprob":-4.993719577789307},{"id":6483,"token":" whose","bytes":[32,119,104,111,115,101],"logprob":-5.033426761627197},{"id":4364,"token":" defined","bytes":[32,100,101,102,105,110,101,100],"logprob":-5.118124485015869},{"id":9238,"token":" mixed","bytes":[32,109,105,120,101,100],"logprob":-5.134734630584717},{"id":12405,"token":" picked","bytes":[32,112,105,99,107,101,100],"logprob":-5.356703281402588},{"id":6505,"token":" pair","bytes":[32,112,97,105,114],"logprob":-5.397775173187256}]} +{"id":369,"token":" is","bytes":[32,105,115],"logprob":-0.9694944024085999,"top_logprobs":[{"id":369,"token":" is","bytes":[32,105,115],"logprob":-0.9694944024085999},{"id":4021,"token":" cannot","bytes":[32,99,97,110,110,111,116],"logprob":-2.0581955909729004},{"id":682,"token":" has","bytes":[32,104,97,115],"logprob":-2.9001717567443848},{"id":11,"token":",","bytes":[44],"logprob":-3.055054187774658},{"id":31341,"token":" combines","bytes":[32,99,111,109,98,105,110,101,115],"logprob":-3.298262119293213},{"id":628,"token":" can","bytes":[32,99,97,110],"logprob":-3.414320468902588},{"id":6513,"token":" exists","bytes":[32,101,120,105,115,116,115],"logprob":-3.473972797393799},{"id":1902,"token":" must","bytes":[32,109,117,115,116],"logprob":-3.7295632362365723},{"id":7701,"token":" appears","bytes":[32,97,112,112,101,97,114,115],"logprob":-3.879713535308838},{"id":12965,"token":" stands","bytes":[32,115,116,97,110,100,115],"logprob":-3.9847683906555176},{"id":7346,"token":" forms","bytes":[32,102,111,114,109,115],"logprob":-4.248701572418213},{"id":264,"token":" a","bytes":[32,97],"logprob":-4.3995842933654785},{"id":3520,"token":" makes","bytes":[32,109,97,107,101,115],"logprob":-4.454859256744385},{"id":13273,"token":" occurs","bytes":[32,111,99,99,117,114,115],"logprob":-4.510050296783447},{"id":3000,"token":" light","bytes":[32,108,105,103,104,116],"logprob":-4.749857425689697},{"id":279,"token":" the","bytes":[32,116,104,101],"logprob":-4.8580498695373535},{"id":14995,"token":" lies","bytes":[32,108,105,101,115],"logprob":-4.863223552703857},{"id":11039,"token":" adds","bytes":[32,97,100,100,115],"logprob":-4.868286609649658},{"id":10551,"token":" represents","bytes":[32,114,101,112,114,101,115,101,110,116,115],"logprob":-4.979218006134033},{"id":1503,"token":" does","bytes":[32,100,111,101,115],"logprob":-5.0906901359558105}]} diff --git a/docs/audits/prm-001/c11-4026-live/p3.oracle_as_apr.json b/docs/audits/prm-001/c11-4026-live/p3.oracle_as_apr.json new file mode 100644 index 0000000000..e02820539d --- /dev/null +++ b/docs/audits/prm-001/c11-4026-live/p3.oracle_as_apr.json @@ -0,0 +1,698 @@ +{ + "choices": [ + { + "logprobs": { + "content": [ + { + "token_id": 32, + "logprob": -1.071533441543579, + "top_logprobs": [ + { + "token_id": 32, + "logprob": -1.071533441543579 + }, + { + "token_id": 332, + "logprob": -1.1200430393218994 + }, + { + "token_id": 760, + "logprob": -1.8397066593170166 + }, + { + "token_id": 3733, + "logprob": -3.579895257949829 + }, + { + "token_id": 15500, + "logprob": -3.630493640899658 + }, + { + "token_id": 3833, + "logprob": -3.885202646255493 + }, + { + "token_id": 7732, + "logprob": -4.172399997711182 + }, + { + "token_id": 657, + "logprob": -5.336533069610596 + }, + { + "token_id": 623, + "logprob": -5.352776050567627 + }, + { + "token_id": 10522, + "logprob": -5.444912433624268 + }, + { + "token_id": 40, + "logprob": -5.463242053985596 + }, + { + "token_id": 2014, + "logprob": -5.874866962432861 + }, + { + "token_id": 27775, + "logprob": -5.935917377471924 + }, + { + "token_id": 13672, + "logprob": -6.026829242706299 + }, + { + "token_id": 14004, + "logprob": -6.098835468292236 + }, + { + "token_id": 33613, + "logprob": -6.1494140625 + }, + { + "token_id": 8114, + "logprob": -6.365744113922119 + }, + { + "token_id": 10025, + "logprob": -6.449535846710205 + }, + { + "token_id": 8181, + "logprob": -6.457029819488525 + }, + { + "token_id": 8347, + "logprob": -6.559885501861572 + } + ] + }, + { + "token_id": 5839, + "logprob": -0.08975578099489212, + "top_logprobs": [ + { + "token_id": 5839, + "logprob": -0.08975578099489212 + }, + { + "token_id": 2972, + "logprob": -2.9284169673919678 + }, + { + "token_id": 4047, + "logprob": -4.1467719078063965 + }, + { + "token_id": 1546, + "logprob": -5.481477737426758 + }, + { + "token_id": 13177, + "logprob": -6.161518096923828 + }, + { + "token_id": 11088, + "logprob": -6.93243932723999 + }, + { + "token_id": 5129, + "logprob": -7.340293884277344 + }, + { + "token_id": 1575, + "logprob": -7.500030517578125 + }, + { + "token_id": 19565, + "logprob": -7.520591735839844 + }, + { + "token_id": 804, + "logprob": -7.710727214813232 + }, + { + "token_id": 13852, + "logprob": -7.743495941162109 + }, + { + "token_id": 16138, + "logprob": -7.855956554412842 + }, + { + "token_id": 7792, + "logprob": -8.20311164855957 + }, + { + "token_id": 15346, + "logprob": -8.206281661987305 + }, + { + "token_id": 8343, + "logprob": -8.419899940490723 + }, + { + "token_id": 2342, + "logprob": -8.421042442321777 + }, + { + "token_id": 4145, + "logprob": -8.445547103881836 + }, + { + "token_id": 10213, + "logprob": -8.46473217010498 + }, + { + "token_id": 803, + "logprob": -8.495408058166504 + }, + { + "token_id": 58391, + "logprob": -8.577983856201172 + } + ] + }, + { + "token_id": 1829, + "logprob": -0.00425683194771409, + "top_logprobs": [ + { + "token_id": 1829, + "logprob": -0.00425683194771409 + }, + { + "token_id": 318, + "logprob": -6.518619060516357 + }, + { + "token_id": 2972, + "logprob": -7.577304363250732 + }, + { + "token_id": 3000, + "logprob": -7.603311061859131 + }, + { + "token_id": 2438, + "logprob": -8.677526473999023 + }, + { + "token_id": 46026, + "logprob": -8.686500549316406 + }, + { + "token_id": 3611, + "logprob": -9.06228256225586 + }, + { + "token_id": 5839, + "logprob": -9.334321975708008 + }, + { + "token_id": 73959, + "logprob": -9.409225463867188 + }, + { + "token_id": 5984, + "logprob": -9.688760757446289 + }, + { + "token_id": 423, + "logprob": -9.926101684570313 + }, + { + "token_id": 2919, + "logprob": -9.952198028564453 + }, + { + "token_id": 5629, + "logprob": -9.977592468261719 + }, + { + "token_id": 12106, + "logprob": -10.244378089904785 + }, + { + "token_id": 303, + "logprob": -10.457124710083008 + }, + { + "token_id": 4047, + "logprob": -10.47756576538086 + }, + { + "token_id": 11, + "logprob": -10.517362594604492 + }, + { + "token_id": 1576, + "logprob": -10.613184928894043 + }, + { + "token_id": 37747, + "logprob": -10.78063678741455 + }, + { + "token_id": 1880, + "logprob": -10.880351066589355 + } + ] + }, + { + "token_id": 369, + "logprob": -0.2885439991950989, + "top_logprobs": [ + { + "token_id": 369, + "logprob": -0.2885439991950989 + }, + { + "token_id": 303, + "logprob": -1.6184141635894775 + }, + { + "token_id": 383, + "logprob": -4.787460803985596 + }, + { + "token_id": 364, + "logprob": -4.980970859527588 + }, + { + "token_id": 318, + "logprob": -5.045620441436768 + }, + { + "token_id": 628, + "logprob": -5.076292514801025 + }, + { + "token_id": 11, + "logprob": -5.334011554718018 + }, + { + "token_id": 1429, + "logprob": -6.118626117706299 + }, + { + "token_id": 314, + "logprob": -6.244320392608643 + }, + { + "token_id": 18675, + "logprob": -6.406418323516846 + }, + { + "token_id": 494, + "logprob": -6.544653415679932 + }, + { + "token_id": 4082, + "logprob": -7.073530673980713 + }, + { + "token_id": 5802, + "logprob": -7.227379322052002 + }, + { + "token_id": 8524, + "logprob": -7.394110202789307 + }, + { + "token_id": 421, + "logprob": -7.399153709411621 + }, + { + "token_id": 16661, + "logprob": -7.4406609535217285 + }, + { + "token_id": 13402, + "logprob": -7.5282063484191895 + }, + { + "token_id": 10813, + "logprob": -7.541653156280518 + }, + { + "token_id": 803, + "logprob": -7.551076889038086 + }, + { + "token_id": 513, + "logprob": -7.634177207946777 + } + ] + }, + { + "token_id": 264, + "logprob": -1.0750583410263062, + "top_logprobs": [ + { + "token_id": 264, + "logprob": -1.0750583410263062 + }, + { + "token_id": 799, + "logprob": -1.2597602605819702 + }, + { + "token_id": 279, + "logprob": -2.750683307647705 + }, + { + "token_id": 866, + "logprob": -2.751688241958618 + }, + { + "token_id": 4364, + "logprob": -2.805025339126587 + }, + { + "token_id": 2972, + "logprob": -3.0838208198547363 + }, + { + "token_id": 8524, + "logprob": -3.846597909927368 + }, + { + "token_id": 10813, + "logprob": -4.157089710235596 + }, + { + "token_id": 5802, + "logprob": -4.448530673980713 + }, + { + "token_id": 449, + "logprob": -4.686405658721924 + }, + { + "token_id": 3428, + "logprob": -4.8459553718566895 + }, + { + "token_id": 10521, + "logprob": -4.877230167388916 + }, + { + "token_id": 6725, + "logprob": -4.9115891456604 + }, + { + "token_id": 2590, + "logprob": -5.450462818145752 + }, + { + "token_id": 2512, + "logprob": -5.684446811676025 + }, + { + "token_id": 34075, + "logprob": -5.744984149932861 + }, + { + "token_id": 524, + "logprob": -5.778433322906494 + }, + { + "token_id": 1379, + "logprob": -5.780783176422119 + }, + { + "token_id": 25, + "logprob": -5.872226238250732 + }, + { + "token_id": 14807, + "logprob": -5.9709248542785645 + } + ] + }, + { + "token_id": 1829, + "logprob": -0.872937798500061, + "top_logprobs": [ + { + "token_id": 1829, + "logprob": -0.872937798500061 + }, + { + "token_id": 3050, + "logprob": -2.6793839931488037 + }, + { + "token_id": 6557, + "logprob": -2.8913114070892334 + }, + { + "token_id": 5839, + "logprob": -2.928636312484741 + }, + { + "token_id": 15346, + "logprob": -2.989755153656006 + }, + { + "token_id": 73959, + "logprob": -3.0805070400238037 + }, + { + "token_id": 6979, + "logprob": -3.426826238632202 + }, + { + "token_id": 2972, + "logprob": -3.6996495723724365 + }, + { + "token_id": 4496, + "logprob": -3.780423879623413 + }, + { + "token_id": 716, + "logprob": -4.007289409637451 + }, + { + "token_id": 803, + "logprob": -4.208184719085693 + }, + { + "token_id": 37747, + "logprob": -4.316415309906006 + }, + { + "token_id": 913, + "logprob": -4.601918697357178 + }, + { + "token_id": 10213, + "logprob": -4.6612067222595215 + }, + { + "token_id": 26609, + "logprob": -4.698535442352295 + }, + { + "token_id": 5129, + "logprob": -4.758004665374756 + }, + { + "token_id": 5407, + "logprob": -4.791621685028076 + }, + { + "token_id": 5629, + "logprob": -5.102899074554443 + }, + { + "token_id": 1809, + "logprob": -5.113021373748779 + }, + { + "token_id": 3074, + "logprob": -5.1606669425964355 + } + ] + }, + { + "token_id": 421, + "logprob": -0.824985921382904, + "top_logprobs": [ + { + "token_id": 421, + "logprob": -0.824985921382904 + }, + { + "token_id": 1429, + "logprob": -2.105898380279541 + }, + { + "token_id": 383, + "logprob": -2.839155673980713 + }, + { + "token_id": 303, + "logprob": -3.0154099464416504 + }, + { + "token_id": 494, + "logprob": -3.5576729774475098 + }, + { + "token_id": 11543, + "logprob": -3.6659398078918457 + }, + { + "token_id": 314, + "logprob": -3.7362847328186035 + }, + { + "token_id": 1669, + "logprob": -4.40273904800415 + }, + { + "token_id": 3568, + "logprob": -4.596193790435791 + }, + { + "token_id": 6725, + "logprob": -4.643208026885986 + }, + { + "token_id": 803, + "logprob": -4.664583683013916 + }, + { + "token_id": 13717, + "logprob": -4.792622089385986 + }, + { + "token_id": 440, + "logprob": -4.794378757476807 + }, + { + "token_id": 14807, + "logprob": -4.843843936920166 + }, + { + "token_id": 2945, + "logprob": -4.993719577789307 + }, + { + "token_id": 6483, + "logprob": -5.033426761627197 + }, + { + "token_id": 4364, + "logprob": -5.118124485015869 + }, + { + "token_id": 9238, + "logprob": -5.134734630584717 + }, + { + "token_id": 12405, + "logprob": -5.356703281402588 + }, + { + "token_id": 6505, + "logprob": -5.397775173187256 + } + ] + }, + { + "token_id": 369, + "logprob": -0.9694944024085999, + "top_logprobs": [ + { + "token_id": 369, + "logprob": -0.9694944024085999 + }, + { + "token_id": 4021, + "logprob": -2.0581955909729004 + }, + { + "token_id": 682, + "logprob": -2.9001717567443848 + }, + { + "token_id": 11, + "logprob": -3.055054187774658 + }, + { + "token_id": 31341, + "logprob": -3.298262119293213 + }, + { + "token_id": 628, + "logprob": -3.414320468902588 + }, + { + "token_id": 6513, + "logprob": -3.473972797393799 + }, + { + "token_id": 1902, + "logprob": -3.7295632362365723 + }, + { + "token_id": 7701, + "logprob": -3.879713535308838 + }, + { + "token_id": 12965, + "logprob": -3.9847683906555176 + }, + { + "token_id": 7346, + "logprob": -4.248701572418213 + }, + { + "token_id": 264, + "logprob": -4.3995842933654785 + }, + { + "token_id": 3520, + "logprob": -4.454859256744385 + }, + { + "token_id": 13273, + "logprob": -4.510050296783447 + }, + { + "token_id": 3000, + "logprob": -4.749857425689697 + }, + { + "token_id": 279, + "logprob": -4.8580498695373535 + }, + { + "token_id": 14995, + "logprob": -4.863223552703857 + }, + { + "token_id": 11039, + "logprob": -4.868286609649658 + }, + { + "token_id": 10551, + "logprob": -4.979218006134033 + }, + { + "token_id": 1503, + "logprob": -5.0906901359558105 + } + ] + } + ] + } + } + ] +} diff --git a/docs/audits/prm-001/c11-4026-live/p3.posctl.json b/docs/audits/prm-001/c11-4026-live/p3.posctl.json new file mode 100644 index 0000000000..66b0ff746b --- /dev/null +++ b/docs/audits/prm-001/c11-4026-live/p3.posctl.json @@ -0,0 +1,77 @@ +{ + "steps": [ + { + "step": 0, + "top1_apr": 32, + "top1_oracle": 32, + "overlap": 20, + "max_abs_dlogprob": 0, + "chosen_dlogprob": 0, + "apr_mass": 0.9545903584070158 + }, + { + "step": 1, + "top1_apr": 5839, + "top1_oracle": 5839, + "overlap": 20, + "max_abs_dlogprob": 0, + "chosen_dlogprob": 0, + "apr_mass": 0.9955175913458778 + }, + { + "step": 2, + "top1_apr": 1829, + "top1_oracle": 1829, + "overlap": 20, + "max_abs_dlogprob": 0, + "chosen_dlogprob": 0, + "apr_mass": 0.9992527869051492 + }, + { + "step": 3, + "top1_apr": 369, + "top1_oracle": 369, + "overlap": 20, + "max_abs_dlogprob": 0, + "chosen_dlogprob": 0, + "apr_mass": 0.9929689823254247 + }, + { + "step": 4, + "top1_apr": 264, + "top1_oracle": 264, + "overlap": 20, + "max_abs_dlogprob": 0, + "chosen_dlogprob": 0, + "apr_mass": 0.9621929549106528 + }, + { + "step": 5, + "top1_apr": 1829, + "top1_oracle": 1829, + "overlap": 20, + "max_abs_dlogprob": 0, + "chosen_dlogprob": 0, + "apr_mass": 0.8812884299865295 + }, + { + "step": 6, + "top1_apr": 421, + "top1_oracle": 421, + "overlap": 20, + "max_abs_dlogprob": 0, + "chosen_dlogprob": 0, + "apr_mass": 0.8456490646008205 + }, + { + "step": 7, + "top1_apr": 369, + "top1_oracle": 369, + "overlap": 20, + "max_abs_dlogprob": 0, + "chosen_dlogprob": 0, + "apr_mass": 0.8672382768879738 + } + ], + "pass": true +} diff --git a/docs/audits/prm-001/c11-4026-live/p4.apr.json b/docs/audits/prm-001/c11-4026-live/p4.apr.json new file mode 100644 index 0000000000..a8d9be0da8 --- /dev/null +++ b/docs/audits/prm-001/c11-4026-live/p4.apr.json @@ -0,0 +1 @@ +{"id":"chatcmpl-q4k-1790386257064","object":"chat.completion","created":1790386257,"model":"q","choices":[{"index":0,"message":{"role":"assistant","content":"Hola."},"finish_reason":"stop","logprobs":{"content":[{"token":"Hola","token_id":65717,"logprob":-0.9664516448974609,"bytes":null,"top_logprobs":[{"token":"Hola","token_id":65717,"logprob":-0.9664516448974609,"bytes":null},{"token":"¡","token_id":38511,"logprob":-1.0917015075683594,"bytes":null},{"token":"\"","token_id":1,"logprob":-1.8184356689453125,"bytes":null},{"token":"Sal","token_id":16737,"logprob":-3.9859962463378906,"bytes":null},{"token":"\"H","token_id":44022,"logprob":-4.148551940917969,"bytes":null},{"token":"\"B","token_id":61442,"logprob":-4.456241607666016,"bytes":null},{"token":"¿","token_id":29197,"logprob":-4.6325225830078125,"bytes":null},{"token":"B","token_id":33,"logprob":-4.865528106689453,"bytes":null},{"token":"«","token_id":22961,"logprob":-4.922328948974609,"bytes":null},{"token":"Al","token_id":2033,"logprob":-5.430307388305664,"bytes":null},{"token":"Hello","token_id":9419,"logprob":-5.439037322998047,"bytes":null},{"token":"Bien","token_id":78696,"logprob":-5.654888153076172,"bytes":null},{"token":"H","token_id":39,"logprob":-6.139312744140625,"bytes":null},{"token":"Est","token_id":13386,"logprob":-6.232491493225098,"bytes":null},{"token":"Por","token_id":27186,"logprob":-6.3416948318481445,"bytes":null},{"token":"A","token_id":32,"logprob":-6.68797492980957,"bytes":null},{"token":"C","token_id":34,"logprob":-6.993671417236328,"bytes":null},{"token":"Bonjour","token_id":78768,"logprob":-7.154939651489258,"bytes":null},{"token":"S","token_id":50,"logprob":-7.164162635803223,"bytes":null},{"token":"Ah","token_id":23982,"logprob":-7.189451217651367,"bytes":null}],"logsumexp_full":21.150135040283203},{"token":".","token_id":13,"logprob":-0.14423751831054688,"bytes":null,"top_logprobs":[{"token":".","token_id":13,"logprob":-0.14423751831054688,"bytes":null},{"token":" en","token_id":644,"logprob":-2.6025867462158203,"bytes":null},{"token":" a","token_id":264,"logprob":-4.587734222412109,"bytes":null},{"token":"<|im_end|>","token_id":248046,"logprob":-4.692110061645508,"bytes":null},{"token":",","token_id":11,"logprob":-4.757179260253906,"bytes":null},{"token":" al","token_id":444,"logprob":-5.257894515991211,"bytes":null},{"token":"!","token_id":0,"logprob":-5.508231163024902,"bytes":null},{"token":" (","token_id":318,"logprob":-5.529754638671875,"bytes":null},{"token":" español","token_id":67522,"logprob":-5.659646987915039,"bytes":null},{"token":" por","token_id":4018,"logprob":-5.939964294433594,"bytes":null},{"token":" →","token_id":11069,"logprob":-6.99955940246582,"bytes":null},{"token":" mundo","token_id":27429,"logprob":-7.218631744384766,"bytes":null},{"token":" am","token_id":1044,"logprob":-7.298161506652832,"bytes":null},{"token":" in","token_id":303,"logprob":-7.301063537597656,"bytes":null},{"token":" de","token_id":401,"logprob":-7.392913818359375,"bytes":null},{"token":"翻译成","token_id":144886,"logprob":-7.53591251373291,"bytes":null},{"token":" es","token_id":1477,"logprob":-7.753961563110352,"bytes":null},{"token":" /","token_id":593,"logprob":-8.131599426269531,"bytes":null},{"token":" mund","token_id":39227,"logprob":-8.187868118286133,"bytes":null},{"token":" =","token_id":283,"logprob":-8.20262622833252,"bytes":null}],"logsumexp_full":18.79656410217285}]}}],"usage":{"prompt_tokens":17,"completion_tokens":2,"total_tokens":19}} \ No newline at end of file diff --git a/docs/audits/prm-001/c11-4026-live/p4.cmp.json b/docs/audits/prm-001/c11-4026-live/p4.cmp.json new file mode 100644 index 0000000000..75df6b3c93 --- /dev/null +++ b/docs/audits/prm-001/c11-4026-live/p4.cmp.json @@ -0,0 +1,23 @@ +{ + "steps": [ + { + "step": 0, + "top1_apr": 65717, + "top1_oracle": 65717, + "overlap": 19, + "max_abs_dlogprob": 0.38282012939453125, + "chosen_dlogprob": 0.11561405658721924, + "apr_mass": 0.9716228961360075 + }, + { + "step": 1, + "top1_apr": 13, + "top1_oracle": 13, + "overlap": 20, + "max_abs_dlogprob": 0.5592637062072754, + "chosen_dlogprob": 0.016131892800331116, + "apr_mass": 0.9924601219868365 + } + ], + "pass": false +} diff --git a/docs/audits/prm-001/c11-4026-live/p4.ids.json b/docs/audits/prm-001/c11-4026-live/p4.ids.json new file mode 100644 index 0000000000..85f97facc3 --- /dev/null +++ b/docs/audits/prm-001/c11-4026-live/p4.ids.json @@ -0,0 +1 @@ +{"tokens":[248045,846,198,26583,23066,1083,14712,13,248046,198,248045,74455,198,248068,271,248069,271]} \ No newline at end of file diff --git a/docs/audits/prm-001/c11-4026-live/p4.oracle.jsonl b/docs/audits/prm-001/c11-4026-live/p4.oracle.jsonl new file mode 100644 index 0000000000..aaadc1c31e --- /dev/null +++ b/docs/audits/prm-001/c11-4026-live/p4.oracle.jsonl @@ -0,0 +1,2 @@ +{"id":65717,"token":"Hola","bytes":[72,111,108,97],"logprob":-0.8508375883102417,"top_logprobs":[{"id":65717,"token":"Hola","bytes":[72,111,108,97],"logprob":-0.8508375883102417},{"id":38511,"token":"¡","bytes":[194,161],"logprob":-1.1468695402145386},{"id":1,"token":"\"","bytes":[34],"logprob":-2.134570837020874},{"id":16737,"token":"Sal","bytes":[83,97,108],"logprob":-3.7512738704681396},{"id":44022,"token":"\"H","bytes":[34,72],"logprob":-4.12403678894043},{"id":29197,"token":"¿","bytes":[194,191],"logprob":-4.391870498657227},{"id":61442,"token":"\"B","bytes":[34,66],"logprob":-4.488897323608398},{"id":33,"token":"B","bytes":[66],"logprob":-4.776777267456055},{"id":22961,"token":"«","bytes":[194,171],"logprob":-4.908634185791016},{"id":2033,"token":"Al","bytes":[65,108],"logprob":-5.134635925292969},{"id":9419,"token":"Hello","bytes":[72,101,108,108,111],"logprob":-5.240152359008789},{"id":78696,"token":"Bien","bytes":[66,105,101,110],"logprob":-5.556014060974121},{"id":39,"token":"H","bytes":[72],"logprob":-5.979272842407227},{"id":13386,"token":"Est","bytes":[69,115,116],"logprob":-6.190364837646484},{"id":27186,"token":"Por","bytes":[80,111,114],"logprob":-6.1987409591674805},{"id":32,"token":"A","bytes":[65],"logprob":-6.305154800415039},{"id":50,"token":"S","bytes":[83],"logprob":-6.858058929443359},{"id":23982,"token":"Ah","bytes":[65,104],"logprob":-6.881217956542969},{"id":78768,"token":"Bonjour","bytes":[66,111,110,106,111,117,114],"logprob":-6.982666969299316},{"id":75403,"token":"Hallo","bytes":[72,97,108,108,111],"logprob":-7.003927230834961}]} +{"id":13,"token":".","bytes":[46],"logprob":-0.160369411110878,"top_logprobs":[{"id":13,"token":".","bytes":[46],"logprob":-0.160369411110878},{"id":644,"token":" en","bytes":[32,101,110],"logprob":-2.3747916221618652},{"id":264,"token":" a","bytes":[32,97],"logprob":-4.507951259613037},{"id":11,"token":",","bytes":[44],"logprob":-4.719797611236572},{"id":444,"token":" al","bytes":[32,97,108],"logprob":-5.093003749847412},{"id":248046,"token":"","bytes":[],"logprob":-5.251373767852783},{"id":318,"token":" (","bytes":[32,40],"logprob":-5.641900539398193},{"id":0,"token":"!","bytes":[33],"logprob":-5.70700216293335},{"id":4018,"token":" por","bytes":[32,112,111,114],"logprob":-5.863807201385498},{"id":67522,"token":" español","bytes":[32,101,115,112,97,195,177,111,108],"logprob":-5.915187358856201},{"id":303,"token":" in","bytes":[32,105,110],"logprob":-7.170114040374756},{"id":11069,"token":" →","bytes":[32,226,134,146],"logprob":-7.2482733726501465},{"id":27429,"token":" mundo","bytes":[32,109,117,110,100,111],"logprob":-7.285539150238037},{"id":1044,"token":" am","bytes":[32,97,109],"logprob":-7.403501033782959},{"id":144886,"token":"翻译成","bytes":[231,191,187,232,175,145,230,136,144],"logprob":-7.426675319671631},{"id":401,"token":" de","bytes":[32,100,101],"logprob":-7.5972580909729},{"id":1477,"token":" es","bytes":[32,101,115],"logprob":-8.026260375976563},{"id":283,"token":" =","bytes":[32,61],"logprob":-8.187079429626465},{"id":593,"token":" /","bytes":[32,47],"logprob":-8.195954322814941},{"id":39227,"token":" mund","bytes":[32,109,117,110,100],"logprob":-8.356134414672852}]} diff --git a/docs/audits/prm-001/c11-4026-live/p5.apr.json b/docs/audits/prm-001/c11-4026-live/p5.apr.json new file mode 100644 index 0000000000..b655fb4f1e --- /dev/null +++ b/docs/audits/prm-001/c11-4026-live/p5.apr.json @@ -0,0 +1 @@ +{"id":"chatcmpl-q4k-1790386258524","object":"chat.completion","created":1790386259,"model":"q","choices":[{"index":0,"message":{"role":"assistant","content":"$12 \\times 12"},"finish_reason":"length","logprobs":{"content":[{"token":"$","token_id":3,"logprob":-0.5491275787353516,"bytes":null,"top_logprobs":[{"token":"$","token_id":3,"logprob":-0.5491275787353516,"bytes":null},{"token":"To","token_id":1206,"logprob":-1.5241565704345703,"bytes":null},{"token":"1","token_id":16,"logprob":-1.9766311645507812,"bytes":null},{"token":"The","token_id":760,"logprob":-3.3944931030273438,"bytes":null},{"token":"$$","token_id":13682,"logprob":-4.671483993530273,"bytes":null},{"token":"\\(","token_id":42821,"logprob":-5.386211395263672,"bytes":null},{"token":"**","token_id":332,"logprob":-6.070335388183594,"bytes":null},{"token":"Let","token_id":9764,"logprob":-6.646240234375,"bytes":null},{"token":"Here","token_id":8160,"logprob":-6.853309631347656,"bytes":null},{"token":"It","token_id":2064,"logprob":-6.916126251220703,"bytes":null},{"token":"When","token_id":4350,"logprob":-6.923789978027344,"bytes":null},{"token":"There","token_id":3733,"logprob":-7.124187469482422,"bytes":null},{"token":"First","token_id":5170,"logprob":-7.253654479980469,"bytes":null},{"token":"Based","token_id":27775,"logprob":-7.264522552490234,"bytes":null},{"token":"Multip","token_id":55334,"logprob":-7.26544189453125,"bytes":null},{"token":"We","token_id":1596,"logprob":-7.347015380859375,"bytes":null},{"token":"Calcul","token_id":55964,"logprob":-7.601968765258789,"bytes":null},{"token":"Calc","token_id":45593,"logprob":-7.81281852722168,"bytes":null},{"token":"Tw","token_id":22106,"logprob":-7.859615325927734,"bytes":null},{"token":"No","token_id":2665,"logprob":-7.887687683105469,"bytes":null}],"logsumexp_full":24.971572875976562},{"token":"1","token_id":16,"logprob":-0.000148773193359375,"bytes":null,"top_logprobs":[{"token":"1","token_id":16,"logprob":-0.000148773193359375,"bytes":null},{"token":"${","token_id":2288,"logprob":-10.043207168579102,"bytes":null},{"token":"9","token_id":24,"logprob":-10.527238845825195,"bytes":null},{"token":"2","token_id":17,"logprob":-11.06820297241211,"bytes":null},{"token":"$(","token_id":8263,"logprob":-11.379283905029297,"bytes":null},{"token":"3","token_id":18,"logprob":-11.391221046447754,"bytes":null},{"token":"4","token_id":19,"logprob":-12.154810905456543,"bytes":null},{"token":"5","token_id":20,"logprob":-12.962358474731445,"bytes":null},{"token":"8","token_id":23,"logprob":-13.031352996826172,"bytes":null},{"token":"\n","token_id":198,"logprob":-13.055243492126465,"bytes":null},{"token":" ","token_id":220,"logprob":-13.248899459838867,"bytes":null},{"token":"$","token_id":3,"logprob":-13.264203071594238,"bytes":null},{"token":"{{","token_id":2884,"logprob":-13.33021354675293,"bytes":null},{"token":"_{","token_id":14717,"logprob":-13.42330265045166,"bytes":null},{"token":"6","token_id":21,"logprob":-13.622716903686523,"bytes":null},{"token":"$_","token_id":6200,"logprob":-13.694522857666016,"bytes":null},{"token":"^","token_id":61,"logprob":-13.750343322753906,"bytes":null},{"token":"0","token_id":15,"logprob":-13.807716369628906,"bytes":null},{"token":"\n\n","token_id":271,"logprob":-13.851606369018555,"bytes":null},{"token":"N","token_id":45,"logprob":-13.864416122436523,"bytes":null}],"logsumexp_full":27.335681915283203},{"token":"2","token_id":17,"logprob":-0.0002613067626953125,"bytes":null,"top_logprobs":[{"token":"2","token_id":17,"logprob":-0.0002613067626953125,"bytes":null},{"token":"1","token_id":16,"logprob":-9.317550659179688,"bytes":null},{"token":" \\","token_id":1088,"logprob":-9.610769271850586,"bytes":null},{"token":"4","token_id":19,"logprob":-10.748706817626953,"bytes":null},{"token":"6","token_id":21,"logprob":-10.78221321105957,"bytes":null},{"token":"3","token_id":18,"logprob":-11.168764114379883,"bytes":null},{"token":"0","token_id":15,"logprob":-11.215719223022461,"bytes":null},{"token":"9","token_id":24,"logprob":-11.402318954467773,"bytes":null},{"token":"5","token_id":20,"logprob":-11.525701522827148,"bytes":null},{"token":"8","token_id":23,"logprob":-12.088167190551758,"bytes":null},{"token":"7","token_id":22,"logprob":-12.494665145874023,"bytes":null},{"token":"$","token_id":3,"logprob":-13.073858261108398,"bytes":null},{"token":"^","token_id":61,"logprob":-14.239195823669434,"bytes":null},{"token":".","token_id":13,"logprob":-14.736146926879883,"bytes":null},{"token":" +","token_id":478,"logprob":-15.634318351745605,"bytes":null},{"token":",","token_id":11,"logprob":-15.813819885253906,"bytes":null},{"token":" ","token_id":220,"logprob":-16.0018310546875,"bytes":null},{"token":"^{","token_id":46227,"logprob":-16.138593673706055,"bytes":null},{"token":"\\","token_id":59,"logprob":-16.3203182220459,"bytes":null},{"token":"{","token_id":90,"logprob":-16.6174373626709,"bytes":null}],"logsumexp_full":29.863603591918945},{"token":" \\","token_id":1088,"logprob":-0.05092048645019531,"bytes":null,"top_logprobs":[{"token":" \\","token_id":1088,"logprob":-0.05092048645019531,"bytes":null},{"token":"$","token_id":3,"logprob":-3.042783737182617,"bytes":null},{"token":"^","token_id":61,"logprob":-6.575777053833008,"bytes":null},{"token":"\\","token_id":59,"logprob":-8.309793472290039,"bytes":null},{"token":"^{","token_id":46227,"logprob":-9.647354125976562,"bytes":null},{"token":"0","token_id":15,"logprob":-10.242731094360352,"bytes":null},{"token":"1","token_id":16,"logprob":-10.371749877929688,"bytes":null},{"token":"\\t","token_id":4795,"logprob":-10.74116325378418,"bytes":null},{"token":" +","token_id":478,"logprob":-10.843915939331055,"bytes":null},{"token":" ","token_id":220,"logprob":-10.86497688293457,"bytes":null},{"token":"8","token_id":23,"logprob":-11.092355728149414,"bytes":null},{"token":"6","token_id":21,"logprob":-11.581223487854004,"bytes":null},{"token":" times","token_id":2942,"logprob":-11.687987327575684,"bytes":null},{"token":"!","token_id":0,"logprob":-12.170156478881836,"bytes":null},{"token":"5","token_id":20,"logprob":-12.186765670776367,"bytes":null},{"token":"9","token_id":24,"logprob":-12.29745101928711,"bytes":null},{"token":" $\\","token_id":56014,"logprob":-12.376274108886719,"bytes":null},{"token":"4","token_id":19,"logprob":-12.411216735839844,"bytes":null},{"token":"7","token_id":22,"logprob":-12.507391929626465,"bytes":null},{"token":"乘以","token_id":133270,"logprob":-12.527080535888672,"bytes":null}],"logsumexp_full":27.41411018371582},{"token":"times","token_id":14695,"logprob":-0.0063724517822265625,"bytes":null,"top_logprobs":[{"token":"times","token_id":14695,"logprob":-0.0063724517822265625,"bytes":null},{"token":"text","token_id":1272,"logprob":-5.088979721069336,"bytes":null},{"token":" times","token_id":2942,"logprob":-9.4681396484375,"bytes":null},{"token":"cdot","token_id":49142,"logprob":-9.666810989379883,"bytes":null},{"token":"div","token_id":596,"logprob":-10.517511367797852,"bytes":null},{"token":" \\","token_id":1088,"logprob":-12.35597038269043,"bytes":null},{"token":"imes","token_id":1672,"logprob":-12.983573913574219,"bytes":null},{"token":"乘以","token_id":133270,"logprob":-13.52280044555664,"bytes":null},{"token":"time","token_id":1619,"logprob":-13.5841064453125,"bytes":null},{"token":",","token_id":11,"logprob":-13.703010559082031,"bytes":null},{"token":"boxed","token_id":76367,"logprob":-14.158330917358398,"bytes":null},{"token":"mathrm","token_id":88396,"logprob":-14.316943168640137,"bytes":null},{"token":"Times","token_id":18319,"logprob":-14.372506141662598,"bytes":null},{"token":"p","token_id":79,"logprob":-14.58421516418457,"bytes":null},{"token":"quad","token_id":30571,"logprob":-14.69854736328125,"bytes":null},{"token":" ","token_id":220,"logprob":-14.843910217285156,"bytes":null},{"token":" Times","token_id":8261,"logprob":-14.980949401855469,"bytes":null},{"token":"_times","token_id":21657,"logprob":-15.006775856018066,"bytes":null},{"token":"rightarrow","token_id":18899,"logprob":-15.178359031677246,"bytes":null},{"token":"math","token_id":10065,"logprob":-15.205141067504883,"bytes":null}],"logsumexp_full":28.91029167175293},{"token":" ","token_id":220,"logprob":0.0,"bytes":null,"top_logprobs":[{"token":" ","token_id":220,"logprob":0.0,"bytes":null},{"token":" \\","token_id":1088,"logprob":-14.445615768432617,"bytes":null},{"token":"$","token_id":3,"logprob":-15.561071395874023,"bytes":null},{"token":" (","token_id":318,"logprob":-17.661422729492188,"bytes":null},{"token":" \\$","token_id":31790,"logprob":-18.219274520874023,"bytes":null},{"token":" (-","token_id":9987,"logprob":-18.238182067871094,"bytes":null},{"token":"1","token_id":16,"logprob":-18.4290828704834,"bytes":null},{"token":"_{","token_id":14717,"logprob":-18.49753189086914,"bytes":null},{"token":" -","token_id":471,"logprob":-18.62664031982422,"bytes":null},{"token":" =","token_id":283,"logprob":-18.953433990478516,"bytes":null},{"token":" $","token_id":393,"logprob":-19.156694412231445,"bytes":null},{"token":" ?","token_id":907,"logprob":-19.58550262451172,"bytes":null},{"token":" ","token_id":3966,"logprob":-19.958051681518555,"bytes":null},{"token":" {}","token_id":4535,"logprob":-19.994173049926758,"bytes":null},{"token":" a","token_id":264,"logprob":-20.146141052246094,"bytes":null},{"token":"{","token_id":90,"logprob":-20.321273803710938,"bytes":null},{"token":" [","token_id":498,"logprob":-20.414505004882812,"bytes":null},{"token":" +","token_id":478,"logprob":-20.540266036987305,"bytes":null},{"token":" x","token_id":830,"logprob":-20.58926010131836,"bytes":null},{"token":" !","token_id":729,"logprob":-20.607624053955078,"bytes":null}],"logsumexp_full":31.562990188598633},{"token":"1","token_id":16,"logprob":-3.814697265625e-6,"bytes":null,"top_logprobs":[{"token":"1","token_id":16,"logprob":-3.814697265625e-6,"bytes":null},{"token":"2","token_id":17,"logprob":-13.294303894042969,"bytes":null},{"token":"9","token_id":24,"logprob":-14.673873901367188,"bytes":null},{"token":"4","token_id":19,"logprob":-14.688217163085938,"bytes":null},{"token":"3","token_id":18,"logprob":-14.795621871948242,"bytes":null},{"token":"8","token_id":23,"logprob":-15.431455612182617,"bytes":null},{"token":"5","token_id":20,"logprob":-15.46092414855957,"bytes":null},{"token":"6","token_id":21,"logprob":-15.713462829589844,"bytes":null},{"token":"7","token_id":22,"logprob":-16.311172485351562,"bytes":null},{"token":"0","token_id":15,"logprob":-16.941314697265625,"bytes":null},{"token":" =","token_id":283,"logprob":-19.596067428588867,"bytes":null},{"token":" ","token_id":220,"logprob":-19.665180206298828,"bytes":null},{"token":"","token_id":248068,"logprob":-21.034788131713867,"bytes":null},{"token":"\b","token_id":196,"logprob":-21.865821838378906,"bytes":null},{"token":"\f","token_id":200,"logprob":-22.407085418701172,"bytes":null},{"token":"十二","token_id":98175,"logprob":-22.827272415161133,"bytes":null},{"token":"_{","token_id":14717,"logprob":-23.038236618041992,"bytes":null},{"token":"ict","token_id":823,"logprob":-23.094715118408203,"bytes":null},{"token":"$.","token_id":12576,"logprob":-23.3775634765625,"bytes":null},{"token":"$","token_id":3,"logprob":-23.49100112915039,"bytes":null}],"logsumexp_full":34.10512161254883},{"token":"2","token_id":17,"logprob":-0.000095367431640625,"bytes":null,"top_logprobs":[{"token":"2","token_id":17,"logprob":-0.000095367431640625,"bytes":null},{"token":"0","token_id":15,"logprob":-9.66815185546875,"bytes":null},{"token":"3","token_id":18,"logprob":-11.35238265991211,"bytes":null},{"token":"1","token_id":16,"logprob":-11.754997253417969,"bytes":null},{"token":"4","token_id":19,"logprob":-12.178682327270508,"bytes":null},{"token":"8","token_id":23,"logprob":-12.761768341064453,"bytes":null},{"token":"5","token_id":20,"logprob":-12.850650787353516,"bytes":null},{"token":"$","token_id":3,"logprob":-13.55267333984375,"bytes":null},{"token":"6","token_id":21,"logprob":-13.817028045654297,"bytes":null},{"token":"9","token_id":24,"logprob":-14.422637939453125,"bytes":null},{"token":" =","token_id":283,"logprob":-14.898929595947266,"bytes":null},{"token":"7","token_id":22,"logprob":-14.941789627075195,"bytes":null},{"token":" \\","token_id":1088,"logprob":-18.37343406677246,"bytes":null},{"token":" +","token_id":478,"logprob":-18.90819549560547,"bytes":null},{"token":"$,","token_id":53031,"logprob":-19.303564071655273,"bytes":null},{"token":" ","token_id":220,"logprob":-19.528690338134766,"bytes":null},{"token":"^","token_id":61,"logprob":-19.64374542236328,"bytes":null},{"token":"\\","token_id":59,"logprob":-19.71939468383789,"bytes":null},{"token":"{","token_id":90,"logprob":-20.131908416748047,"bytes":null},{"token":"$.","token_id":12576,"logprob":-20.349014282226562,"bytes":null}],"logsumexp_full":32.45522689819336}]}}],"usage":{"prompt_tokens":22,"completion_tokens":8,"total_tokens":30}} \ No newline at end of file diff --git a/docs/audits/prm-001/c11-4026-live/p5.cmp.json b/docs/audits/prm-001/c11-4026-live/p5.cmp.json new file mode 100644 index 0000000000..1151779572 --- /dev/null +++ b/docs/audits/prm-001/c11-4026-live/p5.cmp.json @@ -0,0 +1,77 @@ +{ + "steps": [ + { + "step": 0, + "top1_apr": 3, + "top1_oracle": 3, + "overlap": 19, + "max_abs_dlogprob": 0.37291622161865234, + "chosen_dlogprob": 0.0032587647438049316, + "apr_mass": 0.9931514917707769 + }, + { + "step": 1, + "top1_apr": 16, + "top1_oracle": 16, + "overlap": 19, + "max_abs_dlogprob": 1.1312856674194336, + "chosen_dlogprob": 6.10313072684221e-05, + "apr_mass": 0.9999847144564954 + }, + { + "step": 2, + "top1_apr": 17, + "top1_oracle": 17, + "overlap": 20, + "max_abs_dlogprob": 0.8994979858398438, + "chosen_dlogprob": 0.00011284198262728751, + "apr_mass": 0.9999995896980145 + }, + { + "step": 3, + "top1_apr": 1088, + "top1_oracle": 1088, + "overlap": 19, + "max_abs_dlogprob": 0.504298210144043, + "chosen_dlogprob": 0.004473257809877396, + "apr_mass": 0.9999511791930462 + }, + { + "step": 4, + "top1_apr": 14695, + "top1_oracle": 14695, + "overlap": 19, + "max_abs_dlogprob": 0.4799470901489258, + "chosen_dlogprob": 0.002074435818940401, + "apr_mass": 0.9999943652934382 + }, + { + "step": 5, + "top1_apr": 220, + "top1_oracle": 220, + "overlap": 18, + "max_abs_dlogprob": 0.5460624694824219, + "chosen_dlogprob": 5.960466182841628e-07, + "apr_mass": 1.0000008061051011 + }, + { + "step": 6, + "top1_apr": 16, + "top1_oracle": 16, + "overlap": 19, + "max_abs_dlogprob": 0.384429931640625, + "chosen_dlogprob": 5.960412181593711e-07, + "apr_mass": 0.999999761818531 + }, + { + "step": 7, + "top1_apr": 17, + "top1_oracle": 17, + "overlap": 20, + "max_abs_dlogprob": 0.39574527740478516, + "chosen_dlogprob": 3.386379103176296e-05, + "apr_mass": 1.0000016665789389 + } + ], + "pass": false +} diff --git a/docs/audits/prm-001/c11-4026-live/p5.ids.json b/docs/audits/prm-001/c11-4026-live/p5.ids.json new file mode 100644 index 0000000000..3683e7febc --- /dev/null +++ b/docs/audits/prm-001/c11-4026-live/p5.ids.json @@ -0,0 +1 @@ +{"tokens":[248045,846,198,3710,369,220,16,17,2942,220,16,17,30,248046,198,248045,74455,198,248068,271,248069,271]} \ No newline at end of file diff --git a/docs/audits/prm-001/c11-4026-live/p5.oracle.jsonl b/docs/audits/prm-001/c11-4026-live/p5.oracle.jsonl new file mode 100644 index 0000000000..0f1ce990ff --- /dev/null +++ b/docs/audits/prm-001/c11-4026-live/p5.oracle.jsonl @@ -0,0 +1,8 @@ +{"id":3,"token":"$","bytes":[36],"logprob":-0.5523863434791565,"top_logprobs":[{"id":3,"token":"$","bytes":[36],"logprob":-0.5523863434791565},{"id":1206,"token":"To","bytes":[84,111],"logprob":-1.4258947372436523},{"id":16,"token":"1","bytes":[49],"logprob":-2.1272459030151367},{"id":760,"token":"The","bytes":[84,104,101],"logprob":-3.3344335556030273},{"id":13682,"token":"$$","bytes":[36,36],"logprob":-4.886507987976074},{"id":42821,"token":"\\(","bytes":[92,40],"logprob":-5.357701301574707},{"id":332,"token":"**","bytes":[42,42],"logprob":-6.443251609802246},{"id":9764,"token":"Let","bytes":[76,101,116],"logprob":-6.6543989181518555},{"id":8160,"token":"Here","bytes":[72,101,114,101],"logprob":-6.851864814758301},{"id":2064,"token":"It","bytes":[73,116],"logprob":-6.89975643157959},{"id":27775,"token":"Based","bytes":[66,97,115,101,100],"logprob":-7.060057640075684},{"id":4350,"token":"When","bytes":[87,104,101,110],"logprob":-7.125363349914551},{"id":55334,"token":"Multip","bytes":[77,117,108,116,105,112],"logprob":-7.220942497253418},{"id":3733,"token":"There","bytes":[84,104,101,114,101],"logprob":-7.273064613342285},{"id":5170,"token":"First","bytes":[70,105,114,115,116],"logprob":-7.404475212097168},{"id":1596,"token":"We","bytes":[87,101],"logprob":-7.590773582458496},{"id":55964,"token":"Calcul","bytes":[67,97,108,99,117,108],"logprob":-7.656323432922363},{"id":45593,"token":"Calc","bytes":[67,97,108,99],"logprob":-7.7374677658081055},{"id":8549,"token":"Math","bytes":[77,97,116,104],"logprob":-7.83115291595459},{"id":22106,"token":"Tw","bytes":[84,119],"logprob":-8.00405216217041}]} +{"id":16,"token":"1","bytes":[49],"logprob":-0.0000877418860909529,"top_logprobs":[{"id":16,"token":"1","bytes":[49],"logprob":-0.0000877418860909529},{"id":2288,"token":"${","bytes":[36,123],"logprob":-10.40201187133789},{"id":17,"token":"2","bytes":[50],"logprob":-11.30348014831543},{"id":24,"token":"9","bytes":[57],"logprob":-11.382110595703125},{"id":8263,"token":"$(","bytes":[36,40],"logprob":-11.842077255249023},{"id":18,"token":"3","bytes":[51],"logprob":-12.522506713867188},{"id":19,"token":"4","bytes":[52],"logprob":-13.133440971374512},{"id":20,"token":"5","bytes":[53],"logprob":-13.33542537689209},{"id":23,"token":"8","bytes":[56],"logprob":-13.468399047851563},{"id":198,"token":"\n","bytes":[10],"logprob":-13.489533424377441},{"id":220,"token":" ","bytes":[32],"logprob":-13.591241836547852},{"id":14717,"token":"_{","bytes":[95,123],"logprob":-13.673796653747559},{"id":2884,"token":"{{","bytes":[123,123],"logprob":-13.822044372558594},{"id":271,"token":"\n\n","bytes":[10,10],"logprob":-13.882964134216309},{"id":45,"token":"N","bytes":[78],"logprob":-13.90045166015625},{"id":3,"token":"$","bytes":[36],"logprob":-13.90062141418457},{"id":61,"token":"^","bytes":[94],"logprob":-13.919361114501953},{"id":21,"token":"6","bytes":[54],"logprob":-14.097366333007813},{"id":15,"token":"0","bytes":[48],"logprob":-14.141250610351563},{"id":46227,"token":"^{","bytes":[94,123],"logprob":-14.357416152954102}]} +{"id":17,"token":"2","bytes":[50],"logprob":-0.0003741487453226,"top_logprobs":[{"id":17,"token":"2","bytes":[50],"logprob":-0.0003741487453226},{"id":16,"token":"1","bytes":[49],"logprob":-9.079343795776367},{"id":1088,"token":" \\","bytes":[32,92],"logprob":-9.382213592529297},{"id":19,"token":"4","bytes":[52],"logprob":-9.84920883178711},{"id":21,"token":"6","bytes":[54],"logprob":-10.178337097167969},{"id":18,"token":"3","bytes":[51],"logprob":-10.80101203918457},{"id":15,"token":"0","bytes":[48],"logprob":-10.993280410766602},{"id":20,"token":"5","bytes":[53],"logprob":-11.023000717163086},{"id":24,"token":"9","bytes":[57],"logprob":-11.082862854003906},{"id":23,"token":"8","bytes":[56],"logprob":-11.851987838745117},{"id":22,"token":"7","bytes":[55],"logprob":-12.405269622802734},{"id":3,"token":"$","bytes":[36],"logprob":-12.813425064086914},{"id":61,"token":"^","bytes":[94],"logprob":-14.030912399291992},{"id":13,"token":".","bytes":[46],"logprob":-14.801498413085938},{"id":11,"token":",","bytes":[44],"logprob":-15.267434120178223},{"id":478,"token":" +","bytes":[32,43],"logprob":-15.39840030670166},{"id":46227,"token":"^{","bytes":[94,123],"logprob":-15.912498474121094},{"id":220,"token":" ","bytes":[32],"logprob":-15.984084129333496},{"id":59,"token":"\\","bytes":[92],"logprob":-16.04022216796875},{"id":90,"token":"{","bytes":[123],"logprob":-16.29948616027832}]} +{"id":1088,"token":" \\","bytes":[32,92],"logprob":-0.04644722864031792,"top_logprobs":[{"id":1088,"token":" \\","bytes":[32,92],"logprob":-0.04644722864031792},{"id":3,"token":"$","bytes":[36],"logprob":-3.146117687225342},{"id":61,"token":"^","bytes":[94],"logprob":-6.315114498138428},{"id":59,"token":"\\","bytes":[92],"logprob":-8.63491153717041},{"id":46227,"token":"^{","bytes":[94,123],"logprob":-9.311007499694824},{"id":15,"token":"0","bytes":[48],"logprob":-9.738432884216309},{"id":16,"token":"1","bytes":[49],"logprob":-10.264351844787598},{"id":478,"token":" +","bytes":[32,43],"logprob":-10.710094451904297},{"id":23,"token":"8","bytes":[56],"logprob":-10.767436027526855},{"id":4795,"token":"\\t","bytes":[92,116],"logprob":-10.876282691955566},{"id":220,"token":" ","bytes":[32],"logprob":-11.007972717285156},{"id":21,"token":"6","bytes":[54],"logprob":-11.109976768493652},{"id":0,"token":"!","bytes":[33],"logprob":-11.758404731750488},{"id":20,"token":"5","bytes":[53],"logprob":-11.932018280029297},{"id":24,"token":"9","bytes":[57],"logprob":-11.9805269241333},{"id":19,"token":"4","bytes":[52],"logprob":-12.035801887512207},{"id":2942,"token":" times","bytes":[32,116,105,109,101,115],"logprob":-12.043855667114258},{"id":22,"token":"7","bytes":[55],"logprob":-12.077516555786133},{"id":56014,"token":" $\\","bytes":[32,36,92],"logprob":-12.227265357971191},{"id":283,"token":" =","bytes":[32,61],"logprob":-12.413653373718262}]} +{"id":14695,"token":"times","bytes":[116,105,109,101,115],"logprob":-0.004298015963286161,"top_logprobs":[{"id":14695,"token":"times","bytes":[116,105,109,101,115],"logprob":-0.004298015963286161},{"id":1272,"token":"text","bytes":[116,101,120,116],"logprob":-5.494624137878418},{"id":2942,"token":" times","bytes":[32,116,105,109,101,115],"logprob":-9.579747200012207},{"id":49142,"token":"cdot","bytes":[99,100,111,116],"logprob":-9.68251895904541},{"id":596,"token":"div","bytes":[100,105,118],"logprob":-10.378045082092285},{"id":1088,"token":" \\","bytes":[32,92],"logprob":-12.835917472839355},{"id":1672,"token":"imes","bytes":[105,109,101,115],"logprob":-12.885579109191895},{"id":133270,"token":"乘以","bytes":[228,185,152,228,187,165],"logprob":-13.560702323913574},{"id":1619,"token":"time","bytes":[116,105,109,101],"logprob":-13.669720649719238},{"id":11,"token":",","bytes":[44],"logprob":-13.859526634216309},{"id":88396,"token":"mathrm","bytes":[109,97,116,104,114,109],"logprob":-14.122965812683105},{"id":18319,"token":"Times","bytes":[84,105,109,101,115],"logprob":-14.1281099319458},{"id":76367,"token":"boxed","bytes":[98,111,120,101,100],"logprob":-14.13825511932373},{"id":79,"token":"p","bytes":[112],"logprob":-14.345987319946289},{"id":220,"token":" ","bytes":[32],"logprob":-14.742576599121094},{"id":21657,"token":"_times","bytes":[95,116,105,109,101,115],"logprob":-14.787135124206543},{"id":30571,"token":"quad","bytes":[113,117,97,100],"logprob":-14.8257474899292},{"id":8261,"token":" Times","bytes":[32,84,105,109,101,115],"logprob":-14.963685035705566},{"id":10065,"token":"math","bytes":[109,97,116,104],"logprob":-15.077479362487793},{"id":25429,"token":"multi","bytes":[109,117,108,116,105],"logprob":-15.099539756774902}]} +{"id":220,"token":" ","bytes":[32],"logprob":-5.960466182841628E-7,"top_logprobs":[{"id":220,"token":" ","bytes":[32],"logprob":-5.960466182841628E-7},{"id":1088,"token":" \\","bytes":[32,92],"logprob":-14.62151050567627},{"id":3,"token":"$","bytes":[36],"logprob":-16.015121459960938},{"id":318,"token":" (","bytes":[32,40],"logprob":-17.442834854125977},{"id":14717,"token":"_{","bytes":[95,123],"logprob":-17.95146942138672},{"id":31790,"token":" \\$","bytes":[32,92,36],"logprob":-18.22037124633789},{"id":9987,"token":" (-","bytes":[32,40,45],"logprob":-18.247947692871094},{"id":16,"token":"1","bytes":[49],"logprob":-18.370241165161133},{"id":471,"token":" -","bytes":[32,45],"logprob":-18.518943786621094},{"id":283,"token":" =","bytes":[32,61],"logprob":-18.52491569519043},{"id":907,"token":" ?","bytes":[32,63],"logprob":-19.164583206176758},{"id":393,"token":" $","bytes":[32,36],"logprob":-19.46263885498047},{"id":4535,"token":" {}","bytes":[32,123,125],"logprob":-19.613067626953125},{"id":264,"token":" a","bytes":[32,97],"logprob":-19.890357971191406},{"id":90,"token":"{","bytes":[123],"logprob":-20.072574615478516},{"id":498,"token":" [","bytes":[32,91],"logprob":-20.22020149230957},{"id":3966,"token":" ","bytes":[194,160],"logprob":-20.26770782470703},{"id":66887,"token":" Twelve","bytes":[32,84,119,101,108,118,101],"logprob":-20.315868377685547},{"id":19185,"token":" (\\","bytes":[32,40,92],"logprob":-20.360855102539063},{"id":729,"token":" !","bytes":[32,33],"logprob":-20.404022216796875}]} +{"id":16,"token":"1","bytes":[49],"logprob":-0.000003218656047465629,"top_logprobs":[{"id":16,"token":"1","bytes":[49],"logprob":-0.000003218656047465629},{"id":17,"token":"2","bytes":[50],"logprob":-13.450392723083496},{"id":24,"token":"9","bytes":[57],"logprob":-14.66565990447998},{"id":19,"token":"4","bytes":[52],"logprob":-14.7916841506958},{"id":18,"token":"3","bytes":[51],"logprob":-14.802199363708496},{"id":23,"token":"8","bytes":[56],"logprob":-15.422768592834473},{"id":20,"token":"5","bytes":[53],"logprob":-15.556344032287598},{"id":21,"token":"6","bytes":[54],"logprob":-15.6122407913208},{"id":22,"token":"7","bytes":[55],"logprob":-16.220844268798828},{"id":15,"token":"0","bytes":[48],"logprob":-17.049209594726563},{"id":283,"token":" =","bytes":[32,61],"logprob":-19.385173797607422},{"id":220,"token":" ","bytes":[32],"logprob":-19.67953109741211},{"id":248068,"token":"","bytes":[60,116,104,105,110,107,62],"logprob":-21.11679458618164},{"id":196,"token":"\b","bytes":[8],"logprob":-22.25025177001953},{"id":200,"token":"\f","bytes":[12],"logprob":-22.57638931274414},{"id":14717,"token":"_{","bytes":[95,123],"logprob":-22.877866744995117},{"id":98175,"token":"十二","bytes":[229,141,129,228,186,140],"logprob":-22.930397033691406},{"id":823,"token":"ict","bytes":[105,99,116],"logprob":-23.193775177001953},{"id":248044,"token":"","bytes":[],"logprob":-23.33780288696289},{"id":12576,"token":"$.","bytes":[36,46],"logprob":-23.442291259765625}]} +{"id":17,"token":"2","bytes":[50],"logprob":-0.00012923122267238796,"top_logprobs":[{"id":17,"token":"2","bytes":[50],"logprob":-0.00012923122267238796},{"id":15,"token":"0","bytes":[48],"logprob":-9.272406578063965},{"id":18,"token":"3","bytes":[51],"logprob":-11.324191093444824},{"id":16,"token":"1","bytes":[49],"logprob":-11.63136100769043},{"id":19,"token":"4","bytes":[52],"logprob":-12.306384086608887},{"id":23,"token":"8","bytes":[56],"logprob":-12.660528182983398},{"id":20,"token":"5","bytes":[53],"logprob":-12.784611701965332},{"id":3,"token":"$","bytes":[36],"logprob":-13.591208457946777},{"id":21,"token":"6","bytes":[54],"logprob":-13.72807502746582},{"id":24,"token":"9","bytes":[57],"logprob":-14.171477317810059},{"id":283,"token":" =","bytes":[32,61],"logprob":-14.80181884765625},{"id":22,"token":"7","bytes":[55],"logprob":-14.820798873901367},{"id":1088,"token":" \\","bytes":[32,92],"logprob":-18.10513687133789},{"id":478,"token":" +","bytes":[32,43],"logprob":-18.632247924804688},{"id":53031,"token":"$,","bytes":[36,44],"logprob":-19.27086067199707},{"id":61,"token":"^","bytes":[94],"logprob":-19.48488426208496},{"id":220,"token":" ","bytes":[32],"logprob":-19.63347816467285},{"id":59,"token":"\\","bytes":[92],"logprob":-19.834213256835938},{"id":90,"token":"{","bytes":[123],"logprob":-20.108848571777344},{"id":12576,"token":"$.","bytes":[36,46],"logprob":-20.311649322509766}]} diff --git a/docs/audits/prm-001/c11-4026-live/p5.oracle_as_apr.json b/docs/audits/prm-001/c11-4026-live/p5.oracle_as_apr.json new file mode 100644 index 0000000000..e8ecd35962 --- /dev/null +++ b/docs/audits/prm-001/c11-4026-live/p5.oracle_as_apr.json @@ -0,0 +1,698 @@ +{ + "choices": [ + { + "logprobs": { + "content": [ + { + "token_id": 3, + "logprob": -0.5523863434791565, + "top_logprobs": [ + { + "token_id": 3, + "logprob": -0.5523863434791565 + }, + { + "token_id": 1206, + "logprob": -1.4258947372436523 + }, + { + "token_id": 16, + "logprob": -2.1272459030151367 + }, + { + "token_id": 760, + "logprob": -3.3344335556030273 + }, + { + "token_id": 13682, + "logprob": -4.886507987976074 + }, + { + "token_id": 42821, + "logprob": -5.357701301574707 + }, + { + "token_id": 332, + "logprob": -6.443251609802246 + }, + { + "token_id": 9764, + "logprob": -6.6543989181518555 + }, + { + "token_id": 8160, + "logprob": -6.851864814758301 + }, + { + "token_id": 2064, + "logprob": -6.89975643157959 + }, + { + "token_id": 27775, + "logprob": -7.060057640075684 + }, + { + "token_id": 4350, + "logprob": -7.125363349914551 + }, + { + "token_id": 55334, + "logprob": -7.220942497253418 + }, + { + "token_id": 3733, + "logprob": -7.273064613342285 + }, + { + "token_id": 5170, + "logprob": -7.404475212097168 + }, + { + "token_id": 1596, + "logprob": -7.590773582458496 + }, + { + "token_id": 55964, + "logprob": -7.656323432922363 + }, + { + "token_id": 45593, + "logprob": -7.7374677658081055 + }, + { + "token_id": 8549, + "logprob": -7.83115291595459 + }, + { + "token_id": 22106, + "logprob": -8.00405216217041 + } + ] + }, + { + "token_id": 16, + "logprob": -0.0000877418860909529, + "top_logprobs": [ + { + "token_id": 16, + "logprob": -0.0000877418860909529 + }, + { + "token_id": 2288, + "logprob": -10.40201187133789 + }, + { + "token_id": 17, + "logprob": -11.30348014831543 + }, + { + "token_id": 24, + "logprob": -11.382110595703125 + }, + { + "token_id": 8263, + "logprob": -11.842077255249023 + }, + { + "token_id": 18, + "logprob": -12.522506713867188 + }, + { + "token_id": 19, + "logprob": -13.133440971374512 + }, + { + "token_id": 20, + "logprob": -13.33542537689209 + }, + { + "token_id": 23, + "logprob": -13.468399047851563 + }, + { + "token_id": 198, + "logprob": -13.489533424377441 + }, + { + "token_id": 220, + "logprob": -13.591241836547852 + }, + { + "token_id": 14717, + "logprob": -13.673796653747559 + }, + { + "token_id": 2884, + "logprob": -13.822044372558594 + }, + { + "token_id": 271, + "logprob": -13.882964134216309 + }, + { + "token_id": 45, + "logprob": -13.90045166015625 + }, + { + "token_id": 3, + "logprob": -13.90062141418457 + }, + { + "token_id": 61, + "logprob": -13.919361114501953 + }, + { + "token_id": 21, + "logprob": -14.097366333007813 + }, + { + "token_id": 15, + "logprob": -14.141250610351563 + }, + { + "token_id": 46227, + "logprob": -14.357416152954102 + } + ] + }, + { + "token_id": 17, + "logprob": -0.0003741487453226, + "top_logprobs": [ + { + "token_id": 17, + "logprob": -0.0003741487453226 + }, + { + "token_id": 16, + "logprob": -9.079343795776367 + }, + { + "token_id": 1088, + "logprob": -9.382213592529297 + }, + { + "token_id": 19, + "logprob": -9.84920883178711 + }, + { + "token_id": 21, + "logprob": -10.178337097167969 + }, + { + "token_id": 18, + "logprob": -10.80101203918457 + }, + { + "token_id": 15, + "logprob": -10.993280410766602 + }, + { + "token_id": 20, + "logprob": -11.023000717163086 + }, + { + "token_id": 24, + "logprob": -11.082862854003906 + }, + { + "token_id": 23, + "logprob": -11.851987838745117 + }, + { + "token_id": 22, + "logprob": -12.405269622802734 + }, + { + "token_id": 3, + "logprob": -12.813425064086914 + }, + { + "token_id": 61, + "logprob": -14.030912399291992 + }, + { + "token_id": 13, + "logprob": -14.801498413085938 + }, + { + "token_id": 11, + "logprob": -15.267434120178223 + }, + { + "token_id": 478, + "logprob": -15.39840030670166 + }, + { + "token_id": 46227, + "logprob": -15.912498474121094 + }, + { + "token_id": 220, + "logprob": -15.984084129333496 + }, + { + "token_id": 59, + "logprob": -16.04022216796875 + }, + { + "token_id": 90, + "logprob": -16.29948616027832 + } + ] + }, + { + "token_id": 1088, + "logprob": -0.04644722864031792, + "top_logprobs": [ + { + "token_id": 1088, + "logprob": -0.04644722864031792 + }, + { + "token_id": 3, + "logprob": -3.146117687225342 + }, + { + "token_id": 61, + "logprob": -6.315114498138428 + }, + { + "token_id": 59, + "logprob": -8.63491153717041 + }, + { + "token_id": 46227, + "logprob": -9.311007499694824 + }, + { + "token_id": 15, + "logprob": -9.738432884216309 + }, + { + "token_id": 16, + "logprob": -10.264351844787598 + }, + { + "token_id": 478, + "logprob": -10.710094451904297 + }, + { + "token_id": 23, + "logprob": -10.767436027526855 + }, + { + "token_id": 4795, + "logprob": -10.876282691955566 + }, + { + "token_id": 220, + "logprob": -11.007972717285156 + }, + { + "token_id": 21, + "logprob": -11.109976768493652 + }, + { + "token_id": 0, + "logprob": -11.758404731750488 + }, + { + "token_id": 20, + "logprob": -11.932018280029297 + }, + { + "token_id": 24, + "logprob": -11.9805269241333 + }, + { + "token_id": 19, + "logprob": -12.035801887512207 + }, + { + "token_id": 2942, + "logprob": -12.043855667114258 + }, + { + "token_id": 22, + "logprob": -12.077516555786133 + }, + { + "token_id": 56014, + "logprob": -12.227265357971191 + }, + { + "token_id": 283, + "logprob": -12.413653373718262 + } + ] + }, + { + "token_id": 14695, + "logprob": -0.004298015963286161, + "top_logprobs": [ + { + "token_id": 14695, + "logprob": -0.004298015963286161 + }, + { + "token_id": 1272, + "logprob": -5.494624137878418 + }, + { + "token_id": 2942, + "logprob": -9.579747200012207 + }, + { + "token_id": 49142, + "logprob": -9.68251895904541 + }, + { + "token_id": 596, + "logprob": -10.378045082092285 + }, + { + "token_id": 1088, + "logprob": -12.835917472839355 + }, + { + "token_id": 1672, + "logprob": -12.885579109191895 + }, + { + "token_id": 133270, + "logprob": -13.560702323913574 + }, + { + "token_id": 1619, + "logprob": -13.669720649719238 + }, + { + "token_id": 11, + "logprob": -13.859526634216309 + }, + { + "token_id": 88396, + "logprob": -14.122965812683105 + }, + { + "token_id": 18319, + "logprob": -14.1281099319458 + }, + { + "token_id": 76367, + "logprob": -14.13825511932373 + }, + { + "token_id": 79, + "logprob": -14.345987319946289 + }, + { + "token_id": 220, + "logprob": -14.742576599121094 + }, + { + "token_id": 21657, + "logprob": -14.787135124206543 + }, + { + "token_id": 30571, + "logprob": -14.8257474899292 + }, + { + "token_id": 8261, + "logprob": -14.963685035705566 + }, + { + "token_id": 10065, + "logprob": -15.077479362487793 + }, + { + "token_id": 25429, + "logprob": -15.099539756774902 + } + ] + }, + { + "token_id": 220, + "logprob": -5.960466182841628E-7, + "top_logprobs": [ + { + "token_id": 220, + "logprob": -5.960466182841628E-7 + }, + { + "token_id": 1088, + "logprob": -14.62151050567627 + }, + { + "token_id": 3, + "logprob": -16.015121459960938 + }, + { + "token_id": 318, + "logprob": -17.442834854125977 + }, + { + "token_id": 14717, + "logprob": -17.95146942138672 + }, + { + "token_id": 31790, + "logprob": -18.22037124633789 + }, + { + "token_id": 9987, + "logprob": -18.247947692871094 + }, + { + "token_id": 16, + "logprob": -18.370241165161133 + }, + { + "token_id": 471, + "logprob": -18.518943786621094 + }, + { + "token_id": 283, + "logprob": -18.52491569519043 + }, + { + "token_id": 907, + "logprob": -19.164583206176758 + }, + { + "token_id": 393, + "logprob": -19.46263885498047 + }, + { + "token_id": 4535, + "logprob": -19.613067626953125 + }, + { + "token_id": 264, + "logprob": -19.890357971191406 + }, + { + "token_id": 90, + "logprob": -20.072574615478516 + }, + { + "token_id": 498, + "logprob": -20.22020149230957 + }, + { + "token_id": 3966, + "logprob": -20.26770782470703 + }, + { + "token_id": 66887, + "logprob": -20.315868377685547 + }, + { + "token_id": 19185, + "logprob": -20.360855102539063 + }, + { + "token_id": 729, + "logprob": -20.404022216796875 + } + ] + }, + { + "token_id": 16, + "logprob": -0.000003218656047465629, + "top_logprobs": [ + { + "token_id": 16, + "logprob": -0.000003218656047465629 + }, + { + "token_id": 17, + "logprob": -13.450392723083496 + }, + { + "token_id": 24, + "logprob": -14.66565990447998 + }, + { + "token_id": 19, + "logprob": -14.7916841506958 + }, + { + "token_id": 18, + "logprob": -14.802199363708496 + }, + { + "token_id": 23, + "logprob": -15.422768592834473 + }, + { + "token_id": 20, + "logprob": -15.556344032287598 + }, + { + "token_id": 21, + "logprob": -15.6122407913208 + }, + { + "token_id": 22, + "logprob": -16.220844268798828 + }, + { + "token_id": 15, + "logprob": -17.049209594726563 + }, + { + "token_id": 283, + "logprob": -19.385173797607422 + }, + { + "token_id": 220, + "logprob": -19.67953109741211 + }, + { + "token_id": 248068, + "logprob": -21.11679458618164 + }, + { + "token_id": 196, + "logprob": -22.25025177001953 + }, + { + "token_id": 200, + "logprob": -22.57638931274414 + }, + { + "token_id": 14717, + "logprob": -22.877866744995117 + }, + { + "token_id": 98175, + "logprob": -22.930397033691406 + }, + { + "token_id": 823, + "logprob": -23.193775177001953 + }, + { + "token_id": 248044, + "logprob": -23.33780288696289 + }, + { + "token_id": 12576, + "logprob": -23.442291259765625 + } + ] + }, + { + "token_id": 17, + "logprob": -0.00012923122267238796, + "top_logprobs": [ + { + "token_id": 17, + "logprob": -0.00012923122267238796 + }, + { + "token_id": 15, + "logprob": -9.272406578063965 + }, + { + "token_id": 18, + "logprob": -11.324191093444824 + }, + { + "token_id": 16, + "logprob": -11.63136100769043 + }, + { + "token_id": 19, + "logprob": -12.306384086608887 + }, + { + "token_id": 23, + "logprob": -12.660528182983398 + }, + { + "token_id": 20, + "logprob": -12.784611701965332 + }, + { + "token_id": 3, + "logprob": -13.591208457946777 + }, + { + "token_id": 21, + "logprob": -13.72807502746582 + }, + { + "token_id": 24, + "logprob": -14.171477317810059 + }, + { + "token_id": 283, + "logprob": -14.80181884765625 + }, + { + "token_id": 22, + "logprob": -14.820798873901367 + }, + { + "token_id": 1088, + "logprob": -18.10513687133789 + }, + { + "token_id": 478, + "logprob": -18.632247924804688 + }, + { + "token_id": 53031, + "logprob": -19.27086067199707 + }, + { + "token_id": 61, + "logprob": -19.48488426208496 + }, + { + "token_id": 220, + "logprob": -19.63347816467285 + }, + { + "token_id": 59, + "logprob": -19.834213256835938 + }, + { + "token_id": 90, + "logprob": -20.108848571777344 + }, + { + "token_id": 12576, + "logprob": -20.311649322509766 + } + ] + } + ] + } + } + ] +} diff --git a/docs/audits/prm-001/c11-4026-live/p5.posctl.json b/docs/audits/prm-001/c11-4026-live/p5.posctl.json new file mode 100644 index 0000000000..6ad08e7867 --- /dev/null +++ b/docs/audits/prm-001/c11-4026-live/p5.posctl.json @@ -0,0 +1,77 @@ +{ + "steps": [ + { + "step": 0, + "top1_apr": 3, + "top1_oracle": 3, + "overlap": 20, + "max_abs_dlogprob": 0, + "chosen_dlogprob": 0, + "apr_mass": 0.9937147252729654 + }, + { + "step": 1, + "top1_apr": 16, + "top1_oracle": 16, + "overlap": 20, + "max_abs_dlogprob": 0, + "chosen_dlogprob": 0, + "apr_mass": 0.9999927226428112 + }, + { + "step": 2, + "top1_apr": 17, + "top1_oracle": 17, + "overlap": 20, + "max_abs_dlogprob": 0, + "chosen_dlogprob": 0, + "apr_mass": 0.9999997808730922 + }, + { + "step": 3, + "top1_apr": 1088, + "top1_oracle": 1088, + "overlap": 20, + "max_abs_dlogprob": 0, + "chosen_dlogprob": 0, + "apr_mass": 0.9999454921702258 + }, + { + "step": 4, + "top1_apr": 14695, + "top1_oracle": 14695, + "overlap": 20, + "max_abs_dlogprob": 0, + "chosen_dlogprob": 0, + "apr_mass": 0.9999959911507754 + }, + { + "step": 5, + "top1_apr": 220, + "top1_oracle": 220, + "overlap": 20, + "max_abs_dlogprob": 0, + "chosen_dlogprob": 0, + "apr_mass": 1.0000000797889739 + }, + { + "step": 6, + "top1_apr": 16, + "top1_oracle": 16, + "overlap": 20, + "max_abs_dlogprob": 0, + "chosen_dlogprob": 0, + "apr_mass": 1.0000000783905607 + }, + { + "step": 7, + "top1_apr": 17, + "top1_oracle": 17, + "overlap": 20, + "max_abs_dlogprob": 0, + "chosen_dlogprob": 0, + "apr_mass": 1.000000040313696 + } + ], + "pass": true +} diff --git a/docs/audits/prm-001/c11-4026-live/p6.apr.json b/docs/audits/prm-001/c11-4026-live/p6.apr.json new file mode 100644 index 0000000000..d64d65f52d --- /dev/null +++ b/docs/audits/prm-001/c11-4026-live/p6.apr.json @@ -0,0 +1 @@ +{"id":"chatcmpl-q4k-1790386262065","object":"chat.completion","created":1790386263,"model":"q","choices":[{"index":0,"message":{"role":"assistant","content":"**Blue**."},"finish_reason":"stop","logprobs":{"content":[{"token":"**","token_id":332,"logprob":-1.5561370849609375,"bytes":null,"top_logprobs":[{"token":"**","token_id":332,"logprob":-1.5561370849609375,"bytes":null},{"token":"I","token_id":40,"logprob":-1.8854808807373047,"bytes":null},{"token":"blue","token_id":11855,"logprob":-2.042989730834961,"bytes":null},{"token":"Blue","token_id":10025,"logprob":-2.295564651489258,"bytes":null},{"token":"The","token_id":760,"logprob":-2.5000572204589844,"bytes":null},{"token":"Yes","token_id":9175,"logprob":-2.700521469116211,"bytes":null},{"token":"","token_id":248068,"logprob":-4.159662246704102,"bytes":null},{"token":"Say","token_id":44240,"logprob":-4.198873519897461,"bytes":null},{"token":"Hello","token_id":9419,"logprob":-4.215118408203125,"bytes":null},{"token":"Oh","token_id":11569,"logprob":-4.363748550415039,"bytes":null},{"token":"\"","token_id":1,"logprob":-4.415910720825195,"bytes":null},{"token":"It","token_id":2064,"logprob":-4.472719192504883,"bytes":null},{"token":"yes","token_id":9405,"logprob":-4.664194107055664,"bytes":null},{"token":"BLUE","token_id":62158,"logprob":-5.039285659790039,"bytes":null},{"token":"That","token_id":4639,"logprob":-5.060266494750977,"bytes":null},{"token":"*","token_id":9,"logprob":-5.133200645446777,"bytes":null},{"token":"Sure","token_id":38493,"logprob":-5.156074523925781,"bytes":null},{"token":"the","token_id":1719,"logprob":-5.38173770904541,"bytes":null},{"token":"You","token_id":2523,"logprob":-5.3954877853393555,"bytes":null},{"token":"hello","token_id":14556,"logprob":-5.420358657836914,"bytes":null}],"logsumexp_full":20.972265243530273},{"token":"Blue","token_id":10025,"logprob":-0.7315807342529297,"bytes":null,"top_logprobs":[{"token":"Blue","token_id":10025,"logprob":-0.7315807342529297,"bytes":null},{"token":"blue","token_id":11855,"logprob":-0.9811897277832031,"bytes":null},{"token":"BLUE","token_id":62158,"logprob":-2.8375930786132812,"bytes":null},{"token":"B","token_id":33,"logprob":-3.4410839080810547,"bytes":null},{"token":"Yes","token_id":9175,"logprob":-4.670036315917969,"bytes":null},{"token":"**","token_id":332,"logprob":-5.730693817138672,"bytes":null},{"token":"BL","token_id":9110,"logprob":-5.979923248291016,"bytes":null},{"token":"I","token_id":40,"logprob":-6.356101036071777,"bytes":null},{"token":"yes","token_id":9405,"logprob":-6.390974044799805,"bytes":null},{"token":"b","token_id":65,"logprob":-6.516862869262695,"bytes":null},{"token":"bl","token_id":1956,"logprob":-6.771800994873047,"bytes":null},{"token":"O","token_id":46,"logprob":-6.895188331604004,"bytes":null},{"token":"蓝色","token_id":104145,"logprob":-6.984025955200195,"bytes":null},{"token":"A","token_id":32,"logprob":-7.029640197753906,"bytes":null},{"token":"True","token_id":2434,"logprob":-7.059074401855469,"bytes":null},{"token":"The","token_id":760,"logprob":-7.063764572143555,"bytes":null},{"token":"Bl","token_id":4764,"logprob":-7.141380310058594,"bytes":null},{"token":"YES","token_id":13602,"logprob":-7.268370628356934,"bytes":null},{"token":"Hello","token_id":9419,"logprob":-7.377812385559082,"bytes":null},{"token":"E","token_id":36,"logprob":-7.524876594543457,"bytes":null}],"logsumexp_full":22.21578598022461},{"token":"**.","token_id":159034,"logprob":-0.8592700958251953,"bytes":null,"top_logprobs":[{"token":"**.","token_id":159034,"logprob":-0.8592700958251953,"bytes":null},{"token":"**","token_id":332,"logprob":-0.9791107177734375,"bytes":null},{"token":".**","token_id":159029,"logprob":-1.6622486114501953,"bytes":null},{"token":"**!","token_id":201757,"logprob":-5.616539001464844,"bytes":null},{"token":"!**","token_id":171458,"logprob":-5.877538681030273,"bytes":null},{"token":"**?","token_id":193260,"logprob":-6.513189315795898,"bytes":null},{"token":"**,","token_id":93868,"logprob":-6.581104278564453,"bytes":null},{"token":"**。","token_id":221794,"logprob":-7.740056991577148,"bytes":null},{"token":".","token_id":13,"logprob":-8.440500259399414,"bytes":null},{"token":"***","token_id":11862,"logprob":-8.908097267150879,"bytes":null},{"token":"?**","token_id":170286,"logprob":-9.320948600769043,"bytes":null},{"token":"**:","token_id":92217,"logprob":-9.56282901763916,"bytes":null},{"token":"**-","token_id":172064,"logprob":-9.630867958068848,"bytes":null},{"token":"y","token_id":88,"logprob":-9.762134552001953,"bytes":null},{"token":" �","token_id":10838,"logprob":-9.941422462463379,"bytes":null},{"token":"。**","token_id":173609,"logprob":-10.138162612915039,"bytes":null},{"token":"**;","token_id":190632,"logprob":-10.21324348449707,"bytes":null},{"token":"****","token_id":422,"logprob":-10.307234764099121,"bytes":null},{"token":"！**","token_id":200386,"logprob":-10.329475402832031,"bytes":null},{"token":"**—","token_id":197005,"logprob":-10.434432983398438,"bytes":null}],"logsumexp_full":23.17915916442871}]}}],"usage":{"prompt_tokens":17,"completion_tokens":3,"total_tokens":20}} \ No newline at end of file diff --git a/docs/audits/prm-001/c11-4026-live/p6.cmp.json b/docs/audits/prm-001/c11-4026-live/p6.cmp.json new file mode 100644 index 0000000000..013a04d30d --- /dev/null +++ b/docs/audits/prm-001/c11-4026-live/p6.cmp.json @@ -0,0 +1,32 @@ +{ + "steps": [ + { + "step": 0, + "top1_apr": 332, + "top1_oracle": 332, + "overlap": 20, + "max_abs_dlogprob": 0.4042646884918213, + "chosen_dlogprob": 0.1427290439605713, + "apr_mass": 0.871400920435213 + }, + { + "step": 1, + "top1_apr": 10025, + "top1_oracle": 10025, + "overlap": 19, + "max_abs_dlogprob": 0.25193071365356445, + "chosen_dlogprob": 0.04702723026275635, + "apr_mass": 0.9749844810285127 + }, + { + "step": 2, + "top1_apr": 159034, + "top1_oracle": 159034, + "overlap": 19, + "max_abs_dlogprob": 0.3921699523925781, + "chosen_dlogprob": 0.0021566152572631836, + "apr_mass": 0.9994260901551654 + } + ], + "pass": false +} diff --git a/docs/audits/prm-001/c11-4026-live/p6.ids.json b/docs/audits/prm-001/c11-4026-live/p6.ids.json new file mode 100644 index 0000000000..8b444f3450 --- /dev/null +++ b/docs/audits/prm-001/c11-4026-live/p6.ids.json @@ -0,0 +1 @@ +{"tokens":[248045,846,198,44240,279,3299,6105,13,248046,198,248045,74455,198,248068,271,248069,271]} \ No newline at end of file diff --git a/docs/audits/prm-001/c11-4026-live/p6.oracle.jsonl b/docs/audits/prm-001/c11-4026-live/p6.oracle.jsonl new file mode 100644 index 0000000000..47cb310c50 --- /dev/null +++ b/docs/audits/prm-001/c11-4026-live/p6.oracle.jsonl @@ -0,0 +1,3 @@ +{"id":332,"token":"**","bytes":[42,42],"logprob":-1.4134080410003662,"top_logprobs":[{"id":332,"token":"**","bytes":[42,42],"logprob":-1.4134080410003662},{"id":40,"token":"I","bytes":[73],"logprob":-1.8483903408050537},{"id":11855,"token":"blue","bytes":[98,108,117,101],"logprob":-2.2725846767425537},{"id":760,"token":"The","bytes":[84,104,101],"logprob":-2.3502519130706787},{"id":10025,"token":"Blue","bytes":[66,108,117,101],"logprob":-2.5306146144866943},{"id":9175,"token":"Yes","bytes":[89,101,115],"logprob":-2.8620316982269287},{"id":44240,"token":"Say","bytes":[83,97,121],"logprob":-3.7946088314056396},{"id":1,"token":"\"","bytes":[34],"logprob":-4.05697774887085},{"id":9419,"token":"Hello","bytes":[72,101,108,108,111],"logprob":-4.138566493988037},{"id":248068,"token":"","bytes":[60,116,104,105,110,107,62],"logprob":-4.251149654388428},{"id":2064,"token":"It","bytes":[73,116],"logprob":-4.470620632171631},{"id":11569,"token":"Oh","bytes":[79,104],"logprob":-4.511037349700928},{"id":4639,"token":"That","bytes":[84,104,97,116],"logprob":-4.855554103851318},{"id":9405,"token":"yes","bytes":[121,101,115],"logprob":-4.957545757293701},{"id":9,"token":"*","bytes":[42],"logprob":-4.991764545440674},{"id":38493,"token":"Sure","bytes":[83,117,114,101],"logprob":-5.163415431976318},{"id":2523,"token":"You","bytes":[89,111,117],"logprob":-5.226050853729248},{"id":62158,"token":"BLUE","bytes":[66,76,85,69],"logprob":-5.346618175506592},{"id":1719,"token":"the","bytes":[116,104,101],"logprob":-5.3573527336120605},{"id":14556,"token":"hello","bytes":[104,101,108,108,111],"logprob":-5.460299968719482}]} +{"id":10025,"token":"Blue","bytes":[66,108,117,101],"logprob":-0.778607964515686,"top_logprobs":[{"id":10025,"token":"Blue","bytes":[66,108,117,101],"logprob":-0.778607964515686},{"id":11855,"token":"blue","bytes":[98,108,117,101],"logprob":-0.9376217126846313},{"id":62158,"token":"BLUE","bytes":[66,76,85,69],"logprob":-2.8719210624694824},{"id":33,"token":"B","bytes":[66],"logprob":-3.3384549617767334},{"id":9175,"token":"Yes","bytes":[89,101,115],"logprob":-4.604161739349365},{"id":332,"token":"**","bytes":[42,42],"logprob":-5.898107051849365},{"id":9110,"token":"BL","bytes":[66,76],"logprob":-6.020013332366943},{"id":40,"token":"I","bytes":[73],"logprob":-6.169748783111572},{"id":9405,"token":"yes","bytes":[121,101,115],"logprob":-6.217698574066162},{"id":65,"token":"b","bytes":[98],"logprob":-6.424925327301025},{"id":104145,"token":"蓝色","bytes":[232,147,157,232,137,178],"logprob":-6.733023166656494},{"id":2434,"token":"True","bytes":[84,114,117,101],"logprob":-6.807143688201904},{"id":1956,"token":"bl","bytes":[98,108],"logprob":-6.82313871383667},{"id":46,"token":"O","bytes":[79],"logprob":-6.862432956695557},{"id":760,"token":"The","bytes":[84,104,101],"logprob":-6.892918109893799},{"id":32,"token":"A","bytes":[65],"logprob":-7.00538969039917},{"id":13602,"token":"YES","bytes":[89,69,83],"logprob":-7.131120204925537},{"id":9419,"token":"Hello","bytes":[72,101,108,108,111],"logprob":-7.233447551727295},{"id":4764,"token":"Bl","bytes":[66,108],"logprob":-7.251384258270264},{"id":58,"token":"[","bytes":[91],"logprob":-7.300139904022217}]} +{"id":159034,"token":"**.","bytes":[42,42,46],"logprob":-0.8614267110824585,"top_logprobs":[{"id":159034,"token":"**.","bytes":[42,42,46],"logprob":-0.8614267110824585},{"id":332,"token":"**","bytes":[42,42],"logprob":-0.9330915212631226},{"id":159029,"token":".**","bytes":[46,42,42],"logprob":-1.7704004049301147},{"id":201757,"token":"**!","bytes":[42,42,33],"logprob":-5.306993007659912},{"id":171458,"token":"!**","bytes":[33,42,42],"logprob":-5.681878566741943},{"id":93868,"token":"**,","bytes":[42,42,44],"logprob":-6.270991802215576},{"id":193260,"token":"**?","bytes":[42,42,63],"logprob":-6.327672481536865},{"id":221794,"token":"**。","bytes":[42,42,227,128,130],"logprob":-7.743163585662842},{"id":13,"token":".","bytes":[46],"logprob":-8.385009765625},{"id":11862,"token":"***","bytes":[42,42,42],"logprob":-9.013392448425293},{"id":92217,"token":"**:","bytes":[42,42,58],"logprob":-9.170659065246582},{"id":170286,"token":"?**","bytes":[63,42,42],"logprob":-9.27776050567627},{"id":88,"token":"y","bytes":[121],"logprob":-9.8964262008667},{"id":172064,"token":"**-","bytes":[42,42,45],"logprob":-9.906830787658691},{"id":190632,"token":"**;","bytes":[42,42,59],"logprob":-9.963706016540527},{"id":10838,"token":" ","bytes":[32,240,159],"logprob":-10.026280403137207},{"id":200386,"token":"！**","bytes":[239,188,129,42,42],"logprob":-10.144526481628418},{"id":173609,"token":"。**","bytes":[227,128,130,42,42],"logprob":-10.17459774017334},{"id":64700,"token":":**","bytes":[58,42,42],"logprob":-10.189746856689453},{"id":197005,"token":"**—","bytes":[42,42,226,128,148],"logprob":-10.424439430236816}]} diff --git a/docs/audits/prm-001/c11-4026-live/positive_control.sh b/docs/audits/prm-001/c11-4026-live/positive_control.sh new file mode 100755 index 0000000000..f3f609d073 --- /dev/null +++ b/docs/audits/prm-001/c11-4026-live/positive_control.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash +# #4026 positive control, offline: the comparator in c4026_live.sh, fed the +# oracle against ITSELF (the oracle's jsonl reshaped into apr's response shape), +# must pass with max |Δlogprob| = 0. If it cannot pass on identical data, a RED +# from it says nothing about apr. The comparator is not copied: it is extracted +# from c4026_live.sh at run time, so this tests the exact pre-registered bar. +# Usage: positive_control.sh [DIR] (DIR defaults to this receipt's directory) +set -euo pipefail +here="$(cd "$(dirname "$0")" && pwd)" +dir="${1:-$here}" +cmp_fn="$(sed -n '/^compare() {$/,/^}$/p' "$here/c4026_live.sh")" +[ -n "$cmp_fn" ] || { echo "RED: compare() not found in c4026_live.sh"; exit 2; } +eval "$cmp_fn" +rc=0 +for t in p2 p3 p5; do + # oracle jsonl -> {choices:[{logprobs:{content:[{token_id,logprob,top_logprobs:[{token_id,logprob}]}]}}]} + jq -s '{choices:[{logprobs:{content:[.[] | {token_id:.id, logprob:.logprob, + top_logprobs:[.top_logprobs[] | {token_id:.id, logprob:.logprob}]}]}}]}' \ + "$dir/$t.oracle.jsonl" >"$dir/$t.oracle_as_apr.json" + compare "$dir/$t.oracle_as_apr.json" "$dir/$t.oracle.jsonl" 0 >"$dir/$t.posctl.json" + pass=$(jq .pass "$dir/$t.posctl.json") + maxd=$(jq '[.steps[].max_abs_dlogprob] | max' "$dir/$t.posctl.json") + echo "$t positive control: pass=$pass steps=$(jq '.steps|length' "$dir/$t.posctl.json") max_abs_dlogprob=$maxd" + [ "$pass" = true ] && [ "$maxd" = 0 ] || rc=1 +done +exit "$rc" diff --git a/docs/audits/prm-001/prereg.lock b/docs/audits/prm-001/prereg.lock new file mode 100644 index 0000000000..417025d834 --- /dev/null +++ b/docs/audits/prm-001/prereg.lock @@ -0,0 +1,7 @@ +# PRM-001 pre-registration lock (rex-prereg-v2) over spec PRM-001 v3. Written by PRM-00b; v1 had 0 data rows. +# A mismatch against the tree is a new spec version (R-1), never an edit here. +spec_s2_s5=f53db26dee2d5f66ab597f2da012fa13a87a0a5002532b267a28091f0e5626f7 +stats_rs=abb51113c5971f9460f8c990fdb58094ca04943cbdb863f00a916472d9b088cc +analysis_plan=2b84b3b0026300364bdcf09e66967a25c43dc47749abf790f7c238380e9f2969 +prompt_v1=dd9d67c2df458d92d66aed65fd7f3f6f2e5602499c05e6b0967465b9bd2662e6 +prereg_sha=f0b4e4ac07e605e313e61ce5efa9e43f7339ca5a5e5275b0d5f06b0277011041 diff --git a/docs/audits/prm-001/prm-09/RECEIPT.md b/docs/audits/prm-001/prm-09/RECEIPT.md new file mode 100644 index 0000000000..ca3c995cfe --- /dev/null +++ b/docs/audits/prm-001/prm-09/RECEIPT.md @@ -0,0 +1,31 @@ +# PRM-09 — ladder extension: the three planted reports (rex-promotion-ladder-v2 1.1.0) + +**Outcome: acceptance met.** The PRM-001 §9 row reads "H4 only → tripwire; H4+H5+H7 → tie-breaker; lambda primary → refused". All three run through the real binary, `rex ladder`, and not only through unit tests. + +The reports below are `prm-001-report-v1` documents (spec PRM-001, version 3). They carry the **live** prereg lock, `docs/audits/prm-001/prereg.lock` `prereg_sha` `f0b4e4ac…`, so they exercise the gate and not the stale-lock refusal. Every run is recorded verbatim in `runs.txt`. + +| Planted report | sha256 (prefix) | `--current` | mode | why | +|---|---|---|---|---| +| `planted-h4-only.json`: H4 holds (Holm p 0.01), within budget, gx10-cpu primary | `3922acbb9040d0f3` | shadow | **tripwire** | H5, H7, H1 and H2 are not in the report | +| `planted-h4-h5-h7.json`: adds H5 (p 0.04), H7, H1, H2; availability 0.97 | `b4e225242cd0f0eb` | tripwire | **tie-breaker** | vote needs ≥ 20 decided tie-breaks, and it has 0 | +| `planted-lambda-primary.json`: the same report with primary `lambda-cuda` | `e9e875dc457aa891` | tripwire | **tripwire (refused)** | `S-4: primary cell lambda-cuda is lambda or unnamed; lambda is shadow-only (R-9)` | +| the same lambda report | ″ | tie-breaker | **tripwire** | S-4 caps at once. It is the one exception to the one-rung step | +| the H4+H5+H7 report | ″ | shadow | tripwire | the positive control for the step rule: one rung per report, Shadow → Tripwire, never straight to TieBreaker | + +## Defect fixed on the way + +The S-4 reason printed the Rust Debug rendering of the cell, `primary cell Some("lambda-cuda")`. That string would have reached the quorum gate and the arbiter's receipts. It now names the cell as written, or `(unnamed)`. The fix is FALSIFY-RXG-006 in contract 1.1.0: planted RED in `cf7bdbfbb`, then GREEN. + +## Named follow-ups (not in this change) + +- **pi#428**: the paiml-implement quorum gate reads `rex ladder … | .mode`. +- **infra#1095**: the arbiter reads the same `.mode`. + +Both consume the `.mode` field and must never re-derive it (contract description). + +## Gates + +- `cargo test -p aprender-review-experiment --lib`: 167 passed. +- `cargo clippy -p aprender-review-experiment --all-targets -- -D warnings`: clean. +- `cargo fmt`: clean. +- `pv validate contracts/rex-promotion-ladder-v2.yaml`: valid. diff --git a/docs/audits/prm-001/prm-09/planted-h4-h5-h7.json b/docs/audits/prm-001/prm-09/planted-h4-h5-h7.json new file mode 100644 index 0000000000..0a303671c1 --- /dev/null +++ b/docs/audits/prm-001/prm-09/planted-h4-h5-h7.json @@ -0,0 +1,50 @@ +{ + "spec": "PRM-001", + "version": 3, + "prereg_sha": "f0b4e4ac07e605e313e61ce5efa9e43f7339ca5a5e5275b0d5f06b0277011041", + "lane": { + "rung": "shadow", + "availability_7d": 0.97 + }, + "speed": { + "within_budget": true + }, + "results": { + "hypotheses": [ + { + "id": "H4", + "ci": [ + 0.01, + 0.2 + ], + "p_holm": 0.01, + "verdict": "holds" + }, + { + "id": "H5", + "ci": [ + 0.01, + 0.2 + ], + "p_holm": 0.04, + "verdict": "holds" + }, + { + "id": "H7", + "verdict": "holds" + }, + { + "id": "H1", + "verdict": "holds" + }, + { + "id": "H2", + "verdict": "holds" + } + ] + }, + "hardware_ruling": { + "primary": "gx10-cpu", + "rung_eligible": "vote" + } +} diff --git a/docs/audits/prm-001/prm-09/planted-h4-only.json b/docs/audits/prm-001/prm-09/planted-h4-only.json new file mode 100644 index 0000000000..4a63a57df3 --- /dev/null +++ b/docs/audits/prm-001/prm-09/planted-h4-only.json @@ -0,0 +1,29 @@ +{ + "spec": "PRM-001", + "version": 3, + "prereg_sha": "f0b4e4ac07e605e313e61ce5efa9e43f7339ca5a5e5275b0d5f06b0277011041", + "lane": { + "rung": "shadow", + "availability_7d": 0.97 + }, + "speed": { + "within_budget": true + }, + "results": { + "hypotheses": [ + { + "id": "H4", + "ci": [ + 0.01, + 0.2 + ], + "p_holm": 0.01, + "verdict": "holds" + } + ] + }, + "hardware_ruling": { + "primary": "gx10-cpu", + "rung_eligible": "vote" + } +} diff --git a/docs/audits/prm-001/prm-09/planted-lambda-primary.json b/docs/audits/prm-001/prm-09/planted-lambda-primary.json new file mode 100644 index 0000000000..b1405e0394 --- /dev/null +++ b/docs/audits/prm-001/prm-09/planted-lambda-primary.json @@ -0,0 +1,50 @@ +{ + "spec": "PRM-001", + "version": 3, + "prereg_sha": "f0b4e4ac07e605e313e61ce5efa9e43f7339ca5a5e5275b0d5f06b0277011041", + "lane": { + "rung": "shadow", + "availability_7d": 0.97 + }, + "speed": { + "within_budget": true + }, + "results": { + "hypotheses": [ + { + "id": "H4", + "ci": [ + 0.01, + 0.2 + ], + "p_holm": 0.01, + "verdict": "holds" + }, + { + "id": "H5", + "ci": [ + 0.01, + 0.2 + ], + "p_holm": 0.04, + "verdict": "holds" + }, + { + "id": "H7", + "verdict": "holds" + }, + { + "id": "H1", + "verdict": "holds" + }, + { + "id": "H2", + "verdict": "holds" + } + ] + }, + "hardware_ruling": { + "primary": "lambda-cuda", + "rung_eligible": "vote" + } +} diff --git a/docs/audits/prm-001/prm-09/runs.txt b/docs/audits/prm-001/prm-09/runs.txt new file mode 100644 index 0000000000..16648ce615 --- /dev/null +++ b/docs/audits/prm-001/prm-09/runs.txt @@ -0,0 +1,5 @@ +rex ladder planted-h4-only.json --current shadow → rc=0 {"mode":"tripwire","reasons":["H5: not in the report","H7: not in the report","H1: not in the report","H2: not in the report"]} +rex ladder planted-h4-h5-h7.json --current tripwire → rc=0 {"mode":"tie-breaker","reasons":["tie_breaks_decided 0 is not ≥ 20"]} +rex ladder planted-lambda-primary.json --current tripwire → rc=0 {"mode":"tripwire","reasons":["tie_breaks_decided 0 is not ≥ 20","S-4: primary cell lambda-cuda is lambda or unnamed; lambda is shadow-only (R-9)"]} +rex ladder planted-lambda-primary.json --current tie-breaker → rc=0 {"mode":"tripwire","reasons":["tie_breaks_decided 0 is not ≥ 20","S-4: primary cell lambda-cuda is lambda or unnamed; lambda is shadow-only (R-9)"]} +rex ladder planted-h4-h5-h7.json --current shadow → rc=0 {"mode":"tripwire","reasons":["tie_breaks_decided 0 is not ≥ 20","one rung per report: Shadow → Tripwire, not TieBreaker"]} diff --git a/docs/audits/review-corpus/corpus-v1.jsonl b/docs/audits/review-corpus/corpus-v1.jsonl new file mode 100644 index 0000000000..22785f6111 --- /dev/null +++ b/docs/audits/review-corpus/corpus-v1.jsonl @@ -0,0 +1,150 @@ +{"id":"G-pr1026","class":"G","stratum":"S","split":"Test","diff_sha256":"dbf2e76f07bdf31bde4a9c92e2d8698c46b80224ad101876525711f71afca173","bytes":3887,"approx_tokens":972,"source":"pr#1026 51c5b433ea667716bf4017871101d8f55ac472a0","defect":[]} +{"id":"G-pr1052","class":"G","stratum":"L","split":"Test","diff_sha256":"5e72ec8a21cc3687e1bc84eb3b99b58fe737f7fabe7788546348dbddbed1bc95","bytes":43976,"approx_tokens":10994,"source":"pr#1052 32053ac73770a77a66dcb09a5d49cd18464c1224","defect":[]} +{"id":"G-pr1057","class":"G","stratum":"S","split":"Dev","diff_sha256":"873740a1b60ffd6d875e57cd4cde7707630d96db414a5451d0c50538a3fcc15c","bytes":3754,"approx_tokens":939,"source":"pr#1057 c4210b44a7cf390ab6e23d579fb40598776c1276","defect":[]} +{"id":"G-pr1058","class":"G","stratum":"S","split":"Test","diff_sha256":"52b05d908f3e82ae8c8a0a2fd25a91b4a1a8fbc035892ca41a35b779ec7c5296","bytes":1841,"approx_tokens":461,"source":"pr#1058 e8394323017d96e641adfa7a6a3002a1528515cd","defect":[]} +{"id":"G-pr1081","class":"G","stratum":"M","split":"Test","diff_sha256":"d2091f65f7ed18515023b8905f660ccda8ae9bc26cf72d9b398e2a70ef12fdec","bytes":14750,"approx_tokens":3688,"source":"pr#1081 108f8f71cbe0dfad15e0d4d4badaa3f91fdccca4","defect":[]} +{"id":"G-pr1102","class":"G","stratum":"M","split":"Dev","diff_sha256":"4e54b8b0c73f08f1776c8315ec2ec4aad1b69fe979b1f74d615eb56cfa9ba598","bytes":17932,"approx_tokens":4483,"source":"pr#1102 2e003aca563c480482cc095659e88a812ec96e7d","defect":[]} +{"id":"G-pr1136","class":"G","stratum":"M","split":"Test","diff_sha256":"9631361a48f554f6dddd39803ed12ac599663ef3bc18458b4e64273589e0eeec","bytes":10995,"approx_tokens":2749,"source":"pr#1136 1b921caf26444554319c71f4a4655c0486729188","defect":[]} +{"id":"G-pr1144","class":"G","stratum":"M","split":"Dev","diff_sha256":"47a4c7e7674f97b03fd99c4058e433a24459d9bbe86e13fca6d7c13e64d567ac","bytes":12198,"approx_tokens":3050,"source":"pr#1144 39d46ce28105d9e824d08e65e9c3fd648f3c8528","defect":[]} +{"id":"G-pr1173","class":"G","stratum":"M","split":"Dev","diff_sha256":"fef20b9364000f04aaab336ddf0aa537f4bf4ac78c9892f8f072a28be249f60c","bytes":10036,"approx_tokens":2509,"source":"pr#1173 97a5984e5d8146b6f6d80b9d81fa90430e9a83a3","defect":[]} +{"id":"G-pr1408","class":"G","stratum":"S","split":"Test","diff_sha256":"b5a1581d712f18d2f11c32d33ed588a358f79fcc706f9441705e66fe75fa0bfb","bytes":7052,"approx_tokens":1763,"source":"pr#1408 f1a6afea127caa36c786af3e9be19ee92f77cdaa","defect":[]} +{"id":"G-pr1516","class":"G","stratum":"S","split":"Test","diff_sha256":"8a64769fd7a2021c6559ed5334f4b8519b71e30a656c0e688c985fdcb47d857f","bytes":7908,"approx_tokens":1977,"source":"pr#1516 3138d134dd245d3ea81a0be1a6335857c185ac7e","defect":[]} +{"id":"G-pr1536","class":"G","stratum":"S","split":"Test","diff_sha256":"ede8268135e2f9e3d03eefe0237a494ab458eebea31bd518f1e6d3b14bb28fbc","bytes":5258,"approx_tokens":1315,"source":"pr#1536 496e9555365226ab31ba0f709de296fa1c11a35f","defect":[]} +{"id":"G-pr1543","class":"G","stratum":"M","split":"Test","diff_sha256":"fbbfea4dfbb011561f92c791a1399818cccc4951580defb1fe217c20a37291a4","bytes":12075,"approx_tokens":3019,"source":"pr#1543 313864184d437fe68750c7f6c0244df183cb38f1","defect":[]} +{"id":"G-pr1553","class":"G","stratum":"M","split":"Dev","diff_sha256":"2927c63f9161979b74400076b87c1740a8e3558af6250ec4303f06df4d0b6733","bytes":31526,"approx_tokens":7882,"source":"pr#1553 a2617e241e780fb27399e2c9fb351753ef23b536","defect":[]} +{"id":"G-pr1555","class":"G","stratum":"L","split":"Test","diff_sha256":"f0dc753d991a0fc4f2649e6efe4d31c259f510e4b620080f7b74e950f02167eb","bytes":33333,"approx_tokens":8334,"source":"pr#1555 5abd613bb9009802a7a50626f95911e9069a01d6","defect":[]} +{"id":"G-pr1568","class":"G","stratum":"M","split":"Test","diff_sha256":"9826cd5a6fcc1cbac765a87322a66dffbabe7149ac76e7a095642a9482d9d9cb","bytes":18848,"approx_tokens":4712,"source":"pr#1568 f7df2f168492a8378ba2d6ea2fb450e01f119e2e","defect":[]} +{"id":"G-pr1572","class":"G","stratum":"S","split":"Dev","diff_sha256":"0707095b461633bd78de98dff4e0a281d06b5393743aa6aaf2104ee2717d34e2","bytes":2588,"approx_tokens":647,"source":"pr#1572 efca65fe31b45b89808f60a0014bdb452fcf873c","defect":[]} +{"id":"G-pr1573","class":"G","stratum":"S","split":"Dev","diff_sha256":"7e4eb08f7ae958940df057551ffce5d680f607b10f81b0870d7cb2150fceb799","bytes":3927,"approx_tokens":982,"source":"pr#1573 036fa7b57cb72e6a550f99c97ec38a115414a15d","defect":[]} +{"id":"G-pr1602","class":"G","stratum":"M","split":"Test","diff_sha256":"0e8e87d8a2b062f04e24c651677a13e50c75ab032f1891388dffd83fc2118287","bytes":14135,"approx_tokens":3534,"source":"pr#1602 86ad83b80051c793f78f55e0879336d2c0a91fd7","defect":[]} +{"id":"G-pr1664","class":"G","stratum":"S","split":"Test","diff_sha256":"03fe31a5ed2150c50a8f2ed444803f49bacf6dd3edd304d38bd61dc9f5ba370b","bytes":4602,"approx_tokens":1151,"source":"pr#1664 e089c95ed0955aaffd9ff0a37d18f63cea8a47fa","defect":[]} +{"id":"G-pr1694","class":"G","stratum":"S","split":"Test","diff_sha256":"565c6d9d2ac74ddb3174bd31894a5ecbbcee13d5d227a0cd451cc16c6a6b14ee","bytes":6124,"approx_tokens":1531,"source":"pr#1694 8cdbd90b396bb1ef8a2a87abae7862a8ead6d284","defect":[]} +{"id":"G-pr1742","class":"G","stratum":"L","split":"Dev","diff_sha256":"a528841846193ecb62f9c1068264abac1b0e2b1ed036f38591aacf657b59bb9f","bytes":42636,"approx_tokens":10659,"source":"pr#1742 06f722dd6d6dc6ae48974eea2a69f7f5aec1dd56","defect":[]} +{"id":"G-pr1779","class":"G","stratum":"L","split":"Test","diff_sha256":"0220fcea591f9c9bb66c9784dbb17d321ff41420790014b628838d6322694de5","bytes":36182,"approx_tokens":9046,"source":"pr#1779 74aaee49f950702413c731e8c6c1ed0246147484","defect":[]} +{"id":"G-pr1804","class":"G","stratum":"S","split":"Test","diff_sha256":"e31677b8f712468175040fec4f3ae1be70f7973d62aa83884a38a307c425c9da","bytes":3059,"approx_tokens":765,"source":"pr#1804 8026f0aafd1ad70998c5966b39d18361e9a1fa76","defect":[]} +{"id":"G-pr1919","class":"G","stratum":"S","split":"Test","diff_sha256":"fffd8169289f9084d6b09fd2ec19c49b18592f2d4d1e4d0fc09aaeee78936ad4","bytes":7994,"approx_tokens":1999,"source":"pr#1919 546c782020f70e4c18ea917e3c1ed89b03ea59b1","defect":[]} +{"id":"G-pr1927","class":"G","stratum":"M","split":"Dev","diff_sha256":"895ad06b66462c381ac7e1590f6b08128509c00ec6331422993e3cc50c7eba4d","bytes":9174,"approx_tokens":2294,"source":"pr#1927 e07f15f8a5cb5f0dda0b5c1486104ec45061bdb2","defect":[]} +{"id":"G-pr1946","class":"G","stratum":"S","split":"Dev","diff_sha256":"34c1ef5554f8965121bea73fd860afe73f5905cff0dfdf34fe5c0387096ec508","bytes":4964,"approx_tokens":1241,"source":"pr#1946 70b162b119abb3e7956b4f30368acd5422d26930","defect":[]} +{"id":"G-pr1951","class":"G","stratum":"S","split":"Dev","diff_sha256":"20d131de0f7401d1229fbaf14f52627eca71b0ce97733acaa494f2d333af26f5","bytes":1025,"approx_tokens":257,"source":"pr#1951 a5cdf11ba3ed0c4dfbab2d452299b51492ea2051","defect":[]} +{"id":"G-pr1962","class":"G","stratum":"S","split":"Test","diff_sha256":"f5422e3d66d0dce51be91b3c04f513962c9a2727bfc33fc38e772818debbebc1","bytes":4966,"approx_tokens":1242,"source":"pr#1962 92dacaddeb3a67524fa1030ad225380588aba924","defect":[]} +{"id":"G-pr2064","class":"G","stratum":"L","split":"Test","diff_sha256":"c74635eb50fc1949e0a49941b83f15f36b9422c516e2ac25d42b861d9f9bb4ce","bytes":33686,"approx_tokens":8422,"source":"pr#2064 52c933da702894f4cb7563261b31e71e3560f061","defect":[]} +{"id":"G-pr2100","class":"G","stratum":"L","split":"Dev","diff_sha256":"b90787bbc1de342594e578fd36e90070d17a0a6b3a5fe03d76617b094fca01a2","bytes":62238,"approx_tokens":15560,"source":"pr#2100 1a1bf6fe65f350718abe8dce96324e678ee23cbd","defect":[]} +{"id":"G-pr2169","class":"G","stratum":"L","split":"Test","diff_sha256":"41532670a29989bfb28f4a7dbc5fbd8a5e1a3c378661a981f0ba7338a3e4edfb","bytes":39468,"approx_tokens":9867,"source":"pr#2169 4330f7d0635a4182b872746c428b71725aae32d7","defect":[]} +{"id":"G-pr2184","class":"G","stratum":"M","split":"Test","diff_sha256":"ee39081c2974f1623c5b6a0613e237659e972f03c5cfd4df1f9d24774cf3f318","bytes":22097,"approx_tokens":5525,"source":"pr#2184 79c3b8b75c7a1031011b6d1dc167d920153e4346","defect":[]} +{"id":"G-pr2216","class":"G","stratum":"L","split":"Test","diff_sha256":"bc3064ff29427cb268689b862153cc6fffff6898f8f3f850b828e5625e010ce4","bytes":51315,"approx_tokens":12829,"source":"pr#2216 53c9b541cd35485086be94796735ec4391a1d241","defect":[]} +{"id":"G-pr2227","class":"G","stratum":"L","split":"Dev","diff_sha256":"28c93295ce9295f699d1ab17eecef2d2ef643f6ec8434694ee70fa20ad70c0b9","bytes":44606,"approx_tokens":11152,"source":"pr#2227 5f780cbd01ecc3bec884a5941ec642314fcd75f1","defect":[]} +{"id":"G-pr2229","class":"G","stratum":"L","split":"Test","diff_sha256":"53480f5468a2203ed2ab24d5e5a170e883a167d8f2d145e90f4c54ad045180b4","bytes":36298,"approx_tokens":9075,"source":"pr#2229 cb0b6e9bcc99cfbdb898175eace1536de6916d57","defect":[]} +{"id":"G-pr2230","class":"G","stratum":"M","split":"Test","diff_sha256":"8bbccaaf96791623b2611e5d9bdbae2e98dcffbc765c7ef57b88388403cbb30a","bytes":24848,"approx_tokens":6212,"source":"pr#2230 821a450934748598110fa161dd3dbee3cc35e1d3","defect":[]} +{"id":"G-pr2233","class":"G","stratum":"L","split":"Test","diff_sha256":"923635105eb240551f34c4cbea2770d05bc980a24c407f91858d8bc635965b53","bytes":74567,"approx_tokens":18642,"source":"pr#2233 6ad4e6fccdca7e4358f309dbd67e18278f6f1693","defect":[]} +{"id":"G-pr2266","class":"G","stratum":"S","split":"Test","diff_sha256":"c42ca8723991273d0b3d44d0be4b57e1c8484bedfb2e0df7b8add569cebe39e4","bytes":5474,"approx_tokens":1369,"source":"pr#2266 e45d888b628e29292302e77fd33523a5c8e01fab","defect":[]} +{"id":"G-pr2337","class":"G","stratum":"M","split":"Test","diff_sha256":"750778c9ef2c8c44a41665133999173cb0ba0ed78831c0414bb19e03df3b4426","bytes":17107,"approx_tokens":4277,"source":"pr#2337 9ca3a8bfe05895afda70ca38604ef599120d9bab","defect":[]} +{"id":"G-pr2710","class":"G","stratum":"L","split":"Test","diff_sha256":"16cc93cf4c3c0e1d724f49231b643265c836da1b70e14bad351f71be3e8b571d","bytes":75895,"approx_tokens":18974,"source":"pr#2710 de8fbc4071fb0aabe0a376ea8e08cdd38597e52e","defect":[]} +{"id":"G-pr2746","class":"G","stratum":"L","split":"Test","diff_sha256":"b47aa64a2b06f68b521fb7d447608cbdd1bd4baab87e22142440f5c638f03ee3","bytes":115149,"approx_tokens":28788,"source":"pr#2746 31732f5dbd64f5e69b7ef802e547c80ac6deba9f","defect":[]} +{"id":"G-pr2837","class":"G","stratum":"L","split":"Dev","diff_sha256":"97d3eb6ce7e7f12f3242c105c8b10e0a6de7ce533584c8b996b9e2ddbe2fb037","bytes":46161,"approx_tokens":11541,"source":"pr#2837 ca5f3e1542405790c82fe5224f540460f0bf228d","defect":[]} +{"id":"G-pr741","class":"G","stratum":"S","split":"Test","diff_sha256":"acb894cb62d2b837954f9ec2c20c38a4f8535a1b6ff95daf5c56109c3281eb53","bytes":3320,"approx_tokens":830,"source":"pr#741 1f18e5b190bae97ae255f093c3c0f5fa1c36f877","defect":[]} +{"id":"G-pr753","class":"G","stratum":"M","split":"Test","diff_sha256":"79bba234bf32fe754fcab5ab41ae01ea4ef5d2ef4088aae490cefeae1155b193","bytes":16370,"approx_tokens":4093,"source":"pr#753 efd774f176ddb61776c0aabd69183e91346f5eb8","defect":[]} +{"id":"G-pr888","class":"G","stratum":"L","split":"Test","diff_sha256":"688c27431c3a9733f4936aefd7edb1eed93bbd3875f35e8704a8da0da1d808f1","bytes":109263,"approx_tokens":27316,"source":"pr#888 df66e5b208ea9947b6e3c0e05391963a0c9758aa","defect":[]} +{"id":"G-pr940","class":"G","stratum":"M","split":"Test","diff_sha256":"a3e65146b9110135d9a93050791429a09dd5fde35cd527e6f0848e0bd3d55a48","bytes":11656,"approx_tokens":2914,"source":"pr#940 b8bed16c2a93ec7836ba276c6c57fb732778a07a","defect":[]} +{"id":"G-pr941","class":"G","stratum":"M","split":"Test","diff_sha256":"5810531966d5c87102685bfb0afd7525795c1867c9dc8029995d155dbcd247cd","bytes":10742,"approx_tokens":2686,"source":"pr#941 af9b2f13966d7caafc2ae916f4cd5c8d651cedcb","defect":[]} +{"id":"G-pr953","class":"G","stratum":"M","split":"Test","diff_sha256":"7172f520ed7187e1ef725d7c17a0ecbf59e9a9c03622bffd5f20ea4b67545d84","bytes":17744,"approx_tokens":4436,"source":"pr#953 9ee2bfff2930095d29e0566cfdabc73b45f75f4d","defect":[]} +{"id":"G-pr983","class":"G","stratum":"L","split":"Dev","diff_sha256":"8de21a4560fa9d040ff370bf1f7973754b29f83140a02a6e2a1677c0abcd8a22","bytes":32504,"approx_tokens":8126,"source":"pr#983 6ab563be2d8f8aa4d9e9ef06907131b6eb1a5b5e","defect":[]} +{"id":"P-033caf9c30ca","class":"P","stratum":"S","split":"Test","diff_sha256":"83858a70fc8c0a0b603dcde3fdf466d2dfc08007a5b79debb7f505177265aecd","bytes":1331,"approx_tokens":333,"source":"cargo-mutants 27.0.0 @ 80437647e: aprender-core :: crates/aprender-core/src/data/pii.rs:364:5: replace try_parse_ipv4 -> Option<(usize, String)> with Some((1, \"xyzzy\".into()))","defect":[{"file":"crates/aprender-core/src/data/pii.rs","line":364}]} +{"id":"P-06a15fbc27d1","class":"P","stratum":"S","split":"Test","diff_sha256":"fcef599862241eb390e8fe36b0e3f679964945f59155eaaf385b24d42d718a48","bytes":670,"approx_tokens":168,"source":"cargo-mutants 27.0.0 @ 80437647e: aprender-core :: crates/aprender-core/src/classification/gaussian_nb.rs:64:19: replace > with == in KNearestNeighbors::fit","defect":[{"file":"crates/aprender-core/src/classification/gaussian_nb.rs","line":64}]} +{"id":"P-0afb0412cc4c","class":"P","stratum":"S","split":"Test","diff_sha256":"8984842123c0eb7fa001f563e26261f23c7cdcf87eef1f66f3ac01f2b7a3631a","bytes":1258,"approx_tokens":315,"source":"cargo-mutants 27.0.0 @ 80437647e: aprender-core :: crates/aprender-core/src/online/cpt.rs:182:9: replace DataMixer::mix_batches -> Vec> with vec![]","defect":[{"file":"crates/aprender-core/src/online/cpt.rs","line":182}]} +{"id":"P-0cfe3b858c03","class":"P","stratum":"S","split":"Test","diff_sha256":"ce09d1364459e580b6ef521f1ca5e7ba651e06ae7e27ec02b6f87dd94d402580","bytes":1052,"approx_tokens":263,"source":"cargo-mutants 27.0.0 @ 80437647e: aprender-serve :: crates/aprender-serve/src/apr_transformer/config.rs:139:44: replace + with - in AprKVCache::append","defect":[{"file":"crates/aprender-serve/src/apr_transformer/config.rs","line":139}]} +{"id":"P-101aa47005a0","class":"P","stratum":"S","split":"Test","diff_sha256":"463d1922155417d9d37b1a2cf50d861df5b388207f94b8096d271369d181e8ff","bytes":798,"approx_tokens":200,"source":"cargo-mutants 27.0.0 @ 80437647e: aprender-core :: crates/aprender-core/src/nn/optim/rm_sprop.rs:224:42: replace * with / in RMSprop::update_param","defect":[{"file":"crates/aprender-core/src/nn/optim/rm_sprop.rs","line":224}]} +{"id":"P-1486e2af4c6e","class":"P","stratum":"S","split":"Test","diff_sha256":"e24f2020c4e1bdb913c863774ef3bfb0c4584a77e45cf1692d2597d2b6b947b0","bytes":731,"approx_tokens":183,"source":"cargo-mutants 27.0.0 @ 80437647e: aprender-serve :: crates/aprender-serve/src/bench/statistics.rs:338:26: replace > with >= in RegressionDetector::compare","defect":[{"file":"crates/aprender-serve/src/bench/statistics.rs","line":338}]} +{"id":"P-16a3b1d9bab2","class":"P","stratum":"S","split":"Test","diff_sha256":"77d23ece37e4eb1bbd486b7314a1be845e7feac3a3b104b27100fb33ae6d8e90","bytes":761,"approx_tokens":191,"source":"cargo-mutants 27.0.0 @ 80437647e: aprender-contracts :: crates/aprender-contracts/src/ontology/extract/lean.rs:47:9: replace Statement::module -> String with \"xyzzy\".into()","defect":[{"file":"crates/aprender-contracts/src/ontology/extract/lean.rs","line":47}]} +{"id":"P-19fde4fa1f1f","class":"P","stratum":"S","split":"Test","diff_sha256":"284183be921ac0288241353520b358288e76a568016621dbe71ef1d911921567","bytes":1435,"approx_tokens":359,"source":"cargo-mutants 27.0.0 @ 80437647e: apr-cli :: crates/apr-cli/src/commands/imatrix_lint.rs:196:5: replace extract_hash_set -> Result, String> with Ok(BTreeSet::from_iter([String::new()]))","defect":[{"file":"crates/apr-cli/src/commands/imatrix_lint.rs","line":196}]} +{"id":"P-1bf73ee9649a","class":"P","stratum":"S","split":"Dev","diff_sha256":"676c805645f1d8841557b13fc82e12516d004da47e409e040b38f217482e212f","bytes":1104,"approx_tokens":276,"source":"cargo-mutants 27.0.0 @ 80437647e: aprender-core :: crates/aprender-core/src/stats/hypothesis.rs:446:62: replace + with * in incomplete_beta","defect":[{"file":"crates/aprender-core/src/stats/hypothesis.rs","line":446}]} +{"id":"P-1e0b33ddf721","class":"P","stratum":"S","split":"Dev","diff_sha256":"4b76a1316364fc05122f1990ad7f73b39804fc31ed86df91e102e8d40d97c15b","bytes":721,"approx_tokens":181,"source":"cargo-mutants 27.0.0 @ 80437647e: apr-cli :: crates/apr-cli/src/commands/finetune.rs:816:25: replace * with + in chunk_lm_head_transposed","defect":[{"file":"crates/apr-cli/src/commands/finetune.rs","line":816}]} +{"id":"P-1e8aa7eff08c","class":"P","stratum":"S","split":"Test","diff_sha256":"f32bd6add6876b9ad0a7af0ce38b1d5fc86b0f3852d9fb0c05194a3cc8bf44bb","bytes":2389,"approx_tokens":598,"source":"cargo-mutants 27.0.0 @ 80437647e: apr-cli :: crates/apr-cli/src/commands/test_llm.rs:251:5: replace print_report with ()","defect":[{"file":"crates/apr-cli/src/commands/test_llm.rs","line":251}]} +{"id":"P-2936deac92fe","class":"P","stratum":"S","split":"Dev","diff_sha256":"2417ec6cda83f4f13dc2bd1e07b21446aa51132d8a44e0ed4a30009d106dfaf0","bytes":946,"approx_tokens":237,"source":"cargo-mutants 27.0.0 @ 80437647e: aprender-core :: crates/aprender-core/src/verify/tolerance.rs:110:9: replace Tolerance::description -> String with String::new()","defect":[{"file":"crates/aprender-core/src/verify/tolerance.rs","line":110}]} +{"id":"P-2f194b384bba","class":"P","stratum":"S","split":"Test","diff_sha256":"4b29e387d50c0bdd8547b35df3e0c6c31b7792a9a27031e146eaee6d68cf71a5","bytes":1072,"approx_tokens":268,"source":"cargo-mutants 27.0.0 @ 80437647e: apr-cli :: crates/apr-cli/src/commands/parity_per_op.rs:108:16: delete ! in run","defect":[{"file":"crates/apr-cli/src/commands/parity_per_op.rs","line":108}]} +{"id":"P-32457b5e3477","class":"P","stratum":"S","split":"Test","diff_sha256":"88ed951903fcb3f92837a43c8fd304ac5dc6ea7133a219c7d4fa33d4e575aeee","bytes":864,"approx_tokens":216,"source":"cargo-mutants 27.0.0 @ 80437647e: aprender-core :: crates/aprender-core/src/audio/noise/spectral.rs:259:9: replace SpectralMLP::weights -> (&[f32], &[f32], &[f32], &[f32], &[f32], &[f32]) with (Vec::leak(Vec::new()), Vec::leak(vec![0.0]), Vec::leak(vec![0.0]), Vec::leak(vec![1.0]), Vec::leak(vec![-1.0]), Vec::leak(vec![1.0]))","defect":[{"file":"crates/aprender-core/src/audio/noise/spectral.rs","line":259}]} +{"id":"P-344775d57610","class":"P","stratum":"S","split":"Test","diff_sha256":"c6f90a22e788c9af6e6ba222717f809ac6287793477888fc41ed342f389e40d0","bytes":737,"approx_tokens":185,"source":"cargo-mutants 27.0.0 @ 80437647e: aprender-serve :: crates/aprender-serve/src/gpu/scheduler/linear_attn.rs:232:39: replace + with - in forward_linear_block_incremental","defect":[{"file":"crates/aprender-serve/src/gpu/scheduler/linear_attn.rs","line":232}]} +{"id":"P-3b35f5a6934a","class":"P","stratum":"S","split":"Test","diff_sha256":"0935fefc74ee00843f623371b9b3f51087616f566c0c53581da5712bff5dc29f","bytes":837,"approx_tokens":210,"source":"cargo-mutants 27.0.0 @ 80437647e: aprender-core :: crates/aprender-core/src/audio/noise/generator.rs:247:46: replace / with % in NoiseGenerator::update_config","defect":[{"file":"crates/aprender-core/src/audio/noise/generator.rs","line":247}]} +{"id":"P-42208d822acf","class":"P","stratum":"S","split":"Dev","diff_sha256":"cd8fe6aa36d3011c0e11427424b9b00b159b8cb256464a1aaa0e1c35b98bb0f5","bytes":786,"approx_tokens":197,"source":"cargo-mutants 27.0.0 @ 80437647e: aprender-serve :: crates/aprender-serve/src/cuda/executor/layers/cublas_prefill/gemm.rs:859:39: replace + with - in CudaExecutor::launch_mw_q4k_wmma_kernel","defect":[{"file":"crates/aprender-serve/src/cuda/executor/layers/cublas_prefill/gemm.rs","line":859}]} +{"id":"P-42dac01a6294","class":"P","stratum":"S","split":"Dev","diff_sha256":"45cc74380a677d4d07e3c47b2216f0355d9e3926af77b946ade206374800f97e","bytes":1017,"approx_tokens":255,"source":"cargo-mutants 27.0.0 @ 80437647e: apr-cli :: crates/apr-cli/src/commands/debug.rs:301:57: replace < with <= in parse_header","defect":[{"file":"crates/apr-cli/src/commands/debug.rs","line":301}]} +{"id":"P-46aa57d24e25","class":"P","stratum":"S","split":"Dev","diff_sha256":"4cd66114ca53103513a7b3c3f15eadacfa05df7b6252facf0901f451349b0b25","bytes":1104,"approx_tokens":276,"source":"cargo-mutants 27.0.0 @ 80437647e: aprender-serve :: crates/aprender-serve/src/gpu/mod.rs:186:9: replace ContiguousAttentionBuffer::get_views -> (&[f32], &[f32], &[f32], &[f32]) with (Vec::leak(vec![0.0]), Vec::leak(vec![0.0]), Vec::leak(vec![1.0]), Vec::leak(Vec::new()))","defect":[{"file":"crates/aprender-serve/src/gpu/mod.rs","line":186}]} +{"id":"P-46e545d0621f","class":"P","stratum":"S","split":"Test","diff_sha256":"9ff8d7e9045c8668ab4a3ed75e5127d05a3b9cf617f151fb165043ed6704a0f1","bytes":1004,"approx_tokens":251,"source":"cargo-mutants 27.0.0 @ 80437647e: aprender-serve :: crates/aprender-serve/src/apr_transformer/generation.rs:52:5: replace process_prompt_tokens -> Result> with Ok(vec![1.0])","defect":[{"file":"crates/aprender-serve/src/apr_transformer/generation.rs","line":52}]} +{"id":"P-4cd71741f944","class":"P","stratum":"S","split":"Test","diff_sha256":"e524bbfade51e4abe56825c03a610a50429ab306d495a0f24970999c67a2a185","bytes":1138,"approx_tokens":285,"source":"cargo-mutants 27.0.0 @ 80437647e: aprender-core :: crates/aprender-core/src/nn/init.rs:101:5: replace normal -> Tensor with Default::default()","defect":[{"file":"crates/aprender-core/src/nn/init.rs","line":101}]} +{"id":"P-4f6092d993a6","class":"P","stratum":"S","split":"Dev","diff_sha256":"a2f30ff056dfbe7f1fda384a2c13b63a196c908941ceac54693fc3e2eb1f4baf","bytes":1092,"approx_tokens":273,"source":"cargo-mutants 27.0.0 @ 80437647e: aprender-serve :: crates/aprender-serve/src/cuda/executor/weights.rs:237:9: replace CudaExecutor::cached_quantized_weight_bytes -> usize with 1","defect":[{"file":"crates/aprender-serve/src/cuda/executor/weights.rs","line":237}]} +{"id":"P-50edd5a3123c","class":"P","stratum":"S","split":"Test","diff_sha256":"7edb9adc685a5a261f3e1c4395f6a8f937f84b13e8b4950b275c64e62b0692e0","bytes":732,"approx_tokens":183,"source":"cargo-mutants 27.0.0 @ 80437647e: aprender-core :: crates/aprender-core/src/loading/cipher.rs:300:9: replace CipherCapabilities::for_suite -> Self with Default::default()","defect":[{"file":"crates/aprender-core/src/loading/cipher.rs","line":300}]} +{"id":"P-553007c9399d","class":"P","stratum":"S","split":"Test","diff_sha256":"82c536482e6de6fbdd1e0a56a0531fab17588b7548331b5e2274b19c90d01bf9","bytes":4622,"approx_tokens":1156,"source":"cargo-mutants 27.0.0 @ 80437647e: aprender-serve :: crates/aprender-serve/src/api/apr_q4k_scheduler.rs:84:5: replace spawn_apr_q4k_inference_thread -> Result, String> with Ok(Sender::new(Default::default()))","defect":[{"file":"crates/aprender-serve/src/api/apr_q4k_scheduler.rs","line":84}]} +{"id":"P-5a4122fffeae","class":"P","stratum":"S","split":"Test","diff_sha256":"48a6fa82013b64ad0eb92a45cee80aaa8581a3c989cc4cee33a1384009fb0cc2","bytes":848,"approx_tokens":212,"source":"cargo-mutants 27.0.0 @ 80437647e: aprender-serve :: crates/aprender-serve/src/quantize/iq4_xs.rs:48:20: replace * with / in dequantize_iq4_xs_block","defect":[{"file":"crates/aprender-serve/src/quantize/iq4_xs.rs","line":48}]} +{"id":"P-5c6c920fb6cf","class":"P","stratum":"S","split":"Dev","diff_sha256":"d481f32b5d79fb581212625945ed3901411b7898886eb86706bc2417764f990c","bytes":845,"approx_tokens":212,"source":"cargo-mutants 27.0.0 @ 80437647e: aprender-serve :: crates/aprender-serve/src/speculative.rs:390:12: delete ! in SpeculativeConfig::adapt_spec_length","defect":[{"file":"crates/aprender-serve/src/speculative.rs","line":390}]} +{"id":"P-62b51b8e169b","class":"P","stratum":"S","split":"Test","diff_sha256":"0de6fa7f2fe125f79fda2d85b9e5ca6a8c628d7becb73a778275624aa279fdc1","bytes":619,"approx_tokens":155,"source":"cargo-mutants 27.0.0 @ 80437647e: aprender-core :: crates/aprender-core/src/text/bpe/qwen2.rs:124:9: replace Qwen2BpeTokenizer::vocab_size -> usize with 0","defect":[{"file":"crates/aprender-core/src/text/bpe/qwen2.rs","line":124}]} +{"id":"P-69fb0db38747","class":"P","stratum":"S","split":"Test","diff_sha256":"3150a112054ddb2389c40c00ec980a9be406a9266f8d11db69658679d2051642","bytes":1050,"approx_tokens":263,"source":"cargo-mutants 27.0.0 @ 80437647e: aprender-serve :: crates/aprender-serve/src/gguf/qwen3_moe_load.rs:581:38: replace * with + in moe_ffn_forward_layer","defect":[{"file":"crates/aprender-serve/src/gguf/qwen3_moe_load.rs","line":581}]} +{"id":"P-6ec735d37bbd","class":"P","stratum":"S","split":"Test","diff_sha256":"1cf0c0b17710680c383d17a531328ab1d32a6cf24632a507c69d6037bc21b1a7","bytes":1022,"approx_tokens":256,"source":"cargo-mutants 27.0.0 @ 80437647e: aprender-core :: crates/aprender-core/src/model_selection/mod.rs:548:70: replace % with / in StratifiedKFold::split","defect":[{"file":"crates/aprender-core/src/model_selection/mod.rs","line":548}]} +{"id":"P-7a1b8a7554c4","class":"P","stratum":"S","split":"Test","diff_sha256":"8ab6d120527caa33beb77cdda306ec2b6cc9d1a68d70f58b86dbaf8934c9ca6c","bytes":668,"approx_tokens":167,"source":"cargo-mutants 27.0.0 @ 80437647e: aprender-serve :: crates/aprender-serve/src/brick/profiler.rs:81:37: replace / with % in OpStats::add","defect":[{"file":"crates/aprender-serve/src/brick/profiler.rs","line":81}]} +{"id":"P-7c3e4fac1fd7","class":"P","stratum":"S","split":"Test","diff_sha256":"eb9f8c994e3e62c7ea3405fef08cd0e06143e0f6cc0106f80951daf1f4cabd62","bytes":763,"approx_tokens":191,"source":"cargo-mutants 27.0.0 @ 80437647e: aprender-core :: crates/aprender-core/src/optim/damped_newton.rs:241:33: replace + with - in ::minimize","defect":[{"file":"crates/aprender-core/src/optim/damped_newton.rs","line":241}]} +{"id":"P-7fffab1fddb8","class":"P","stratum":"S","split":"Test","diff_sha256":"6341ad9d17c9fbf3152ee4d0d579898857ee3b478354c4caa8345ad59623e1ff","bytes":573,"approx_tokens":144,"source":"cargo-mutants 27.0.0 @ 80437647e: aprender-serve :: crates/aprender-serve/src/tui.rs:322:9: replace InferenceTui::throughput_history -> &VecDeque with Box::leak(Box::new(VecDeque::from_iter([-1.0])))","defect":[{"file":"crates/aprender-serve/src/tui.rs","line":322}]} +{"id":"P-8453b248d79b","class":"P","stratum":"S","split":"Test","diff_sha256":"120f798def12e412b4e7981f7b63487a046f13f083c2fe85bfa004df46c58e9c","bytes":1950,"approx_tokens":488,"source":"cargo-mutants 27.0.0 @ 80437647e: aprender-core :: crates/aprender-core/src/audio/noise/train.rs:202:9: replace NoiseTrainer::forward_with_cache -> (Vec, Vec, Vec) with (vec![-1.0], vec![-1.0], vec![-1.0])","defect":[{"file":"crates/aprender-core/src/audio/noise/train.rs","line":202}]} +{"id":"P-87bfc7a6a51e","class":"P","stratum":"S","split":"Dev","diff_sha256":"be504d16c373c1a0f3ac90a387773ff174745240c4313783a85f2c461254dfbd","bytes":829,"approx_tokens":208,"source":"cargo-mutants 27.0.0 @ 80437647e: aprender-core :: crates/aprender-core/src/synthetic/weak_supervision.rs:43:9: replace LabelVote::to_label -> Option with Some(0)","defect":[{"file":"crates/aprender-core/src/synthetic/weak_supervision.rs","line":43}]} +{"id":"P-881db37a1fcc","class":"P","stratum":"S","split":"Dev","diff_sha256":"5e7201fd03d578b5a4f4ee24df1b963481b2308bbbd32976da27fc4013f6a06a","bytes":776,"approx_tokens":194,"source":"cargo-mutants 27.0.0 @ 80437647e: aprender-contracts :: crates/aprender-contracts/src/explain_render.rs:315:65: replace == with != in write_binding_status","defect":[{"file":"crates/aprender-contracts/src/explain_render.rs","line":315}]} +{"id":"P-9e598f519091","class":"P","stratum":"S","split":"Test","diff_sha256":"11d89533d6556fae22cf6e612937588389d83886ce2b2025d9faae72d19e050c","bytes":696,"approx_tokens":174,"source":"cargo-mutants 27.0.0 @ 80437647e: aprender-serve :: crates/aprender-serve/src/audit.rs:153:9: replace AuditRecord::with_latency -> Self with Default::default()","defect":[{"file":"crates/aprender-serve/src/audit.rs","line":153}]} +{"id":"P-a6eea911d35c","class":"P","stratum":"S","split":"Dev","diff_sha256":"07fe0f0001b190728c7d3f237350878c2095310fef7ca97f942cd27a8918e627","bytes":720,"approx_tokens":180,"source":"cargo-mutants 27.0.0 @ 80437647e: aprender-serve :: crates/aprender-serve/src/gguf/inference/forward/forward_qwen35.rs:325:42: replace + with - in apply_partial_neox_rope","defect":[{"file":"crates/aprender-serve/src/gguf/inference/forward/forward_qwen35.rs","line":325}]} +{"id":"P-ab41daaacc1b","class":"P","stratum":"S","split":"Test","diff_sha256":"07845a5850d0ed44224cccb076905bd2f789ee30fbde06d17d3c2632aa081306","bytes":548,"approx_tokens":137,"source":"cargo-mutants 27.0.0 @ 80437647e: aprender-core :: crates/aprender-core/src/online/moe_construction.rs:365:29: replace / with % in compute_expert_load_balance","defect":[{"file":"crates/aprender-core/src/online/moe_construction.rs","line":365}]} +{"id":"P-b18dc9a4b9d9","class":"P","stratum":"S","split":"Test","diff_sha256":"04deb279b9e2be63caec0c27b3bef546bb8e28787ae0d23ab9faf779a4fbff5d","bytes":935,"approx_tokens":234,"source":"cargo-mutants 27.0.0 @ 80437647e: aprender-serve :: crates/aprender-serve/src/gguf/cuda/backend.rs:284:45: replace > with >= in CudaBackend::validate_dimensions","defect":[{"file":"crates/aprender-serve/src/gguf/cuda/backend.rs","line":284}]} +{"id":"P-c3a0a45804a7","class":"P","stratum":"S","split":"Test","diff_sha256":"846365c3aa4228036e2c9427b18e67bacfa4c8ce59e2684ee7a6baf3af9c7947","bytes":585,"approx_tokens":147,"source":"cargo-mutants 27.0.0 @ 80437647e: aprender-serve :: crates/aprender-serve/src/apr/mod.rs:130:9: replace AprFlags::is_lz4 -> bool with false","defect":[{"file":"crates/aprender-serve/src/apr/mod.rs","line":130}]} +{"id":"P-cc6f48e56473","class":"P","stratum":"S","split":"Test","diff_sha256":"c4d56bfc1ce39b4587159f37473dc0cbbdf6432e3e1e1ba6ab5878f01bf039f0","bytes":801,"approx_tokens":201,"source":"cargo-mutants 27.0.0 @ 80437647e: aprender-core :: crates/aprender-core/src/metaheuristics/ga.rs:153:30: replace + with * in GeneticAlgorithm::polynomial_mutate","defect":[{"file":"crates/aprender-core/src/metaheuristics/ga.rs","line":153}]} +{"id":"P-d5518b3d78a5","class":"P","stratum":"S","split":"Dev","diff_sha256":"d518bdabc75ec5fbae5217235badd3ed3aec7287753ea323d201ed34cafc9698","bytes":719,"approx_tokens":180,"source":"cargo-mutants 27.0.0 @ 80437647e: aprender-core :: crates/aprender-core/src/code/mpnn.rs:290:18: replace /= with *= in CodeMPNNLayer::aggregate_messages","defect":[{"file":"crates/aprender-core/src/code/mpnn.rs","line":290}]} +{"id":"P-d624d8619161","class":"P","stratum":"S","split":"Test","diff_sha256":"228655f98951077c9a600509a64ad444117461802443f79d223e595bdf8d6164","bytes":521,"approx_tokens":131,"source":"cargo-mutants 27.0.0 @ 80437647e: aprender-core :: crates/aprender-core/src/speech/asr/mod.rs:413:9: replace LanguageDetection::confidence -> f32 with -1.0","defect":[{"file":"crates/aprender-core/src/speech/asr/mod.rs","line":413}]} +{"id":"P-da4f5964dfb1","class":"P","stratum":"S","split":"Test","diff_sha256":"a1c8a0b962078c2d15f14ee08cca992f6b463b5812d8d25cbcccf104c5619714","bytes":619,"approx_tokens":155,"source":"cargo-mutants 27.0.0 @ 80437647e: aprender-core :: crates/aprender-core/src/embed/mod.rs:158:9: replace EmbeddedTestData::n_features -> usize with 1","defect":[{"file":"crates/aprender-core/src/embed/mod.rs","line":158}]} +{"id":"P-edb10eb7e5ed","class":"P","stratum":"S","split":"Test","diff_sha256":"315abb1e168f2b48e75a3c932a388772d2a8d5478c1fcf5132bd2b4dea0b3772","bytes":462,"approx_tokens":116,"source":"cargo-mutants 27.0.0 @ 80437647e: aprender-serve :: crates/aprender-serve/src/gpu/executor.rs:411:9: replace ::fmt -> std::fmt::Result with Ok(Default::default())","defect":[{"file":"crates/aprender-serve/src/gpu/executor.rs","line":411}]} +{"id":"P-edb6d2e34e8d","class":"P","stratum":"S","split":"Dev","diff_sha256":"10284ca7f62c5763ed76574b8811051fee8965062541e0efb03ba94ae60915e9","bytes":794,"approx_tokens":199,"source":"cargo-mutants 27.0.0 @ 80437647e: aprender-core :: crates/aprender-core/src/metrics/classification.rs:219:30: replace == with != in recall","defect":[{"file":"crates/aprender-core/src/metrics/classification.rs","line":219}]} +{"id":"P-f262289a316b","class":"P","stratum":"S","split":"Dev","diff_sha256":"c46adf2c336ddc6fc1c96440dad44ab5e7334ca55f8643ffccc85b7b9407dcd3","bytes":1522,"approx_tokens":381,"source":"cargo-mutants 27.0.0 @ 80437647e: aprender-core :: crates/aprender-core/src/code/ast.rs:63:9: replace ::fmt -> fmt::Result with Ok(Default::default())","defect":[{"file":"crates/aprender-core/src/code/ast.rs","line":63}]} +{"id":"P-f3abd4438999","class":"P","stratum":"S","split":"Test","diff_sha256":"f91ef43c36cb1a1275fe24210314a6d1d365b8942fe1169af9e36fe1e1224ad8","bytes":711,"approx_tokens":178,"source":"cargo-mutants 27.0.0 @ 80437647e: aprender-core :: crates/aprender-core/src/online/distillation_advanced.rs:452:51: replace / with % in solve_linear_system","defect":[{"file":"crates/aprender-core/src/online/distillation_advanced.rs","line":452}]} +{"id":"P-f72984d91c0d","class":"P","stratum":"S","split":"Test","diff_sha256":"74babb9ba8a27a89df0a0092691c08e076f50ab0792f273365eca699d5766825","bytes":6397,"approx_tokens":1600,"source":"cargo-mutants 27.0.0 @ 80437647e: aprender-serve :: crates/aprender-serve/src/gpu/adapters/apr_q4k.rs:1145:5: replace forward_token_apr_q4k -> Result> with Ok(vec![1.0])","defect":[{"file":"crates/aprender-serve/src/gpu/adapters/apr_q4k.rs","line":1145}]} +{"id":"P-fe943fab7a8d","class":"P","stratum":"S","split":"Dev","diff_sha256":"89351f9cdeeb19f15f577e66c0f9e9e57e1180cface2d80af61ab5ca4905dc3a","bytes":658,"approx_tokens":165,"source":"cargo-mutants 27.0.0 @ 80437647e: aprender-serve :: crates/aprender-serve/src/quantize/types.rs:124:20: replace < with > in Q8_0Block::relative_error","defect":[{"file":"crates/aprender-serve/src/quantize/types.rs","line":124}]} +{"id":"R-pr1411","class":"R","stratum":"M","split":"Test","diff_sha256":"7e10fa511b0956191e864560ef66a623fd0fa78dd03c06c39079a69d4c61959a","bytes":11168,"approx_tokens":2792,"source":"pr#1411 9bc0d4ec4d1be22eb8f1f9a39bb62b2892b5d27a reversed; fixes #1410","defect":[{"file":"crates/apr-cli/src/commands/pull_dataset.rs","line":16},{"file":"crates/apr-cli/src/commands/pull_dataset.rs","line":18},{"file":"crates/apr-cli/src/commands/pull_dataset.rs","line":19},{"file":"crates/apr-cli/src/commands/pull_dataset.rs","line":20},{"file":"crates/apr-cli/src/commands/pull_dataset.rs","line":32},{"file":"crates/apr-cli/src/commands/pull_dataset.rs","line":41},{"file":"crates/apr-cli/src/commands/pull_dataset.rs","line":43},{"file":"crates/apr-cli/src/commands/pull_dataset.rs","line":58},{"file":"crates/apr-cli/src/commands/pull_dataset.rs","line":93},{"file":"crates/apr-cli/src/commands/pull_dataset.rs","line":95},{"file":"crates/apr-cli/src/commands/pull_dataset.rs","line":96},{"file":"crates/apr-cli/src/commands/pull_dataset.rs","line":97},{"file":"crates/apr-cli/src/commands/pull_dataset.rs","line":98},{"file":"crates/apr-cli/src/commands/pull_dataset.rs","line":99},{"file":"crates/apr-cli/src/commands/pull_dataset.rs","line":100},{"file":"crates/apr-cli/src/commands/pull_dataset.rs","line":101},{"file":"crates/apr-cli/src/commands/pull_dataset.rs","line":102},{"file":"crates/apr-cli/src/commands/pull_dataset.rs","line":103},{"file":"crates/apr-cli/src/commands/pull_dataset.rs","line":104},{"file":"crates/apr-cli/src/commands/pull_dataset.rs","line":105},{"file":"crates/apr-cli/src/commands/pull_dataset.rs","line":106},{"file":"crates/apr-cli/src/commands/pull_dataset.rs","line":107},{"file":"crates/apr-cli/src/commands/pull_dataset.rs","line":109},{"file":"crates/apr-cli/src/commands/pull_dataset.rs","line":110},{"file":"crates/apr-cli/src/commands/pull_dataset.rs","line":111},{"file":"crates/apr-cli/src/commands/pull_dataset.rs","line":112},{"file":"crates/apr-cli/src/commands/pull_dataset.rs","line":113},{"file":"crates/apr-cli/src/commands/pull_dataset.rs","line":114},{"file":"crates/apr-cli/src/commands/pull_dataset.rs","line":196},{"file":"crates/apr-cli/src/dispatch.rs","line":656},{"file":"crates/apr-cli/src/dispatch.rs","line":657},{"file":"crates/apr-cli/src/dispatch.rs","line":658}]} +{"id":"R-pr1656","class":"R","stratum":"S","split":"Dev","diff_sha256":"fbc8891d453138483d8802e116e7b30678c642b70dbfbec10932a897b0adb6c2","bytes":3902,"approx_tokens":976,"source":"pr#1656 8ca02aaa16ddc586a2e1b9c20c8551449b719072 reversed; fixes #1623","defect":[{"file":"crates/aprender-core/src/format/gguf/dequant.rs","line":180},{"file":"crates/aprender-core/src/format/gguf/dequant.rs","line":181},{"file":"crates/aprender-core/src/format/gguf/dequant.rs","line":182},{"file":"crates/aprender-core/src/format/gguf/dequant.rs","line":183},{"file":"crates/aprender-core/src/format/gguf/dequant.rs","line":187},{"file":"crates/aprender-core/src/format/gguf/dequant.rs","line":188},{"file":"crates/aprender-core/src/format/gguf/dequant.rs","line":189},{"file":"crates/aprender-core/src/format/gguf/dequant.rs","line":191},{"file":"crates/aprender-core/src/format/gguf/dequant.rs","line":195},{"file":"crates/aprender-core/src/format/gguf/dequant.rs","line":196},{"file":"crates/aprender-core/src/format/gguf/dequant.rs","line":248},{"file":"crates/aprender-core/src/format/gguf/dequant.rs","line":249},{"file":"crates/aprender-core/src/format/gguf/dequant.rs","line":250},{"file":"crates/aprender-core/src/format/gguf/dequant.rs","line":251},{"file":"crates/aprender-core/src/format/gguf/dequant.rs","line":255},{"file":"crates/aprender-core/src/format/gguf/dequant.rs","line":256},{"file":"crates/aprender-core/src/format/gguf/dequant.rs","line":257},{"file":"crates/aprender-core/src/format/gguf/dequant.rs","line":259},{"file":"crates/aprender-core/src/format/gguf/dequant.rs","line":260},{"file":"crates/aprender-core/src/format/gguf/dequant.rs","line":264},{"file":"crates/aprender-core/src/format/gguf/dequant.rs","line":265}]} +{"id":"R-pr1663","class":"R","stratum":"S","split":"Dev","diff_sha256":"3be0adf4d7e15732274ad31f6d083ee120e2bebd3aee8adbe0550bbbaf538e30","bytes":2190,"approx_tokens":548,"source":"pr#1663 caccb68510684e046859bb7b1eff5a1869320729 reversed; fixes #471","defect":[{"file":"crates/apr-cli/src/commands/serve/handlers.rs","line":1011}]} +{"id":"R-pr1714","class":"R","stratum":"S","split":"Test","diff_sha256":"b5feb051c449a939190491b37058a534d03f3522b8484cd1bc2eff342e7e7fc1","bytes":4164,"approx_tokens":1041,"source":"pr#1714 072f55d5204dc13e519780bf02ee78e24f28040e reversed; fixes #1712","defect":[{"file":"crates/aprender-orchestrate/src/agent/driver/apr_serve.rs","line":90},{"file":"crates/aprender-orchestrate/src/agent/driver/apr_serve.rs","line":91},{"file":"crates/aprender-orchestrate/src/agent/driver/apr_serve.rs","line":92},{"file":"crates/aprender-orchestrate/src/agent/driver/apr_serve.rs","line":93},{"file":"crates/aprender-orchestrate/src/agent/driver/apr_serve.rs","line":94},{"file":"crates/aprender-orchestrate/src/agent/driver/apr_serve.rs","line":95},{"file":"crates/aprender-orchestrate/src/agent/driver/apr_serve.rs","line":96},{"file":"crates/aprender-orchestrate/src/agent/driver/apr_serve.rs","line":97},{"file":"crates/aprender-orchestrate/src/agent/driver/apr_serve.rs","line":98},{"file":"crates/aprender-orchestrate/src/agent/driver/apr_serve.rs","line":99},{"file":"crates/aprender-orchestrate/src/agent/driver/apr_serve.rs","line":100},{"file":"crates/aprender-orchestrate/src/agent/driver/apr_serve.rs","line":101},{"file":"crates/aprender-orchestrate/src/agent/driver/apr_serve.rs","line":102},{"file":"crates/aprender-orchestrate/src/agent/driver/apr_serve.rs","line":103},{"file":"crates/aprender-orchestrate/src/agent/driver/apr_serve.rs","line":104},{"file":"crates/aprender-orchestrate/src/agent/driver/apr_serve.rs","line":105},{"file":"crates/aprender-orchestrate/src/agent/driver/apr_serve.rs","line":106},{"file":"crates/aprender-orchestrate/src/agent/driver/apr_serve.rs","line":107},{"file":"crates/aprender-orchestrate/src/agent/driver/apr_serve.rs","line":108},{"file":"crates/aprender-orchestrate/src/agent/driver/apr_serve.rs","line":109},{"file":"crates/aprender-orchestrate/src/agent/driver/apr_serve.rs","line":345}]} +{"id":"R-pr1782","class":"R","stratum":"S","split":"Test","diff_sha256":"5b778f76ef9b15da17d442e3ac2f190ec8409acaf86d68f4a5554669fe04da8f","bytes":5569,"approx_tokens":1393,"source":"pr#1782 ff9d0c99675d74eeb765c3d537b41fe99fba6a53 reversed; fixes #1781","defect":[{"file":"crates/aprender-orchestrate/src/agent/driver/apr_serve.rs","line":33},{"file":"crates/aprender-orchestrate/src/agent/driver/apr_serve.rs","line":120},{"file":"crates/aprender-orchestrate/src/agent/driver/apr_serve.rs","line":133},{"file":"crates/aprender-orchestrate/src/agent/driver/apr_serve.rs","line":143},{"file":"crates/aprender-orchestrate/src/agent/driver/apr_serve.rs","line":148},{"file":"crates/aprender-orchestrate/src/agent/driver/apr_serve.rs","line":183},{"file":"crates/aprender-orchestrate/src/agent/driver/apr_serve.rs","line":267}]} +{"id":"R-pr1806","class":"R","stratum":"M","split":"Test","diff_sha256":"115e56567f24ae649f56e126a7cadcf0b9ca5b33e976bac2f414f1472a1735c4","bytes":15381,"approx_tokens":3846,"source":"pr#1806 9c974524ff553b9ad6e84d96f48789916c66dd40 reversed; fixes #1789","defect":[{"file":"crates/aprender-serve/src/api/cuda_chat_backend.rs","line":597},{"file":"crates/aprender-serve/src/api/cuda_chat_backend.rs","line":635},{"file":"crates/aprender-serve/src/api/mod.rs","line":169},{"file":"crates/aprender-serve/src/api/mod_app_state_gpu.rs","line":62},{"file":"crates/aprender-serve/src/api/mod_app_state_gpu.rs","line":115},{"file":"crates/aprender-serve/src/api/mod_app_state_gpu.rs","line":176},{"file":"crates/aprender-serve/src/api/mod_app_state_gpu.rs","line":240},{"file":"crates/aprender-serve/src/api/mod_app_state_gpu.rs","line":293},{"file":"crates/aprender-serve/src/api/mod_app_state_gpu.rs","line":351},{"file":"crates/aprender-serve/src/api/mod_app_state_gpu.rs","line":571},{"file":"crates/aprender-serve/src/api/mod_app_state_gpu.rs","line":616},{"file":"crates/aprender-serve/src/api/mod_app_state_gpu.rs","line":642},{"file":"crates/aprender-serve/src/api/mod_app_state_new.rs","line":43},{"file":"crates/aprender-serve/src/api/mod_app_state_new.rs","line":101},{"file":"crates/aprender-serve/src/api/mod_app_state_new.rs","line":200},{"file":"crates/aprender-serve/src/api/mod_app_state_new.rs","line":271},{"file":"crates/aprender-serve/src/api/mod_app_state_new.rs","line":324},{"file":"crates/aprender-serve/src/api/mod_app_state_new.rs","line":382},{"file":"crates/aprender-serve/src/api/mod_app_state_new.rs","line":435},{"file":"crates/aprender-serve/src/api/mod_app_state_new.rs","line":501},{"file":"crates/aprender-serve/src/cli/mod_server_commands.rs","line":133}]} +{"id":"R-pr1808","class":"R","stratum":"S","split":"Test","diff_sha256":"4804683d7ff88ca019d882458ebdc858b8adb9795e34fe225d5de08122529f1d","bytes":3520,"approx_tokens":880,"source":"pr#1808 9d9847e00cd64a9116aa6ce5b45cb61b5fa9df32 reversed; fixes #2","defect":[{"file":"crates/apr-cli/src/commands/distill.rs","line":609},{"file":"crates/apr-cli/src/commands/distill.rs","line":675},{"file":"crates/apr-cli/src/commands/distill.rs","line":708}]} +{"id":"R-pr1867","class":"R","stratum":"S","split":"Test","diff_sha256":"49078e8fef4d0f7615a627297e1b0d5ae898a249974021ce25b8c550778ef589","bytes":3644,"approx_tokens":911,"source":"pr#1867 f8a97a9432960c5af40bb0aa7b104942da6d9bb0 reversed; fixes #1862","defect":[{"file":"crates/apr-cli/build.rs","line":1},{"file":"crates/apr-cli/build.rs","line":4},{"file":"crates/apr-cli/build.rs","line":11},{"file":"crates/apr-cli/build.rs","line":12},{"file":"crates/apr-cli/build.rs","line":13},{"file":"crates/apr-cli/build.rs","line":14},{"file":"crates/apr-cli/build.rs","line":15},{"file":"crates/apr-cli/build.rs","line":16},{"file":"crates/apr-cli/build.rs","line":17},{"file":"crates/apr-cli/build.rs","line":31},{"file":"crates/apr-cli/build.rs","line":32},{"file":"crates/apr-cli/build.rs","line":33},{"file":"crates/apr-cli/build.rs","line":34},{"file":"crates/apr-cli/build.rs","line":35},{"file":"crates/apr-cli/build.rs","line":36},{"file":"crates/apr-cli/build.rs","line":37},{"file":"crates/apr-cli/build.rs","line":38},{"file":"crates/apr-cli/build.rs","line":39},{"file":"crates/apr-cli/build.rs","line":40},{"file":"crates/apr-cli/build.rs","line":41}]} +{"id":"R-pr1868","class":"R","stratum":"M","split":"Dev","diff_sha256":"dd2dd38ac5d0be8581e31ca384d06b70085b34318cfc787ac8a4ec90329817fd","bytes":16180,"approx_tokens":4045,"source":"pr#1868 470617aa3dbb0e7fbee613f0f42225c74989b547 reversed; fixes #1865","defect":[{"file":"crates/aprender-core/src/format/converter/export_tests_arch_metadata.rs","line":3},{"file":"crates/aprender-core/src/format/converter/export_tests_arch_metadata.rs","line":5},{"file":"crates/aprender-core/src/format/converter/export_tests_arch_metadata.rs","line":6},{"file":"crates/aprender-core/src/format/converter/export_tests_arch_metadata.rs","line":16},{"file":"crates/aprender-core/src/format/converter/export_tests_arch_metadata.rs","line":19},{"file":"crates/aprender-core/src/format/converter/export_tests_arch_metadata.rs","line":21},{"file":"crates/aprender-core/src/format/converter/export_tests_arch_metadata.rs","line":22},{"file":"crates/aprender-core/src/format/converter/export_tests_arch_metadata.rs","line":32},{"file":"crates/aprender-core/src/format/converter/export_tests_arch_metadata.rs","line":35},{"file":"crates/aprender-core/src/format/converter/export_tests_arch_metadata.rs","line":37},{"file":"crates/aprender-core/src/format/converter/export_tests_arch_metadata.rs","line":38},{"file":"crates/aprender-core/src/format/converter/export_tests_arch_metadata.rs","line":48},{"file":"crates/aprender-core/src/format/converter/export_tests_arch_metadata.rs","line":66},{"file":"crates/aprender-core/src/format/converter/export_tests_arch_metadata.rs","line":92},{"file":"crates/aprender-core/src/format/converter/export_tests_arch_metadata.rs","line":117},{"file":"crates/aprender-core/src/format/converter/export_tests_arch_metadata.rs","line":404},{"file":"crates/aprender-core/src/format/converter/export_tests_detect_quant.rs","line":335},{"file":"crates/aprender-core/src/format/converter/export_tests_detect_quant.rs","line":356},{"file":"crates/aprender-core/src/format/converter/export_tests_e2e_gguf.rs","line":335},{"file":"crates/aprender-core/src/format/converter/metadata.rs","line":2},{"file":"crates/aprender-core/src/format/converter/metadata.rs","line":5},{"file":"crates/aprender-core/src/format/converter/metadata.rs","line":9},{"file":"crates/aprender-core/src/format/converter/metadata.rs","line":10},{"file":"crates/aprender-core/src/format/converter/metadata.rs","line":11},{"file":"crates/aprender-core/src/format/converter/metadata.rs","line":12},{"file":"crates/aprender-core/src/format/converter/metadata.rs","line":13},{"file":"crates/aprender-core/src/format/converter/metadata.rs","line":14},{"file":"crates/aprender-core/src/format/converter/metadata.rs","line":15},{"file":"crates/aprender-core/src/format/converter/metadata.rs","line":16},{"file":"crates/aprender-core/src/format/converter/metadata.rs","line":17},{"file":"crates/aprender-core/src/format/converter/metadata.rs","line":18},{"file":"crates/aprender-core/src/format/converter/metadata.rs","line":19},{"file":"crates/aprender-core/src/format/converter/metadata.rs","line":20},{"file":"crates/aprender-core/src/format/converter/metadata.rs","line":22},{"file":"crates/aprender-core/src/format/converter/metadata.rs","line":23},{"file":"crates/aprender-core/src/format/converter/metadata.rs","line":24},{"file":"crates/aprender-core/src/format/converter/metadata.rs","line":25},{"file":"crates/aprender-core/src/format/converter/metadata.rs","line":26},{"file":"crates/aprender-core/src/format/converter/metadata.rs","line":27},{"file":"crates/aprender-core/src/format/converter/metadata.rs","line":110},{"file":"crates/aprender-core/src/format/converter/metadata.rs","line":378},{"file":"crates/aprender-core/src/format/converter/metadata.rs","line":381},{"file":"crates/aprender-core/src/format/converter/metadata.rs","line":382},{"file":"crates/aprender-core/src/format/converter/metadata.rs","line":383},{"file":"crates/aprender-core/src/format/converter/metadata.rs","line":384},{"file":"crates/aprender-core/src/format/converter/metadata.rs","line":385},{"file":"crates/aprender-core/src/format/converter/metadata.rs","line":386},{"file":"crates/aprender-core/src/format/converter/metadata.rs","line":387},{"file":"crates/aprender-core/src/format/converter/metadata.rs","line":389},{"file":"crates/aprender-core/src/format/converter/metadata.rs","line":390},{"file":"crates/aprender-core/src/format/converter/metadata.rs","line":391},{"file":"crates/aprender-core/src/format/converter/metadata.rs","line":394}]} +{"id":"R-pr1870","class":"R","stratum":"S","split":"Test","diff_sha256":"1e1aefec535ed6a62176ae9f15d538a764a4e1983cb4c3a7c0a768d225721f50","bytes":7050,"approx_tokens":1763,"source":"pr#1870 41b1a9908d627ed73ea1c389cc453227ea6c75ca reversed; fixes #1866","defect":[{"file":"crates/apr-cli/src/commands/validate.rs","line":96},{"file":"crates/apr-cli/src/commands/validate.rs","line":97},{"file":"crates/apr-cli/src/commands/validate.rs","line":98},{"file":"crates/apr-cli/src/commands/validate.rs","line":99},{"file":"crates/apr-cli/src/commands/validate.rs","line":100},{"file":"crates/aprender-core/src/format/validation.rs","line":152},{"file":"crates/aprender-core/src/format/validation_tests_report.rs","line":491}]} +{"id":"R-pr2092","class":"R","stratum":"S","split":"Test","diff_sha256":"157b1b19f40d99af3fc7d80beb5e99998aa9697795dc08b7d91145ce5179b0f8","bytes":7701,"approx_tokens":1926,"source":"pr#2092 634bcc3c1abb80985b0c6b714424c0f0dcb85bed reversed; fixes #1","defect":[{"file":"crates/aprender-train/src/finetune/instruct_pipeline/training.rs","line":68},{"file":"crates/aprender-train/src/finetune/instruct_pipeline/training.rs","line":458},{"file":"crates/aprender-train/src/transformer/attention.rs","line":113},{"file":"crates/aprender-train/src/transformer/attention.rs","line":1068}]} +{"id":"R-pr2249","class":"R","stratum":"M","split":"Dev","diff_sha256":"b314656d1358d5db27a0f8d90879a516a8aa53457091db9e63ed78eab624886c","bytes":15458,"approx_tokens":3865,"source":"pr#2249 05045a135fa089aa73149b0673c8b425a3fb747d reversed; fixes #1","defect":[{"file":"crates/aprender-train/src/autograd/cuda_forward/cache.rs","line":237},{"file":"crates/aprender-train/src/autograd/cuda_forward/normalization.rs","line":10},{"file":"crates/aprender-train/src/autograd/cuda_forward/normalization.rs","line":11},{"file":"crates/aprender-train/src/autograd/cuda_forward/normalization.rs","line":456},{"file":"crates/aprender-train/src/autograd/cuda_forward/normalization.rs","line":463},{"file":"crates/aprender-train/src/autograd/cuda_forward/normalization.rs","line":467},{"file":"crates/aprender-train/src/autograd/cuda_forward/normalization.rs","line":473},{"file":"crates/aprender-train/src/autograd/cuda_forward/normalization.rs","line":474},{"file":"crates/aprender-train/src/autograd/cuda_forward/normalization.rs","line":475},{"file":"crates/aprender-train/src/autograd/cuda_forward/normalization.rs","line":482},{"file":"crates/aprender-train/src/autograd/cuda_forward/normalization.rs","line":489},{"file":"crates/aprender-train/src/autograd/cuda_forward/normalization.rs","line":490},{"file":"crates/aprender-train/src/autograd/cuda_forward/normalization.rs","line":491},{"file":"crates/aprender-train/src/autograd/cuda_forward/normalization.rs","line":492},{"file":"crates/aprender-train/src/autograd/cuda_forward/normalization.rs","line":493},{"file":"crates/aprender-train/src/autograd/cuda_forward/normalization.rs","line":494},{"file":"crates/aprender-train/src/autograd/cuda_forward/normalization.rs","line":495},{"file":"crates/aprender-train/src/autograd/cuda_forward/normalization.rs","line":496},{"file":"crates/aprender-train/src/autograd/cuda_forward/normalization.rs","line":497},{"file":"crates/aprender-train/src/autograd/cuda_forward/normalization.rs","line":498},{"file":"crates/aprender-train/src/autograd/cuda_forward/normalization.rs","line":499},{"file":"crates/aprender-train/src/autograd/cuda_forward/normalization.rs","line":500},{"file":"crates/aprender-train/src/autograd/cuda_forward/normalization.rs","line":501},{"file":"crates/aprender-train/src/autograd/cuda_forward/normalization.rs","line":502},{"file":"crates/aprender-train/src/autograd/cuda_forward/normalization.rs","line":503},{"file":"crates/aprender-train/src/autograd/cuda_forward/normalization.rs","line":506},{"file":"crates/aprender-train/src/autograd/cuda_forward/normalization.rs","line":507},{"file":"crates/aprender-train/src/autograd/cuda_forward/normalization.rs","line":511},{"file":"crates/aprender-train/src/autograd/cuda_forward/normalization.rs","line":512},{"file":"crates/aprender-train/src/autograd/cuda_forward/normalization.rs","line":615},{"file":"crates/aprender-train/src/transformer/cuda_block.rs","line":3270}]} +{"id":"R-pr2321","class":"R","stratum":"S","split":"Test","diff_sha256":"13812c7d8dd7e8352d0f24081fea89a771d1387febb09604dbcf43afcc82ea36","bytes":2959,"approx_tokens":740,"source":"pr#2321 ea3e5335c6a78674394b4cbb4310b8cde58ab6e3 reversed; fixes #2320","defect":[{"file":"crates/apr-cli/src/dispatch.rs","line":92}]} +{"id":"R-pr2365","class":"R","stratum":"S","split":"Test","diff_sha256":"a30a4b965b369d1be55d808c80db3e1714a42dfa0e47226d9afa2dd88f77cc10","bytes":3683,"approx_tokens":921,"source":"pr#2365 3ed7c0d2198a139e1b3e02c2c4509fa4fc8c36a0 reversed; fixes #2406","defect":[{"file":"/dev/null","line":0}]} +{"id":"R-pr2368","class":"R","stratum":"S","split":"Test","diff_sha256":"0405b05a7de60d1f738c7991fdcafbf4f22f965ac3d006133fcfc0cdccc51d58","bytes":7328,"approx_tokens":1832,"source":"pr#2368 8cc3aafeb9ae05443d581f1cf403bd0e3bad7d40 reversed; fixes #2386","defect":[{"file":"crates/apr-cli/src/commands/explain.rs","line":51},{"file":"crates/apr-cli/src/commands/explain.rs","line":699},{"file":"crates/apr-cli/src/commands/explain.rs","line":702},{"file":"crates/apr-cli/src/commands/explain.rs","line":704},{"file":"crates/apr-cli/src/commands/explain.rs","line":705},{"file":"crates/apr-cli/src/commands/explain.rs","line":706},{"file":"crates/apr-cli/src/commands/explain.rs","line":715},{"file":"crates/apr-cli/src/commands/explain.rs","line":722},{"file":"crates/apr-cli/src/commands/explain.rs","line":723},{"file":"crates/apr-cli/src/commands/explain.rs","line":731},{"file":"crates/apr-cli/src/commands/explain.rs","line":734},{"file":"crates/apr-cli/src/commands/explain.rs","line":740},{"file":"crates/apr-cli/src/commands/explain.rs","line":741},{"file":"crates/apr-cli/src/commands/explain.rs","line":746},{"file":"crates/apr-cli/src/commands/explain.rs","line":876},{"file":"crates/apr-cli/src/commands/explain.rs","line":879},{"file":"crates/apr-cli/src/commands/explain.rs","line":885},{"file":"crates/apr-cli/src/commands/explain.rs","line":956}]} +{"id":"R-pr2415","class":"R","stratum":"M","split":"Test","diff_sha256":"a309341f6d6514af7f480f30574f7e86ffbc51f1e0b9d61264c4f77beee09811","bytes":16013,"approx_tokens":4004,"source":"pr#2415 2a97d13bc4675dc7c4f21fea7a85cfb5519eb15b reversed; fixes #2388","defect":[{"file":"crates/aprender-mcp/src/tools/serve.rs","line":14},{"file":"crates/aprender-mcp/src/tools/serve.rs","line":15},{"file":"crates/aprender-mcp/src/tools/serve.rs","line":28},{"file":"crates/aprender-mcp/src/tools/serve.rs","line":73},{"file":"crates/aprender-mcp/src/tools/serve.rs","line":74},{"file":"crates/aprender-mcp/src/tools/serve.rs","line":75},{"file":"crates/aprender-mcp/src/tools/serve.rs","line":76},{"file":"crates/aprender-mcp/src/tools/serve.rs","line":77},{"file":"crates/aprender-mcp/src/tools/serve.rs","line":79},{"file":"crates/aprender-mcp/src/tools/serve.rs","line":80},{"file":"crates/aprender-mcp/src/tools/serve.rs","line":81},{"file":"crates/aprender-mcp/src/tools/serve.rs","line":82},{"file":"crates/aprender-mcp/src/tools/serve.rs","line":85},{"file":"crates/aprender-mcp/src/tools/serve.rs","line":90},{"file":"crates/aprender-mcp/src/tools/serve.rs","line":91},{"file":"crates/aprender-mcp/src/tools/serve.rs","line":92},{"file":"crates/aprender-mcp/src/tools/serve.rs","line":97},{"file":"crates/aprender-mcp/src/tools/serve.rs","line":101},{"file":"crates/aprender-mcp/src/tools/serve.rs","line":172}]} +{"id":"R-pr2416","class":"R","stratum":"L","split":"Test","diff_sha256":"bbcf4ef31aa3cced18e102156f8f861b5052ee15902adcc4640324847adaef59","bytes":33319,"approx_tokens":8330,"source":"pr#2416 a45ec84f8a6155e7368da1dfbf5c08a225c46e3c reversed; fixes #2379","defect":[{"file":"crates/apr-cli/src/commands/chat.rs","line":126},{"file":"crates/apr-cli/src/commands/offline.rs","line":59},{"file":"crates/apr-cli/src/commands/offline.rs","line":129},{"file":"crates/apr-cli/src/commands/pull.rs","line":72},{"file":"crates/apr-cli/src/commands/pull.rs","line":240},{"file":"crates/apr-cli/src/commands/pull_dataset.rs","line":136},{"file":"crates/apr-cli/src/commands/pull_extract_shard.rs","line":73},{"file":"crates/apr-cli/src/commands/pull_extract_shard.rs","line":75},{"file":"crates/apr-cli/src/commands/pull_extract_shard.rs","line":79},{"file":"crates/apr-cli/src/commands/pull_extract_shard.rs","line":105},{"file":"crates/apr-cli/src/commands/pull_extract_shard.rs","line":127},{"file":"crates/apr-cli/src/commands/pull_gh355_356_357.rs","line":84},{"file":"crates/apr-cli/src/commands/pull_gh355_356_357.rs","line":85},{"file":"crates/apr-cli/src/commands/pull_gh355_356_357.rs","line":99},{"file":"crates/apr-cli/src/commands/pull_remove_resolve_model.rs","line":128},{"file":"crates/apr-cli/src/commands/pull_remove_resolve_model.rs","line":301},{"file":"crates/apr-cli/src/commands/pull_remove_resolve_model.rs","line":378},{"file":"crates/apr-cli/src/commands/run.rs","line":294},{"file":"crates/apr-cli/src/commands/run.rs","line":306},{"file":"crates/apr-cli/src/commands/showcase/pipeline.rs","line":29},{"file":"crates/apr-cli/src/commands/showcase/pipeline.rs","line":491},{"file":"crates/apr-cli/src/dispatch.rs","line":819},{"file":"crates/apr-cli/src/dispatch_run.rs","line":335}]} +{"id":"R-pr2420","class":"R","stratum":"L","split":"Test","diff_sha256":"ebec8a3614796e4d5bc48e28b225910f40ee717e1a80bdfd11ccf87c5d01bb2d","bytes":36923,"approx_tokens":9231,"source":"pr#2420 fe7e37a94feab01589602a576c5f02f30f5f446e reversed; fixes #2382","defect":[{"file":"/dev/null","line":0},{"file":"crates/apr-cli/src/commands/fingerprints.rs","line":204},{"file":"crates/apr-cli/src/commands/fingerprints.rs","line":207},{"file":"crates/apr-cli/src/commands/fingerprints.rs","line":208},{"file":"crates/apr-cli/src/commands/fingerprints.rs","line":209},{"file":"crates/apr-cli/src/commands/fingerprints.rs","line":210},{"file":"crates/apr-cli/src/commands/fingerprints.rs","line":211},{"file":"crates/apr-cli/src/commands/fingerprints.rs","line":220},{"file":"crates/apr-cli/src/commands/fingerprints.rs","line":221},{"file":"crates/apr-cli/src/commands/fingerprints.rs","line":222},{"file":"crates/apr-cli/src/commands/fingerprints.rs","line":254},{"file":"crates/apr-cli/src/commands/fingerprints.rs","line":265},{"file":"crates/apr-cli/src/commands/fingerprints.rs","line":286},{"file":"crates/apr-cli/src/commands/fingerprints.rs","line":287},{"file":"crates/apr-cli/src/commands/inference.rs","line":41},{"file":"crates/apr-cli/src/commands/inference.rs","line":59},{"file":"crates/apr-cli/src/commands/inference.rs","line":66},{"file":"crates/apr-cli/src/commands/inference.rs","line":102},{"file":"crates/apr-cli/src/commands/inference.rs","line":120},{"file":"crates/apr-cli/src/commands/inference.rs","line":129},{"file":"crates/apr-cli/src/commands/inference.rs","line":130},{"file":"crates/apr-cli/src/commands/inference.rs","line":131},{"file":"crates/apr-cli/src/commands/inference.rs","line":132},{"file":"crates/apr-cli/src/commands/inference.rs","line":157},{"file":"crates/apr-cli/src/commands/inference_result.rs","line":392},{"file":"crates/apr-cli/src/commands/inference_result.rs","line":393},{"file":"crates/apr-cli/src/commands/inference_result.rs","line":394},{"file":"crates/apr-cli/src/commands/inference_result.rs","line":395},{"file":"crates/apr-cli/src/commands/inference_result.rs","line":396},{"file":"crates/apr-cli/src/commands/inference_result.rs","line":397},{"file":"crates/apr-cli/src/commands/inference_result.rs","line":398},{"file":"crates/apr-cli/src/commands/rosetta_08.rs","line":23},{"file":"crates/apr-cli/src/commands/rosetta_compute_tensor_get.rs","line":410},{"file":"crates/apr-cli/src/commands/rosetta_compute_tensor_get.rs","line":426},{"file":"crates/apr-cli/src/commands/rosetta_compute_tensor_get.rs","line":434},{"file":"crates/apr-cli/src/commands/rosetta_diff_tensor.rs","line":297},{"file":"crates/apr-cli/src/commands/rosetta_diff_tensor.rs","line":317},{"file":"crates/apr-cli/src/commands/rosetta_diff_tensor.rs","line":357},{"file":"crates/apr-cli/src/commands/rosetta_diff_tensor.rs","line":364},{"file":"crates/apr-cli/src/commands/rosetta_diff_tensor.rs","line":365},{"file":"crates/apr-cli/src/commands/rosetta_diff_tensor.rs","line":366},{"file":"crates/apr-cli/src/commands/rosetta_diff_tensor.rs","line":367},{"file":"crates/apr-cli/src/commands/rosetta_diff_tensor.rs","line":368},{"file":"crates/apr-cli/src/commands/rosetta_diff_tensor.rs","line":369},{"file":"crates/apr-cli/src/commands/rosetta_diff_tensor.rs","line":370},{"file":"crates/apr-cli/src/commands/rosetta_diff_tensor.rs","line":371},{"file":"crates/apr-cli/src/commands/rosetta_diff_tensor.rs","line":372},{"file":"crates/apr-cli/src/commands/rosetta_diff_tensor.rs","line":373},{"file":"crates/apr-cli/src/commands/rosetta_diff_tensor.rs","line":374},{"file":"crates/apr-cli/src/commands/rosetta_diff_tensor.rs","line":375},{"file":"crates/apr-cli/src/commands/rosetta_diff_tensor.rs","line":376},{"file":"crates/apr-cli/src/commands/rosetta_diff_tensor.rs","line":377},{"file":"crates/apr-cli/src/commands/rosetta_diff_tensor.rs","line":378},{"file":"crates/apr-cli/src/commands/rosetta_diff_tensor.rs","line":379},{"file":"crates/apr-cli/src/commands/rosetta_diff_tensor.rs","line":380},{"file":"crates/apr-cli/src/commands/rosetta_diff_tensor.rs","line":381},{"file":"crates/apr-cli/src/commands/rosetta_diff_tensor.rs","line":382},{"file":"crates/apr-cli/src/commands/rosetta_print_fingerprint.rs","line":15},{"file":"crates/apr-cli/src/commands/rosetta_print_fingerprint.rs","line":30},{"file":"crates/apr-cli/src/commands/rosetta_print_inference.rs","line":2},{"file":"crates/apr-cli/src/commands/rosetta_print_inference.rs","line":73},{"file":"crates/apr-cli/src/commands/rosetta_print_inference.rs","line":74},{"file":"crates/apr-cli/src/commands/rosetta_print_inference.rs","line":75},{"file":"crates/apr-cli/src/commands/rosetta_print_inference.rs","line":76},{"file":"crates/apr-cli/src/commands/rosetta_print_inference.rs","line":77},{"file":"crates/apr-cli/src/commands/rosetta_print_inference.rs","line":78},{"file":"crates/apr-cli/src/commands/rosetta_print_inference.rs","line":79},{"file":"crates/apr-cli/src/commands/rosetta_print_inference.rs","line":80},{"file":"crates/apr-cli/src/commands/rosetta_print_inference.rs","line":81},{"file":"crates/apr-cli/src/commands/rosetta_print_inference.rs","line":82},{"file":"crates/apr-cli/src/commands/rosetta_print_inference.rs","line":83},{"file":"crates/apr-cli/src/commands/rosetta_print_inference.rs","line":84},{"file":"crates/apr-cli/src/commands/rosetta_print_inference.rs","line":85},{"file":"crates/apr-cli/src/commands/rosetta_print_inference.rs","line":86},{"file":"crates/apr-cli/src/commands/rosetta_print_inference.rs","line":87},{"file":"crates/apr-cli/src/commands/rosetta_print_inference.rs","line":88},{"file":"crates/apr-cli/src/commands/rosetta_print_inference.rs","line":89},{"file":"crates/apr-cli/src/commands/rosetta_print_inference.rs","line":90},{"file":"crates/apr-cli/src/commands/rosetta_print_inference.rs","line":91},{"file":"crates/apr-cli/src/commands/rosetta_print_inference.rs","line":92},{"file":"crates/apr-cli/src/commands/rosetta_print_inference.rs","line":93},{"file":"crates/apr-cli/src/commands/rosetta_print_inference.rs","line":94},{"file":"crates/apr-cli/src/commands/rosetta_print_inference.rs","line":95},{"file":"crates/apr-cli/src/commands/rosetta_print_inference.rs","line":96},{"file":"crates/apr-cli/src/commands/rosetta_print_inference.rs","line":97},{"file":"crates/apr-cli/src/commands/rosetta_print_inference.rs","line":98},{"file":"crates/apr-cli/src/commands/rosetta_print_inference.rs","line":99},{"file":"crates/apr-cli/src/commands/rosetta_print_inference.rs","line":100},{"file":"crates/apr-cli/src/commands/rosetta_print_inference.rs","line":101},{"file":"crates/apr-cli/src/commands/rosetta_print_inference.rs","line":102},{"file":"crates/apr-cli/src/commands/rosetta_tensor_fingerprint_statistical.rs","line":86},{"file":"crates/apr-cli/src/commands/rosetta_tensor_fingerprint_statistical.rs","line":89},{"file":"crates/apr-cli/src/commands/rosetta_tensor_fingerprint_statistical.rs","line":97},{"file":"crates/apr-cli/src/commands/rosetta_tensor_fingerprint_statistical.rs","line":121},{"file":"crates/aprender-core/src/format/rosetta/check.rs","line":238},{"file":"crates/aprender-core/src/format/rosetta/sharded.rs","line":173},{"file":"crates/aprender-core/src/format/rosetta/sharded.rs","line":218},{"file":"crates/aprender-core/src/format/rosetta/sharded.rs","line":219},{"file":"crates/aprender-core/src/format/rosetta/sharded.rs","line":220},{"file":"crates/aprender-core/src/format/rosetta/sharded.rs","line":221},{"file":"crates/aprender-core/src/format/rosetta/sharded.rs","line":222},{"file":"crates/aprender-core/src/format/rosetta/sharded.rs","line":223},{"file":"crates/aprender-core/src/format/rosetta/sharded.rs","line":224},{"file":"crates/aprender-core/src/format/rosetta/sharded.rs","line":225},{"file":"crates/aprender-core/src/format/rosetta/sharded.rs","line":226},{"file":"crates/aprender-core/src/format/rosetta/sharded.rs","line":245},{"file":"crates/aprender-core/src/format/rosetta/sharded.rs","line":271},{"file":"crates/aprender-core/src/format/rosetta/sharded.rs","line":285},{"file":"crates/aprender-core/src/format/rosetta/sharded.rs","line":300},{"file":"crates/aprender-core/src/format/rosetta/sharded.rs","line":305},{"file":"crates/aprender-core/src/format/rosetta/tests_pygmy_validation.rs","line":105},{"file":"crates/aprender-core/src/format/rosetta/tests_pygmy_validation.rs","line":112},{"file":"crates/aprender-core/src/format/rosetta/tests_pygmy_validation.rs","line":113},{"file":"crates/aprender-core/src/format/rosetta/tests_pygmy_validation.rs","line":369}]} +{"id":"R-pr2423","class":"R","stratum":"L","split":"Test","diff_sha256":"02e1a9c17700e11751b85177f1efd90e2d23c49b62d65f2b0b538931db534958","bytes":40614,"approx_tokens":10154,"source":"pr#2423 de6743716287b37fab247e59f970f0cb21a5409e reversed; fixes #2381","defect":[{"file":"crates/apr-cli/src/commands/data.rs","line":121},{"file":"crates/apr-cli/src/commands/data.rs","line":128},{"file":"crates/apr-cli/src/commands/data.rs","line":137},{"file":"crates/apr-cli/src/commands/data.rs","line":258},{"file":"crates/apr-cli/src/commands/data.rs","line":328},{"file":"crates/apr-cli/src/commands/data.rs","line":330},{"file":"crates/apr-cli/src/commands/data.rs","line":331},{"file":"crates/apr-cli/src/commands/data.rs","line":332},{"file":"crates/apr-cli/src/commands/data.rs","line":334},{"file":"crates/apr-cli/src/commands/data.rs","line":335},{"file":"crates/apr-cli/src/commands/data.rs","line":370},{"file":"crates/apr-cli/src/commands/data.rs","line":372},{"file":"crates/apr-cli/src/commands/data.rs","line":396},{"file":"crates/apr-cli/src/commands/data.rs","line":492},{"file":"crates/apr-cli/src/commands/data.rs","line":493},{"file":"crates/apr-cli/src/commands/data.rs","line":494},{"file":"crates/apr-cli/src/commands/data.rs","line":495},{"file":"crates/apr-cli/src/commands/data.rs","line":496},{"file":"crates/apr-cli/src/commands/data.rs","line":497},{"file":"crates/apr-cli/src/commands/data.rs","line":498},{"file":"crates/apr-cli/src/commands/data.rs","line":499},{"file":"crates/apr-cli/src/commands/data.rs","line":500},{"file":"crates/apr-cli/src/commands/data.rs","line":501},{"file":"crates/apr-cli/src/commands/data.rs","line":514},{"file":"crates/apr-cli/src/commands/data.rs","line":537},{"file":"crates/apr-cli/src/commands/data.rs","line":759},{"file":"crates/apr-cli/src/commands/data.rs","line":766},{"file":"crates/apr-cli/src/commands/data.rs","line":778},{"file":"crates/apr-cli/src/commands/data.rs","line":780},{"file":"crates/apr-cli/src/commands/data.rs","line":857},{"file":"crates/apr-cli/src/commands/data.rs","line":864},{"file":"crates/apr-cli/src/commands/data.rs","line":866},{"file":"crates/apr-cli/src/commands/data.rs","line":876},{"file":"crates/apr-cli/src/commands/data.rs","line":883},{"file":"crates/apr-cli/src/commands/data.rs","line":894},{"file":"crates/apr-cli/src/commands/data.rs","line":904},{"file":"crates/apr-cli/src/commands/data.rs","line":955},{"file":"crates/apr-cli/src/commands/data.rs","line":956},{"file":"crates/apr-cli/src/commands/data.rs","line":957},{"file":"crates/apr-cli/src/commands/data.rs","line":960},{"file":"crates/apr-cli/src/commands/data.rs","line":961},{"file":"crates/apr-cli/src/commands/data.rs","line":1244},{"file":"crates/apr-cli/src/commands/data.rs","line":1277},{"file":"crates/apr-cli/src/commands/data.rs","line":1757},{"file":"crates/apr-cli/src/data_commands.rs","line":2},{"file":"crates/apr-cli/src/data_commands.rs","line":57},{"file":"crates/apr-cli/src/data_commands.rs","line":58},{"file":"crates/apr-cli/src/data_commands.rs","line":61},{"file":"crates/apr-cli/src/dispatch_analysis.rs","line":774},{"file":"crates/aprender-data/src/quality/decontaminate.rs","line":43},{"file":"crates/aprender-data/src/quality/decontaminate.rs","line":51},{"file":"crates/aprender-data/src/quality/decontaminate.rs","line":106},{"file":"crates/aprender-data/src/quality/decontaminate.rs","line":149},{"file":"crates/aprender-data/src/quality/decontaminate.rs","line":244}]} +{"id":"R-pr2424","class":"R","stratum":"M","split":"Test","diff_sha256":"31f870a337b8604745500add7627a81463a19482860848cd6df6f56127ec2367","bytes":23631,"approx_tokens":5908,"source":"pr#2424 12fe6c718a9bdcfb2b07bd4d2534e787ce679b51 reversed; fixes #2384","defect":[{"file":"/dev/null","line":0},{"file":"crates/aprender-mcp/src/lib.rs","line":40},{"file":"crates/aprender-mcp/src/tools/finetune.rs","line":128},{"file":"crates/aprender-mcp/src/tools/finetune.rs","line":139},{"file":"crates/aprender-mcp/src/tools/finetune.rs","line":140},{"file":"crates/aprender-mcp/src/tools/run.rs","line":123},{"file":"crates/aprender-mcp/src/tools/run.rs","line":137},{"file":"crates/aprender-mcp/src/tools/run.rs","line":138},{"file":"crates/aprender-mcp/src/tools/serve.rs","line":131},{"file":"crates/aprender-mcp/src/tools/subprocess.rs","line":14},{"file":"crates/aprender-mcp/src/tools/subprocess.rs","line":39},{"file":"crates/aprender-mcp/src/tools/subprocess.rs","line":42},{"file":"crates/aprender-mcp/src/tools/subprocess.rs","line":43},{"file":"crates/aprender-mcp/src/tools/subprocess.rs","line":52},{"file":"crates/aprender-mcp/src/tools/subprocess.rs","line":53},{"file":"crates/aprender-mcp/src/tools/subprocess.rs","line":64},{"file":"crates/aprender-mcp/src/tools/subprocess.rs","line":65},{"file":"crates/aprender-mcp/src/tools/subprocess.rs","line":85},{"file":"crates/aprender-mcp/src/tools/subprocess.rs","line":88},{"file":"crates/aprender-mcp/src/tools/subprocess.rs","line":89},{"file":"crates/aprender-mcp/src/tools/subprocess.rs","line":91},{"file":"crates/aprender-mcp/src/tools/subprocess.rs","line":92},{"file":"crates/aprender-mcp/src/tools/subprocess.rs","line":97},{"file":"crates/aprender-mcp/src/tools/subprocess.rs","line":255},{"file":"crates/aprender-mcp/src/tools/subprocess.rs","line":258},{"file":"crates/aprender-mcp/src/tools/subprocess.rs","line":259},{"file":"crates/aprender-mcp/src/tools/subprocess.rs","line":261},{"file":"crates/aprender-mcp/src/tools/subprocess.rs","line":265},{"file":"crates/aprender-mcp/src/tools/subprocess.rs","line":352},{"file":"crates/aprender-orchestrate/src/agent/driver/apr_serve.rs","line":521},{"file":"crates/aprender-orchestrate/src/agent/driver/apr_serve.rs","line":523},{"file":"crates/aprender-orchestrate/src/agent/driver/apr_serve.rs","line":525}]} +{"id":"R-pr2469","class":"R","stratum":"L","split":"Dev","diff_sha256":"f139395c63dd827d02e1a75c490bcb901ec7d95aff150b0068209462d61d7b7e","bytes":102813,"approx_tokens":25704,"source":"pr#2469 f3b840a0b7a7231e30ad5fee0cab2ae1c2b93e73 reversed; fixes #2465","defect":[{"file":"/dev/null","line":0},{"file":"crates/aprender-contracts/src/lint/strict_test_binding.rs","line":26},{"file":"crates/aprender-contracts/src/lint/strict_test_binding.rs","line":28},{"file":"crates/aprender-contracts/src/lint/strict_test_binding.rs","line":29},{"file":"crates/aprender-contracts/src/lint/strict_test_binding.rs","line":30},{"file":"crates/aprender-contracts/src/lint/strict_test_binding.rs","line":35},{"file":"crates/aprender-contracts/src/lint/strict_test_binding.rs","line":52},{"file":"crates/aprender-contracts/src/lint/strict_test_binding.rs","line":56},{"file":"crates/aprender-contracts/src/lint/strict_test_binding.rs","line":57},{"file":"crates/aprender-contracts/src/lint/strict_test_binding.rs","line":58},{"file":"crates/aprender-contracts/src/lint/strict_test_binding.rs","line":59},{"file":"crates/aprender-contracts/src/lint/strict_test_binding.rs","line":60},{"file":"crates/aprender-contracts/src/lint/strict_test_binding.rs","line":61},{"file":"crates/aprender-contracts/src/lint/strict_test_binding.rs","line":62},{"file":"crates/aprender-contracts/src/lint/strict_test_binding.rs","line":63},{"file":"crates/aprender-contracts/src/lint/strict_test_binding.rs","line":64},{"file":"crates/aprender-contracts/src/lint/strict_test_binding.rs","line":65},{"file":"crates/aprender-contracts/src/lint/strict_test_binding.rs","line":66},{"file":"crates/aprender-contracts/src/lint/strict_test_binding.rs","line":67},{"file":"crates/aprender-contracts/src/lint/strict_test_binding.rs","line":69},{"file":"crates/aprender-contracts/src/lint/strict_test_binding.rs","line":70},{"file":"crates/aprender-contracts/src/lint/strict_test_binding.rs","line":71},{"file":"crates/aprender-contracts/src/lint/strict_test_binding.rs","line":72},{"file":"crates/aprender-contracts/src/lint/strict_test_binding.rs","line":73},{"file":"crates/aprender-contracts/src/lint/strict_test_binding.rs","line":74},{"file":"crates/aprender-contracts/src/lint/strict_test_binding.rs","line":75},{"file":"crates/aprender-contracts/src/lint/strict_test_binding.rs","line":76},{"file":"crates/aprender-contracts/src/lint/strict_test_binding.rs","line":77},{"file":"crates/aprender-contracts/src/lint/strict_test_binding.rs","line":78},{"file":"crates/aprender-contracts/src/lint/strict_test_binding.rs","line":79},{"file":"crates/aprender-contracts/src/lint/strict_test_binding.rs","line":80},{"file":"crates/aprender-contracts/src/lint/strict_test_binding.rs","line":81},{"file":"crates/aprender-contracts/src/lint/strict_test_binding.rs","line":82},{"file":"crates/aprender-contracts/src/lint/strict_test_binding.rs","line":83},{"file":"crates/aprender-contracts/src/lint/strict_test_binding.rs","line":84},{"file":"crates/aprender-contracts/src/lint/strict_test_binding.rs","line":85},{"file":"crates/aprender-contracts/src/lint/strict_test_binding.rs","line":86},{"file":"crates/aprender-contracts/src/lint/strict_test_binding.rs","line":115},{"file":"crates/aprender-contracts/src/lint/strict_test_binding.rs","line":116},{"file":"crates/aprender-contracts/src/lint/strict_test_binding.rs","line":117},{"file":"crates/aprender-contracts/src/lint/strict_test_binding.rs","line":118},{"file":"crates/aprender-contracts/src/lint/strict_test_binding.rs","line":119},{"file":"crates/aprender-contracts/src/lint/strict_test_binding.rs","line":120},{"file":"crates/aprender-contracts/src/lint/strict_test_binding.rs","line":121},{"file":"crates/aprender-contracts/src/lint/strict_test_binding.rs","line":122},{"file":"crates/aprender-contracts/src/lint/strict_test_binding.rs","line":123},{"file":"crates/aprender-contracts/src/lint/strict_test_binding.rs","line":124},{"file":"crates/aprender-contracts/src/lint/strict_test_binding.rs","line":125},{"file":"crates/aprender-contracts/src/lint/strict_test_binding.rs","line":126},{"file":"crates/aprender-contracts/src/lint/strict_test_binding.rs","line":127},{"file":"crates/aprender-contracts/src/lint/strict_test_binding.rs","line":128},{"file":"crates/aprender-contracts/src/lint/strict_test_binding.rs","line":129},{"file":"crates/aprender-contracts/src/lint/strict_test_binding.rs","line":130},{"file":"crates/aprender-contracts/src/lint/strict_test_binding.rs","line":131},{"file":"crates/aprender-contracts/src/lint/strict_test_binding.rs","line":168},{"file":"crates/aprender-contracts/src/lint/strict_test_binding.rs","line":177},{"file":"crates/aprender-contracts/src/lint/strict_test_binding.rs","line":178},{"file":"crates/aprender-contracts/src/lint/strict_test_binding.rs","line":179},{"file":"crates/aprender-contracts/src/lint/strict_test_binding.rs","line":182},{"file":"crates/aprender-contracts/src/lint/strict_test_binding.rs","line":183},{"file":"crates/aprender-contracts/src/lint/strict_test_binding.rs","line":185},{"file":"crates/aprender-contracts/src/lint/strict_test_binding.rs","line":186},{"file":"crates/aprender-contracts/src/lint/strict_test_binding.rs","line":187},{"file":"crates/aprender-contracts/src/lint/strict_test_binding.rs","line":188},{"file":"crates/aprender-contracts/src/lint/strict_test_binding.rs","line":189},{"file":"crates/aprender-contracts/src/lint/strict_test_binding.rs","line":190},{"file":"crates/aprender-contracts/src/lint/strict_test_binding.rs","line":191},{"file":"crates/aprender-contracts/src/lint/strict_test_binding.rs","line":192},{"file":"crates/aprender-contracts/src/lint/strict_test_binding.rs","line":193},{"file":"crates/aprender-contracts/src/lint/strict_test_binding.rs","line":194},{"file":"crates/aprender-contracts/src/lint/strict_test_binding.rs","line":195},{"file":"crates/aprender-contracts/src/lint/strict_test_binding.rs","line":196},{"file":"crates/aprender-contracts/src/lint/strict_test_binding.rs","line":197},{"file":"crates/aprender-contracts/src/lint/strict_test_binding.rs","line":198},{"file":"crates/aprender-contracts/src/lint/strict_test_binding.rs","line":199},{"file":"crates/aprender-contracts/src/lint/strict_test_binding.rs","line":200},{"file":"crates/aprender-contracts/src/lint/strict_test_binding.rs","line":201},{"file":"crates/aprender-contracts/src/lint/strict_test_binding.rs","line":202},{"file":"crates/aprender-contracts/src/lint/strict_test_binding.rs","line":203},{"file":"crates/aprender-contracts/src/lint/strict_test_binding.rs","line":204},{"file":"crates/aprender-contracts/src/lint/strict_test_binding.rs","line":205},{"file":"crates/aprender-contracts/src/lint/strict_test_binding.rs","line":206},{"file":"crates/aprender-contracts/src/lint/strict_test_binding.rs","line":207},{"file":"crates/aprender-contracts/src/lint/strict_test_binding.rs","line":208},{"file":"crates/aprender-contracts/src/lint/strict_test_binding.rs","line":209},{"file":"crates/aprender-contracts/src/lint/strict_test_binding.rs","line":210},{"file":"crates/aprender-contracts/src/lint/strict_test_binding.rs","line":211},{"file":"crates/aprender-contracts/src/lint/strict_test_binding.rs","line":212},{"file":"crates/aprender-contracts/src/lint/strict_test_binding.rs","line":213},{"file":"crates/aprender-contracts/src/lint/strict_test_binding.rs","line":214},{"file":"crates/aprender-contracts/src/lint/strict_test_binding.rs","line":215},{"file":"crates/aprender-contracts/src/lint/strict_test_binding.rs","line":216},{"file":"crates/aprender-contracts/src/lint/strict_test_binding.rs","line":217},{"file":"crates/aprender-contracts/src/lint/strict_test_binding.rs","line":218},{"file":"crates/aprender-contracts/src/lint/strict_test_binding.rs","line":220},{"file":"crates/aprender-contracts/src/lint/strict_test_binding.rs","line":228},{"file":"crates/aprender-contracts/src/lint/strict_test_binding.rs","line":229},{"file":"crates/aprender-contracts/src/lint/strict_test_binding.rs","line":230},{"file":"crates/aprender-contracts/src/lint/strict_test_binding.rs","line":232},{"file":"crates/aprender-contracts/src/lint/strict_test_binding.rs","line":233},{"file":"crates/aprender-contracts/src/lint/strict_test_binding.rs","line":234},{"file":"crates/aprender-contracts/src/lint/strict_test_binding.rs","line":236},{"file":"crates/aprender-contracts/src/lint/strict_test_binding.rs","line":237},{"file":"crates/aprender-contracts/src/lint/strict_test_binding.rs","line":238},{"file":"crates/aprender-contracts/src/lint/strict_test_binding.rs","line":239},{"file":"crates/aprender-contracts/src/lint/strict_test_binding.rs","line":240},{"file":"crates/aprender-contracts/src/lint/strict_test_binding.rs","line":241},{"file":"crates/aprender-contracts/src/lint/strict_test_binding.rs","line":242},{"file":"crates/aprender-contracts/src/lint/strict_test_binding.rs","line":243},{"file":"crates/aprender-contracts/src/lint/strict_test_binding.rs","line":244},{"file":"crates/aprender-contracts/src/lint/strict_test_binding.rs","line":245},{"file":"crates/aprender-contracts/src/lint/strict_test_binding.rs","line":246},{"file":"crates/aprender-contracts/src/lint/strict_test_binding.rs","line":247},{"file":"crates/aprender-contracts/src/lint/strict_test_binding.rs","line":248},{"file":"crates/aprender-contracts/src/lint/strict_test_binding.rs","line":249},{"file":"crates/aprender-contracts/src/lint/strict_test_binding.rs","line":252},{"file":"crates/aprender-contracts/src/lint/strict_test_binding.rs","line":253},{"file":"crates/aprender-contracts/src/lint/strict_test_binding.rs","line":254},{"file":"crates/aprender-contracts/src/lint/strict_test_binding.rs","line":255},{"file":"crates/aprender-contracts/src/lint/strict_test_binding.rs","line":256},{"file":"crates/aprender-contracts/src/lint/strict_test_binding.rs","line":257},{"file":"crates/aprender-contracts/src/lint/strict_test_binding.rs","line":258},{"file":"crates/aprender-contracts/src/lint/strict_test_binding.rs","line":259},{"file":"crates/aprender-contracts/src/lint/strict_test_binding.rs","line":260},{"file":"crates/aprender-contracts/src/lint/strict_test_binding.rs","line":261},{"file":"crates/aprender-contracts/src/lint/strict_test_binding.rs","line":280},{"file":"crates/aprender-contracts/src/lint/strict_test_binding.rs","line":281},{"file":"crates/aprender-contracts/src/lint/strict_test_binding.rs","line":282},{"file":"crates/aprender-contracts/src/lint/strict_test_binding.rs","line":295},{"file":"crates/aprender-contracts/src/lint/strict_test_binding.rs","line":296},{"file":"crates/aprender-contracts/src/lint/strict_test_binding.rs","line":297},{"file":"crates/aprender-contracts/src/lint/strict_test_binding.rs","line":298},{"file":"crates/aprender-contracts/src/lint/strict_test_binding.rs","line":299},{"file":"crates/aprender-contracts/src/lint/strict_test_binding.rs","line":300},{"file":"crates/aprender-contracts/src/lint/strict_test_binding.rs","line":301},{"file":"crates/aprender-contracts/src/lint/strict_test_binding.rs","line":302},{"file":"crates/aprender-contracts/src/lint/strict_test_binding.rs","line":303},{"file":"crates/aprender-contracts/src/lint/strict_test_binding.rs","line":304},{"file":"crates/aprender-contracts/src/lint/strict_test_binding.rs","line":305},{"file":"crates/aprender-contracts/src/lint/strict_test_binding.rs","line":306},{"file":"crates/aprender-contracts/src/lint/strict_test_binding.rs","line":307},{"file":"crates/aprender-contracts/src/lint/strict_test_binding.rs","line":308},{"file":"crates/aprender-contracts/src/lint/strict_test_binding.rs","line":309},{"file":"crates/aprender-contracts/src/lint/strict_test_binding.rs","line":313},{"file":"crates/aprender-contracts/src/lint/strict_test_binding.rs","line":316},{"file":"crates/aprender-contracts/src/lint/strict_test_binding.rs","line":323},{"file":"crates/aprender-contracts/src/lint/strict_test_binding.rs","line":335},{"file":"crates/aprender-contracts/src/lint/strict_test_binding.rs","line":338},{"file":"crates/aprender-contracts/src/lint/strict_test_binding.rs","line":365},{"file":"crates/aprender-contracts/src/lint/strict_test_binding.rs","line":366},{"file":"crates/aprender-contracts/src/lint/strict_test_binding.rs","line":367},{"file":"crates/aprender-contracts/src/lint/strict_test_binding.rs","line":368},{"file":"crates/aprender-contracts/src/lint/strict_test_binding.rs","line":371},{"file":"crates/aprender-contracts/src/lint/strict_test_binding.rs","line":372},{"file":"crates/aprender-contracts/src/lint/strict_test_binding.rs","line":374},{"file":"crates/aprender-contracts/src/lint/strict_test_binding.rs","line":426},{"file":"crates/aprender-contracts/src/lint/strict_test_binding.rs","line":431},{"file":"crates/aprender-contracts/src/lint/strict_test_binding.rs","line":432},{"file":"crates/aprender-contracts/src/lint/strict_test_binding.rs","line":437},{"file":"crates/aprender-contracts/src/lint/strict_test_binding.rs","line":442},{"file":"crates/aprender-contracts/src/lint/strict_test_binding.rs","line":443},{"file":"crates/aprender-contracts/src/lint/strict_test_binding.rs","line":444},{"file":"crates/aprender-contracts/src/lint/strict_test_binding.rs","line":539},{"file":"crates/aprender-contracts/src/lint/strict_test_binding.rs","line":572},{"file":"crates/aprender-contracts/src/lint/strict_test_binding.rs","line":599},{"file":"crates/aprender-contracts/src/schema/types.rs","line":468},{"file":"crates/aprender-qa-runner/src/conversion_tests_c.rs","line":4},{"file":"crates/aprender-qa-runner/src/conversion_tests_c.rs","line":14},{"file":"crates/aprender-qa-runner/src/conversion_tests_executor.rs","line":146},{"file":"crates/aprender-qa-runner/src/conversion_tests_executor.rs","line":156},{"file":"crates/aprender-serve/src/api/apr_q4k_scheduler.rs","line":6},{"file":"crates/aprender-serve/src/api/apr_q4k_scheduler.rs","line":19},{"file":"crates/aprender-serve/src/api/apr_q4k_scheduler.rs","line":212},{"file":"crates/aprender-serve/src/api/apr_q4k_scheduler.rs","line":235},{"file":"crates/aprender-serve/src/api/apr_q4k_scheduler.rs","line":265},{"file":"crates/aprender-serve/src/api/apr_q4k_scheduler.rs","line":271},{"file":"crates/aprender-serve/src/api/apr_q4k_scheduler.rs","line":272},{"file":"crates/aprender-serve/src/api/apr_q4k_scheduler.rs","line":273},{"file":"crates/aprender-serve/src/api/apr_q4k_scheduler.rs","line":274},{"file":"crates/aprender-serve/src/api/apr_q4k_scheduler.rs","line":275},{"file":"crates/aprender-serve/src/api/apr_q4k_scheduler.rs","line":276},{"file":"crates/aprender-serve/src/api/apr_q4k_scheduler.rs","line":277},{"file":"crates/aprender-serve/src/api/apr_q4k_scheduler.rs","line":278},{"file":"crates/aprender-serve/src/api/apr_q4k_scheduler.rs","line":279},{"file":"crates/aprender-serve/src/api/apr_q4k_scheduler.rs","line":280},{"file":"crates/aprender-serve/src/api/apr_q4k_scheduler.rs","line":281},{"file":"crates/aprender-serve/src/api/apr_q4k_scheduler.rs","line":282},{"file":"crates/aprender-serve/src/api/apr_q4k_scheduler.rs","line":283},{"file":"crates/aprender-serve/src/api/apr_q4k_scheduler.rs","line":284},{"file":"crates/aprender-serve/src/api/apr_q4k_scheduler.rs","line":285},{"file":"crates/aprender-serve/src/api/apr_q4k_scheduler.rs","line":286},{"file":"crates/aprender-serve/src/api/apr_q4k_scheduler.rs","line":287},{"file":"crates/aprender-serve/src/api/apr_q4k_scheduler.rs","line":288},{"file":"crates/aprender-serve/src/api/apr_q4k_scheduler.rs","line":289},{"file":"crates/aprender-serve/src/api/apr_q4k_scheduler.rs","line":290},{"file":"crates/aprender-serve/src/api/apr_q4k_scheduler.rs","line":291},{"file":"crates/aprender-serve/src/api/apr_q4k_scheduler.rs","line":292},{"file":"crates/aprender-serve/src/api/apr_q4k_scheduler.rs","line":293},{"file":"crates/aprender-serve/src/api/apr_q4k_scheduler.rs","line":294},{"file":"crates/aprender-serve/src/api/apr_q4k_scheduler.rs","line":295},{"file":"crates/aprender-serve/src/api/apr_q4k_scheduler.rs","line":296},{"file":"crates/aprender-serve/src/api/apr_q4k_scheduler.rs","line":297},{"file":"crates/aprender-serve/src/api/apr_q4k_scheduler.rs","line":298},{"file":"crates/aprender-serve/src/api/apr_q4k_scheduler.rs","line":299},{"file":"crates/aprender-serve/src/api/apr_q4k_scheduler.rs","line":300},{"file":"crates/aprender-serve/src/api/apr_q4k_scheduler.rs","line":301},{"file":"crates/aprender-serve/src/api/apr_q4k_scheduler.rs","line":302},{"file":"crates/aprender-serve/src/api/apr_q4k_scheduler.rs","line":319},{"file":"crates/aprender-serve/src/api/batch.rs","line":176},{"file":"crates/aprender-serve/src/api/batch.rs","line":203},{"file":"crates/aprender-serve/src/api/batch.rs","line":379},{"file":"crates/aprender-serve/src/api/batch_processing.rs","line":217},{"file":"crates/aprender-serve/src/api/batch_processing.rs","line":248},{"file":"crates/aprender-serve/src/api/batch_processing.rs","line":249},{"file":"crates/aprender-serve/src/api/batch_processing.rs","line":250},{"file":"crates/aprender-serve/src/api/batch_processing.rs","line":251},{"file":"crates/aprender-serve/src/api/batch_processing.rs","line":252},{"file":"crates/aprender-serve/src/api/batch_processing.rs","line":253},{"file":"crates/aprender-serve/src/api/batch_processing.rs","line":254},{"file":"crates/aprender-serve/src/api/batch_processing.rs","line":255},{"file":"crates/aprender-serve/src/api/batch_processing.rs","line":256},{"file":"crates/aprender-serve/src/api/batch_processing.rs","line":257},{"file":"crates/aprender-serve/src/api/batch_processing.rs","line":258},{"file":"crates/aprender-serve/src/api/batch_processing.rs","line":312},{"file":"crates/aprender-serve/src/api/batch_processing.rs","line":313},{"file":"crates/aprender-serve/src/api/batch_processing.rs","line":314},{"file":"crates/aprender-serve/src/api/batch_processing.rs","line":315},{"file":"crates/aprender-serve/src/api/batch_processing.rs","line":316},{"file":"crates/aprender-serve/src/api/batch_processing.rs","line":317},{"file":"crates/aprender-serve/src/api/batch_processing.rs","line":318},{"file":"crates/aprender-serve/src/api/batch_processing.rs","line":319},{"file":"crates/aprender-serve/src/api/batch_processing.rs","line":320},{"file":"crates/aprender-serve/src/api/batch_processing.rs","line":321},{"file":"crates/aprender-serve/src/api/batch_processing.rs","line":322},{"file":"crates/aprender-serve/src/api/batch_processing.rs","line":323},{"file":"crates/aprender-serve/src/api/batch_processing.rs","line":324},{"file":"crates/aprender-serve/src/api/batch_processing.rs","line":325},{"file":"crates/aprender-serve/src/api/batch_processing.rs","line":326},{"file":"crates/aprender-serve/src/api/cuda_chat_backend.rs","line":495},{"file":"crates/aprender-serve/src/api/cuda_chat_backend.rs","line":496},{"file":"crates/aprender-serve/src/api/cuda_chat_backend.rs","line":497},{"file":"crates/aprender-serve/src/api/cuda_chat_backend.rs","line":524},{"file":"crates/aprender-serve/src/api/cuda_chat_backend.rs","line":660},{"file":"crates/aprender-serve/src/api/gpu_completions_handler.rs","line":2},{"file":"crates/aprender-serve/src/api/gpu_completions_handler.rs","line":3},{"file":"crates/aprender-serve/src/api/gpu_completions_handler.rs","line":4},{"file":"crates/aprender-serve/src/api/gpu_completions_handler.rs","line":5},{"file":"crates/aprender-serve/src/api/gpu_completions_handler.rs","line":6},{"file":"crates/aprender-serve/src/api/gpu_completions_handler.rs","line":7},{"file":"crates/aprender-serve/src/api/gpu_completions_handler.rs","line":8},{"file":"crates/aprender-serve/src/api/gpu_completions_handler.rs","line":9},{"file":"crates/aprender-serve/src/api/gpu_completions_handler.rs","line":10},{"file":"crates/aprender-serve/src/api/gpu_completions_handler.rs","line":11},{"file":"crates/aprender-serve/src/api/gpu_completions_handler.rs","line":12},{"file":"crates/aprender-serve/src/api/gpu_completions_handler.rs","line":13},{"file":"crates/aprender-serve/src/api/gpu_completions_handler.rs","line":14},{"file":"crates/aprender-serve/src/api/gpu_completions_handler.rs","line":15},{"file":"crates/aprender-serve/src/api/gpu_completions_handler.rs","line":16},{"file":"crates/aprender-serve/src/api/gpu_completions_handler.rs","line":82},{"file":"crates/aprender-serve/src/api/gpu_completions_handler.rs","line":83},{"file":"crates/aprender-serve/src/api/gpu_completions_handler.rs","line":84},{"file":"crates/aprender-serve/src/api/gpu_completions_handler.rs","line":89},{"file":"crates/aprender-serve/src/api/gpu_completions_handler.rs","line":90},{"file":"crates/aprender-serve/src/api/gpu_completions_handler.rs","line":91},{"file":"crates/aprender-serve/src/api/gpu_completions_handler.rs","line":92},{"file":"crates/aprender-serve/src/api/gpu_completions_handler.rs","line":93},{"file":"crates/aprender-serve/src/api/gpu_completions_handler.rs","line":94},{"file":"crates/aprender-serve/src/api/gpu_completions_handler.rs","line":176},{"file":"crates/aprender-serve/src/api/gpu_completions_handler.rs","line":187},{"file":"crates/aprender-serve/src/api/gpu_completions_handler.rs","line":213},{"file":"crates/aprender-serve/src/api/gpu_completions_handler.rs","line":229},{"file":"crates/aprender-serve/src/api/gpu_completions_handler.rs","line":230},{"file":"crates/aprender-serve/src/api/gpu_completions_handler.rs","line":241},{"file":"crates/aprender-serve/src/api/gpu_completions_handler.rs","line":313},{"file":"crates/aprender-serve/src/api/gpu_completions_handler.rs","line":315},{"file":"crates/aprender-serve/src/api/gpu_completions_handler.rs","line":316},{"file":"crates/aprender-serve/src/api/gpu_completions_handler.rs","line":317},{"file":"crates/aprender-serve/src/api/gpu_completions_handler.rs","line":318},{"file":"crates/aprender-serve/src/api/gpu_completions_handler.rs","line":319},{"file":"crates/aprender-serve/src/api/gpu_completions_handler.rs","line":320},{"file":"crates/aprender-serve/src/api/gpu_completions_handler.rs","line":328},{"file":"crates/aprender-serve/src/api/gpu_completions_handler.rs","line":436},{"file":"crates/aprender-serve/src/api/gpu_completions_handler.rs","line":481},{"file":"crates/aprender-serve/src/api/gpu_completions_handler.rs","line":493},{"file":"crates/aprender-serve/src/api/gpu_completions_handler.rs","line":642},{"file":"crates/aprender-serve/src/api/gpu_completions_handler.rs","line":650},{"file":"crates/aprender-serve/src/api/gpu_completions_handler.rs","line":651},{"file":"crates/aprender-serve/src/api/gpu_completions_handler.rs","line":653},{"file":"crates/aprender-serve/src/api/gpu_completions_handler.rs","line":659},{"file":"crates/aprender-serve/src/api/gpu_completions_handler.rs","line":666},{"file":"crates/aprender-serve/src/api/gpu_completions_handler.rs","line":667},{"file":"crates/aprender-serve/src/api/mod.rs","line":58},{"file":"crates/aprender-serve/src/api/openai_handlers.rs","line":357},{"file":"crates/aprender-serve/src/api/openai_handlers.rs","line":358},{"file":"crates/aprender-serve/src/api/openai_handlers.rs","line":359},{"file":"crates/aprender-serve/src/api/openai_handlers.rs","line":360},{"file":"crates/aprender-serve/src/api/openai_handlers.rs","line":361},{"file":"crates/aprender-serve/src/api/openai_handlers.rs","line":362},{"file":"crates/aprender-serve/src/api/openai_handlers.rs","line":363},{"file":"crates/aprender-serve/src/api/openai_handlers.rs","line":364},{"file":"crates/aprender-serve/src/api/openai_handlers.rs","line":365},{"file":"crates/aprender-serve/src/api/openai_handlers.rs","line":366},{"file":"crates/aprender-serve/src/api/realize_handlers_completion_request.rs","line":262},{"file":"crates/aprender-serve/src/api/realize_handlers_completion_request.rs","line":281},{"file":"crates/aprender-serve/src/api/realize_handlers_completion_request.rs","line":294},{"file":"crates/aprender-serve/src/api/realize_handlers_completion_request.rs","line":300},{"file":"crates/aprender-serve/src/api/realize_handlers_completion_request.rs","line":308},{"file":"crates/aprender-serve/src/api/realize_handlers_embed_completion.rs","line":435},{"file":"crates/aprender-serve/src/api/realize_handlers_embed_completion.rs","line":436},{"file":"crates/aprender-serve/src/api/realize_handlers_embed_completion.rs","line":437},{"file":"crates/aprender-serve/src/api/realize_handlers_embed_completion.rs","line":438},{"file":"crates/aprender-serve/src/api/realize_handlers_embed_completion.rs","line":439},{"file":"crates/aprender-serve/src/api/realize_handlers_embed_completion.rs","line":440},{"file":"crates/aprender-serve/src/api/realize_handlers_embed_completion.rs","line":461},{"file":"crates/aprender-serve/src/api/realize_handlers_embed_completion.rs","line":470},{"file":"crates/aprender-serve/src/api/realize_handlers_embed_completion.rs","line":509},{"file":"crates/aprender-serve/src/api/realize_handlers_embed_completion.rs","line":537},{"file":"crates/aprender-serve/src/api/realize_handlers_embed_completion.rs","line":633},{"file":"crates/aprender-serve/src/api/realize_handlers_embed_completion.rs","line":666},{"file":"crates/aprender-serve/src/api/realize_handlers_embed_completion.rs","line":667},{"file":"crates/aprender-serve/src/api/realize_handlers_embed_completion.rs","line":679},{"file":"crates/aprender-serve/src/api/realize_handlers_embed_completion.rs","line":734},{"file":"crates/aprender-serve/src/api/realize_handlers_embed_completion.rs","line":735},{"file":"crates/aprender-serve/src/api/realize_handlers_embed_completion.rs","line":747}]} +{"id":"R-pr2540","class":"R","stratum":"S","split":"Dev","diff_sha256":"68665827d8affd6d813fcbde11285991b1f71778b1ca0abf08b52c983708ac93","bytes":7733,"approx_tokens":1934,"source":"pr#2540 773a39da13c587f9c24eadc8dc5178d451b6a104 reversed; fixes #2539","defect":[{"file":"/dev/null","line":0},{"file":"crates/apr-cli/src/commands_enum.rs","line":831},{"file":"crates/apr-cli/src/commands_enum.rs","line":832},{"file":"crates/apr-cli/src/commands_enum.rs","line":833},{"file":"crates/apr-cli/src/commands_enum.rs","line":834},{"file":"crates/apr-cli/src/commands_enum.rs","line":835},{"file":"crates/apr-cli/src/dispatch.rs","line":94},{"file":"crates/apr-cli/src/dispatch.rs","line":95},{"file":"crates/apr-cli/src/dispatch.rs","line":96},{"file":"crates/apr-cli/src/dispatch.rs","line":97}]} +{"id":"R-pr2617","class":"R","stratum":"L","split":"Test","diff_sha256":"66daca2dab9222f96edd5b06c2bd9c080bde3bb81e11b4dbf21aa82d018713a5","bytes":36726,"approx_tokens":9182,"source":"pr#2617 4d42f79eb8d235c0381d124403691d18db4fdbd0 reversed; fixes #2555,#2559,#2589","defect":[{"file":"/dev/null","line":0},{"file":"crates/aprender-contracts-cli/src/lib.rs","line":22},{"file":"crates/aprender-contracts-cli/src/lib.rs","line":27},{"file":"crates/aprender-contracts-cli/src/lib.rs","line":342},{"file":"crates/aprender-contracts/src/schema/types.rs","line":52},{"file":"crates/aprender-contracts/src/schema/types.rs","line":105},{"file":"crates/aprender-contracts/src/schema/types.rs","line":111},{"file":"crates/aprender-contracts/src/schema/types.rs","line":125},{"file":"crates/aprender-contracts/src/schema/types.rs","line":326},{"file":"crates/aprender-contracts/src/schema/validator.rs","line":49},{"file":"scripts/check_pv_bin_resolution.sh","line":45},{"file":"scripts/check_pv_bin_resolution.sh","line":81},{"file":"scripts/check_pv_bin_resolution.sh","line":82},{"file":"scripts/check_pv_bin_resolution.sh","line":83},{"file":"scripts/check_pv_bin_resolution.sh","line":85},{"file":"scripts/check_pv_bin_resolution.sh","line":129},{"file":"scripts/dogfood_surfaces.sh","line":252},{"file":"scripts/pv_bin.sh","line":72}]} +{"id":"R-pr2707","class":"R","stratum":"M","split":"Test","diff_sha256":"58b4e337288c066ea76f8707cdf92a61dabda178420be0ef719ea8b56ce52cd4","bytes":24702,"approx_tokens":6176,"source":"pr#2707 d1b7d1995e49c8b3e45c191be493b02f60fc9867 reversed; fixes #2696","defect":[{"file":"/dev/null","line":0},{"file":"crates/apr-cli/src/commands/serve/handler_gpu_completion.rs","line":406},{"file":"crates/apr-cli/src/commands/serve/handler_gpu_completion.rs","line":407},{"file":"crates/apr-cli/src/commands/serve/handler_gpu_completion.rs","line":412},{"file":"crates/apr-cli/src/commands/serve/handlers.rs","line":776},{"file":"crates/apr-cli/src/commands/serve/handlers.rs","line":1084},{"file":"crates/apr-cli/src/commands/serve/mod.rs","line":338},{"file":"crates/apr-cli/src/commands/serve/types.rs","line":15},{"file":"crates/apr-cli/src/commands/serve/types.rs","line":110},{"file":"crates/apr-cli/src/commands/serve/types.rs","line":518},{"file":"crates/apr-cli/src/dispatch.rs","line":187},{"file":"crates/apr-cli/src/dispatch_analysis.rs","line":1689},{"file":"crates/apr-cli/src/lib.rs","line":49}]} +{"id":"R-pr2809","class":"R","stratum":"M","split":"Test","diff_sha256":"3dca02a5446ceb7d237d47bd31915d6b0682dfd3fe306da7eda39f4c2a6ece60","bytes":25724,"approx_tokens":6431,"source":"pr#2809 9462b52351ad7daac8debe0c995a2fa732160cf5 reversed; fixes #2753","defect":[{"file":"/dev/null","line":0},{"file":"crates/aprender-serve/src/cuda/executor/layers/cublas_prefill/attention.rs","line":970},{"file":"crates/aprender-serve/src/cuda/executor/layers/cublas_prefill/attention.rs","line":987},{"file":"crates/aprender-serve/src/cuda/executor/layers/cublas_prefill/attention.rs","line":988},{"file":"crates/aprender-serve/src/cuda/executor/layers/cublas_prefill/attention.rs","line":989},{"file":"crates/aprender-serve/src/cuda/executor/layers/cublas_prefill/attention.rs","line":990},{"file":"crates/aprender-serve/src/cuda/executor/layers/cublas_prefill/attention.rs","line":991},{"file":"crates/aprender-serve/src/cuda/executor/layers/cublas_prefill/attention.rs","line":992},{"file":"crates/aprender-serve/src/cuda/executor/layers/cublas_prefill/attention.rs","line":993},{"file":"crates/aprender-serve/src/cuda/executor/layers/cublas_prefill/attention.rs","line":994},{"file":"crates/aprender-serve/src/cuda/executor/layers/cublas_prefill/attention.rs","line":995},{"file":"crates/aprender-serve/src/cuda/executor/layers/cublas_prefill/attention.rs","line":996},{"file":"crates/aprender-serve/src/cuda/executor/layers/cublas_prefill/attention.rs","line":997},{"file":"crates/aprender-serve/src/cuda/executor/layers/cublas_prefill/attention.rs","line":998},{"file":"crates/aprender-serve/src/cuda/executor/layers/cublas_prefill/attention.rs","line":1009},{"file":"crates/aprender-serve/src/cuda/executor/layers/cublas_prefill/attention.rs","line":1010},{"file":"crates/aprender-serve/src/cuda/executor/layers/cublas_prefill/attention.rs","line":1011},{"file":"crates/aprender-serve/src/cuda/executor/layers/cublas_prefill/attention.rs","line":1012},{"file":"crates/aprender-serve/src/cuda/executor/layers/cublas_prefill/attention.rs","line":1013},{"file":"crates/aprender-serve/src/cuda/executor/layers/cublas_prefill/attention.rs","line":1014},{"file":"crates/aprender-serve/src/cuda/executor/layers/cublas_prefill/attention.rs","line":1015},{"file":"crates/aprender-serve/src/cuda/executor/layers/cublas_prefill/attention.rs","line":1016},{"file":"crates/aprender-serve/src/cuda/executor/layers/cublas_prefill/attention.rs","line":1017},{"file":"crates/aprender-serve/src/cuda/executor/layers/cublas_prefill/attention.rs","line":1018},{"file":"crates/aprender-serve/src/cuda/executor/layers/cublas_prefill/attention.rs","line":1019},{"file":"crates/aprender-serve/src/cuda/executor/layers/cublas_prefill/attention.rs","line":1020},{"file":"crates/aprender-serve/src/cuda/executor/layers/cublas_prefill/attention.rs","line":1021},{"file":"crates/aprender-serve/src/cuda/executor/layers/cublas_prefill/attention.rs","line":1022},{"file":"crates/aprender-serve/src/cuda/executor/layers/cublas_prefill/attention.rs","line":1023},{"file":"crates/aprender-serve/src/cuda/executor/layers/cublas_prefill/attention.rs","line":1024},{"file":"crates/aprender-serve/src/cuda/executor/layers/cublas_prefill/attention.rs","line":1036},{"file":"crates/aprender-serve/src/cuda/executor/layers/cublas_prefill/attention.rs","line":1037},{"file":"crates/aprender-serve/src/cuda/executor/layers/cublas_prefill/attention.rs","line":1038},{"file":"crates/aprender-serve/src/cuda/executor/layers/cublas_prefill/attention.rs","line":1039},{"file":"crates/aprender-serve/src/cuda/executor/layers/cublas_prefill/attention.rs","line":1040},{"file":"crates/aprender-serve/src/cuda/executor/layers/cublas_prefill/attention.rs","line":1041},{"file":"crates/aprender-serve/src/cuda/executor/layers/cublas_prefill/attention.rs","line":1042},{"file":"crates/aprender-serve/src/cuda/executor/layers/cublas_prefill/attention.rs","line":1043},{"file":"crates/aprender-serve/src/cuda/executor/layers/cublas_prefill/attention.rs","line":1044},{"file":"crates/aprender-serve/src/cuda/executor/layers/cublas_prefill/attention.rs","line":1045},{"file":"crates/aprender-serve/src/cuda/executor/layers/cublas_prefill/attention.rs","line":1046},{"file":"crates/aprender-serve/src/cuda/executor/layers/cublas_prefill/attention.rs","line":1047},{"file":"crates/aprender-serve/src/cuda/executor/layers/cublas_prefill/attention.rs","line":1048},{"file":"crates/aprender-serve/src/cuda/executor/layers/cublas_prefill/attention.rs","line":1049},{"file":"crates/aprender-serve/src/cuda/executor/layers/cublas_prefill/attention.rs","line":1050},{"file":"crates/aprender-serve/src/cuda/executor/layers/cublas_prefill/attention.rs","line":1062},{"file":"crates/aprender-serve/src/cuda/executor/layers/cublas_prefill/attention.rs","line":1063},{"file":"crates/aprender-serve/src/cuda/executor/layers/cublas_prefill/attention.rs","line":1064},{"file":"crates/aprender-serve/src/cuda/executor/layers/cublas_prefill/attention.rs","line":1065},{"file":"crates/aprender-serve/src/cuda/executor/layers/cublas_prefill/attention.rs","line":1066},{"file":"crates/aprender-serve/src/cuda/executor/layers/cublas_prefill/attention.rs","line":1067},{"file":"crates/aprender-serve/src/cuda/executor/layers/cublas_prefill/attention.rs","line":1068},{"file":"crates/aprender-serve/src/cuda/executor/layers/cublas_prefill/attention.rs","line":1069},{"file":"crates/aprender-serve/src/cuda/executor/layers/cublas_prefill/attention.rs","line":1070},{"file":"crates/aprender-serve/src/cuda/executor/layers/cublas_prefill/attention.rs","line":1071},{"file":"crates/aprender-serve/src/cuda/executor/layers/cublas_prefill/attention.rs","line":1072},{"file":"crates/aprender-serve/src/cuda/executor/layers/cublas_prefill/attention.rs","line":1073},{"file":"crates/aprender-serve/src/cuda/executor/layers/cublas_prefill/attention.rs","line":1074},{"file":"crates/aprender-serve/src/cuda/executor/layers/cublas_prefill/attention.rs","line":1085},{"file":"crates/aprender-serve/src/cuda/executor/layers/cublas_prefill/attention.rs","line":1086},{"file":"crates/aprender-serve/src/cuda/executor/layers/cublas_prefill/attention.rs","line":1087},{"file":"crates/aprender-serve/src/cuda/executor/layers/cublas_prefill/attention.rs","line":1088},{"file":"crates/aprender-serve/src/cuda/executor/layers/cublas_prefill/attention.rs","line":1089},{"file":"crates/aprender-serve/src/cuda/executor/layers/cublas_prefill/attention.rs","line":1090},{"file":"crates/aprender-serve/src/cuda/executor/layers/cublas_prefill/attention.rs","line":1091},{"file":"crates/aprender-serve/src/cuda/executor/layers/cublas_prefill/attention.rs","line":1092},{"file":"crates/aprender-serve/src/cuda/executor/layers/cublas_prefill/attention.rs","line":1093},{"file":"crates/aprender-serve/src/cuda/executor/layers/cublas_prefill/attention.rs","line":1094},{"file":"crates/aprender-serve/src/lib.rs","line":229}]} +{"id":"R-pr3006","class":"R","stratum":"M","split":"Dev","diff_sha256":"721f0797d1066228494794863ec8d11984929af898e6fc7db21857fb3d81e5bb","bytes":11181,"approx_tokens":2796,"source":"pr#3006 63effd9adeef12e2328bb01edaadcc452e667481 reversed; fixes #2873","defect":[{"file":"/dev/null","line":0},{"file":"scripts/check_no_competing_harnesses.sh","line":46}]} +{"id":"R-pr3008","class":"R","stratum":"M","split":"Test","diff_sha256":"22290559256405944315157b0e6d611a4415f6052c21d18a8c5387116305a1c1","bytes":8237,"approx_tokens":2060,"source":"pr#3008 9f5523548f8399f6bc3155a4fd95f26b93f50677 reversed; fixes #2873","defect":[{"file":"crates/apr-cli/src/commands/test_llm_band.rs","line":451},{"file":"crates/apr-cli/src/commands/test_llm_band.rs","line":993},{"file":"crates/apr-cli/src/commands/test_llm_band.rs","line":1164},{"file":"crates/apr-cli/src/commands/test_llm_band.rs","line":1227},{"file":"crates/apr-cli/src/commands/test_llm_band.rs","line":1366},{"file":"crates/apr-cli/src/commands/test_llm_band.rs","line":1458},{"file":"crates/apr-cli/src/commands/test_llm_band.rs","line":1739},{"file":"crates/apr-cli/src/commands/test_llm_band.rs","line":2584}]} +{"id":"R-pr3030","class":"R","stratum":"S","split":"Test","diff_sha256":"93b5a5a36ddef241e29303b75109556e96476c13e674899cba7e98b2556b5b1c","bytes":7709,"approx_tokens":1928,"source":"pr#3030 e4b423790a66d1241697ce586bd5f550225e77dc reversed; fixes #3028","defect":[{"file":"/dev/null","line":0},{"file":"scripts/check_roadmap_diff_additive.sh","line":201}]} +{"id":"R-pr3048","class":"R","stratum":"M","split":"Test","diff_sha256":"ed3ae4adb04eb328661567498dd6f184c8aff2315d10efcd5b3ced5521462200","bytes":8215,"approx_tokens":2054,"source":"pr#3048 3b597d62aabdca1f9df4f9fdc10cc97d535cbe1b reversed; fixes #3047","defect":[{"file":"scripts/check_roadmap_diff_additive.sh","line":407},{"file":"scripts/check_roadmap_diff_additive.sh","line":415}]} +{"id":"R-pr3106","class":"R","stratum":"M","split":"Test","diff_sha256":"14e23cce1103258285ba91ff13086eb0b557225a965c60475ebf5bc7312f54c6","bytes":8536,"approx_tokens":2134,"source":"pr#3106 a75e154601bf7b8bee3654489792589093443b72 reversed; fixes #3105","defect":[{"file":"crates/aprender-serve/src/quantize/bsum_precompute.rs","line":185},{"file":"crates/aprender-serve/src/quantize/bsum_precompute.rs","line":186},{"file":"crates/aprender-serve/src/quantize/fused_k_tests_q4k.rs","line":446},{"file":"crates/aprender-serve/src/quantize/requires.rs","line":130},{"file":"crates/aprender-serve/src/quantize/requires.rs","line":131},{"file":"crates/aprender-serve/src/quantize/requires.rs","line":132},{"file":"crates/aprender-serve/src/quantize/requires.rs","line":154},{"file":"crates/aprender-serve/src/quantize/requires.rs","line":159},{"file":"crates/aprender-serve/src/quantize/requires.rs","line":160}]} +{"id":"R-pr3214","class":"R","stratum":"M","split":"Dev","diff_sha256":"4388b2ca95cdebbb23b914215eaafa2f52f1681f474bdd153a10c1dacde18dfd","bytes":12917,"approx_tokens":3230,"source":"pr#3214 5be32ef79cae32dfe7c56d79020575cbe261ce33 reversed; fixes #3213","defect":[{"file":"scripts/check_shell_lint_ratchet.sh","line":104},{"file":"scripts/check_shell_lint_ratchet.sh","line":105},{"file":"scripts/check_tool_versions.sh","line":99},{"file":"scripts/check_tool_versions.sh","line":112},{"file":"scripts/check_tool_versions.sh","line":123},{"file":"scripts/check_tool_versions.sh","line":138},{"file":"scripts/check_tool_versions.sh","line":203},{"file":"scripts/check_tool_versions.sh","line":238}]} +{"id":"R-pr3216","class":"R","stratum":"M","split":"Test","diff_sha256":"8040b8896518ee29c7f4b6b1803f23d4ff12bb1185a950eb93aac5d52a9049f0","bytes":16608,"approx_tokens":4152,"source":"pr#3216 d42232c6a0a155995c9631f44bc0870f99898f04 reversed; fixes #3215","defect":[{"file":"scripts/check_verifier_pinning.sh","line":95},{"file":"scripts/check_verifier_pinning.sh","line":875},{"file":"scripts/check_verifier_pinning.sh","line":1005},{"file":"scripts/check_verifier_pinning.sh","line":1023},{"file":"scripts/check_verifier_pinning.sh","line":1038},{"file":"scripts/check_verifier_pinning.sh","line":1049},{"file":"scripts/check_verifier_pinning.sh","line":1059},{"file":"scripts/check_verifier_pinning.sh","line":1069},{"file":"scripts/check_verifier_pinning.sh","line":1278},{"file":"scripts/pv_bin.sh","line":545},{"file":"scripts/pv_bin.sh","line":555},{"file":"scripts/pv_bin.sh","line":556},{"file":"scripts/verifier_pin.sh","line":173},{"file":"scripts/verifier_pin.sh","line":180}]} +{"id":"R-pr3265","class":"R","stratum":"L","split":"Dev","diff_sha256":"2f1f58e4f6324f01a09a86b79732605fb71d8254f8427a00f68cfb47ab08f20d","bytes":46239,"approx_tokens":11560,"source":"pr#3265 13243a6baf8838c9cb668a6563dafb46b06d1c9d reversed; fixes #3262","defect":[{"file":"crates/aprender-present-lib/src/webgpu.rs","line":1166},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":208},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":252},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":253},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":254},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":255},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":256},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":257},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":258},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":259},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":260},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":261},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":262},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":263},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":264},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":265},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":266},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":267},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":268},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":269},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":270},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":271},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":272},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":273},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":274},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":275},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":276},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":277},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":278},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":279},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":280},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":281},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":326},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":327},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":328},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":329},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":330},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":331},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":332},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":333},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":334},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":335},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":336},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":337},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":338},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":339},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":340},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":341},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":342},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":343},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":344},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":345},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":346},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":347},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":348},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":349},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":350},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":351},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":352},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":353},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":354},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":355},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":356},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":357},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":358},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":359},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":360},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":361},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":362},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":363},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":364},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":365},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":366},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":367},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":368},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":369},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":370},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":371},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":372},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":373},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":374},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":375},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":376},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":377},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":378},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":379},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":380},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":381},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":382},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":383},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":384},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":385},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":386},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":387},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":388},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":389},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":390},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":391},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":392},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":393},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":394},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":395},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":396},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":397},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":398},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":399},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":400},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":401},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":402},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":403},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":404},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":405},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":406},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":407},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":408},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":409},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":410},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":411},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":412},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":413},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":414},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":415},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":416},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":417},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":418},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":419},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":420},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":421},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":422},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":423},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":424},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":425},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":426},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":427},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":428},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":429},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":430},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":431},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":432},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":433},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":434},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":435},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":436},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":437},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":438},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":440},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":441},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":442},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":443},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":444},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":445},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":446},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":447},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":448},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":449},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":450},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":451},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":452},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":453},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":454},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":455},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":456},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":457},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":458},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":459},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":460},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":461},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":462},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":463},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":464},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":465},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":466},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":467},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":468},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":469},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":470},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":471},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":474},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":475},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":476},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":477},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":478},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":479},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":480},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":481},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":483},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":484},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":485},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":486},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":487},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":527},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":528},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":529},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":531},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":532},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":533},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":534},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":536},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":537},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":539},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":540},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":541},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":542},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":543},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":544},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":545},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":546},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":547},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":548},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":549},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":550},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":551},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":558},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":559},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":560},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":561},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":562},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":563},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":564},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":565},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":567},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":569},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":572},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":573},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":574},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":575},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":576},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":577},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":578},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":580},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":581},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":582},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":583},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":586},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":587},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":588},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":589},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":590},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":591},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":592},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":593},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":594},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":595},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":596},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":597},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":598},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":599},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":604},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":605},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":606},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":607},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":608},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":609},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":610},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":611},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":612},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":613},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":614},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":615},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":616},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":617},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":618},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":619},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":620},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":621},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":622},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":623},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":624},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":629},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":630},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":631},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":632},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":633},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":634},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":635},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":636},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":637},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":638},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":639},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":640},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":641},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":642},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":643},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":644},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":645},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":646},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":647},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":648},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":654},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":655},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":656},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":657},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":658},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":660},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":661},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":662},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":663},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":664},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":665},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":666},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":667},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":668},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":669},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":670},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":671},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":672},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":673},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":674},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":679},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":680},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":681},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":682},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":683},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":684},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":685},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":686},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":687},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":688},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":689},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":690},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":691},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":692},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":693},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":694},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":695},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":696},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":697},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":698},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":704},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":705},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":706},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":707},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":708},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":710},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":711},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":712},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":713},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":714},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":715},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":716},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":717},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":718},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":719},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":720},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":721},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":722},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":723},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":724},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":729},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":730},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":731},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":732},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":733},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":735},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":736},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":737},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":738},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":739},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":740},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":741},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":742},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":743},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":744},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":745},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":746},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":747},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":748},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":749},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":754},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":755},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":756},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":757},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":758},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":761},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":762},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":763},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":764},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":765},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":766},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":767},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":768},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":769},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":770},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":771},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":772},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":773},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":774},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":775},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":780},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":781},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":782},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":783},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":784},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":787},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":788},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":789},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":790},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":791},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":792},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":793},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":794},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":795},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":796},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":797},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":798},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":799},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":800},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":801},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":1220},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":1297},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":1298},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":1299},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":1300},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":1301},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":1302},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":1303},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":1304},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":1305},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":1306},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":1307},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":1308},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":1309},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":1310},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":1311},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":1312},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":1321},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":1322},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":1323},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":1324},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":1325},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":1326},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":1327},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":1349},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":1350},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":1351},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":1352},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":1353},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":1354},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":1355},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":1356},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":1357},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":1358},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":1359},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":1360},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":1361},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":1477},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":1621},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":1656},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":1657},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":1658},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":1659},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":1660},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":1661},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":1662},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":1663},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":1664},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":1665},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":1666},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":1667},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":1668},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":1669},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":1670},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":1671},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":1672},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":1673},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":1674},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":1675},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":1676},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":1677},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":1678},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":1679},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":1680},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":1681},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":1682},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":1683},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":1684},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":1685},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":1686},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":1687},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":1688},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":1689},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":1690},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":1691},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":1692},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":1693},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":1694},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":1695},{"file":"crates/aprender-test-cli/src/handlers/llm.rs","line":1696}]} +{"id":"R-pr3266","class":"R","stratum":"S","split":"Dev","diff_sha256":"55944295a73710632b153407def331f7f4ffd34a1338149b062519cda7835aa3","bytes":1173,"approx_tokens":294,"source":"pr#3266 e0706bd56dc2fd15f81006b2bc89d095ebb6f20c reversed; fixes #3262","defect":[{"file":"crates/aprender-distribute/src/executor/cpu.rs","line":39},{"file":"crates/aprender-distribute/src/executor/cpu.rs","line":41}]} +{"id":"R-pr3301","class":"R","stratum":"S","split":"Dev","diff_sha256":"55c0b3dc9677039d2d26f51a8622d30631f4eb2612a47fcefb8d8909bee094a5","bytes":6943,"approx_tokens":1736,"source":"pr#3301 4bad830fdeb6d28947a9635f753cf07005665a13 reversed; fixes #3300","defect":[{"file":"crates/aprender-compute/src/registry/wgpu_probe.rs","line":35},{"file":"crates/aprender-compute/src/registry/wgpu_probe.rs","line":36},{"file":"crates/aprender-compute/src/registry/wgpu_probe.rs","line":37},{"file":"crates/aprender-compute/src/registry/wgpu_probe.rs","line":43},{"file":"crates/aprender-compute/src/registry/wgpu_probe.rs","line":44},{"file":"crates/aprender-compute/src/registry/wgpu_probe.rs","line":51},{"file":"crates/aprender-compute/src/registry/wgpu_probe.rs","line":52},{"file":"crates/aprender-compute/src/registry/wgpu_probe.rs","line":59},{"file":"crates/aprender-compute/src/registry/wgpu_probe.rs","line":63},{"file":"crates/aprender-compute/src/registry/wgpu_probe.rs","line":64},{"file":"crates/aprender-compute/src/registry/wgpu_probe.rs","line":65},{"file":"crates/aprender-compute/src/registry/wgpu_probe.rs","line":75},{"file":"crates/aprender-compute/src/registry/wgpu_probe.rs","line":76},{"file":"crates/aprender-compute/src/registry/wgpu_probe.rs","line":79},{"file":"crates/aprender-compute/src/registry/wgpu_probe.rs","line":80},{"file":"crates/aprender-compute/src/registry/wgpu_probe.rs","line":81},{"file":"crates/aprender-compute/src/registry/wgpu_probe.rs","line":82},{"file":"crates/aprender-compute/src/registry/wgpu_probe.rs","line":83},{"file":"crates/aprender-compute/src/registry/wgpu_probe.rs","line":88}]} +{"id":"R-pr3307","class":"R","stratum":"M","split":"Test","diff_sha256":"2a572c8c6487e7636101bc8c774d3ef8a68d883bd76667f6cbc9a4e06dac7931","bytes":10763,"approx_tokens":2691,"source":"pr#3307 8563f2b0230e553c08a92fdfeb66ff8218b2113c reversed; fixes #3189,#3305","defect":[{"file":"/dev/null","line":0},{"file":"scripts/check_baseline_ratchets.sh","line":90}]} +{"id":"R-pr3309","class":"R","stratum":"S","split":"Test","diff_sha256":"94f1d098ccbd15f33f168d6eda10f9615ccf7750128b8d163e6cb9f15cea9978","bytes":5228,"approx_tokens":1307,"source":"pr#3309 03dfa8f8fae3a8052f9ec800e132abdea05de2e4 reversed; fixes #3308","defect":[{"file":"scripts/ci_resolve_dirty.sh","line":34},{"file":"scripts/ci_resolve_dirty.sh","line":109},{"file":"scripts/ci_resolve_dirty.sh","line":375},{"file":"scripts/ci_resolve_dirty.sh","line":399},{"file":"scripts/ci_resolve_dirty.sh","line":422},{"file":"scripts/ci_resolve_dirty.sh","line":433}]} +{"id":"R-pr3319","class":"R","stratum":"S","split":"Test","diff_sha256":"fbb3dd43cf1fbe16ce485c6938a30d3c354ac4dd64f176ab3bd3e88ca6a14a8f","bytes":5166,"approx_tokens":1292,"source":"pr#3319 82b43992c134e6006c6a153acec81db64c218eb9 reversed; fixes #3247,#3317","defect":[{"file":"scripts/check_vendored_schemas.sh","line":58},{"file":"scripts/check_vendored_schemas.sh","line":104},{"file":"scripts/check_vendored_schemas.sh","line":118},{"file":"scripts/check_vendored_schemas.sh","line":162},{"file":"scripts/check_vendored_schemas.sh","line":182},{"file":"scripts/check_vendored_schemas.sh","line":198},{"file":"scripts/check_vendored_schemas.sh","line":367},{"file":"scripts/check_vendored_schemas.sh","line":403}]} +{"id":"R-pr3339","class":"R","stratum":"L","split":"Test","diff_sha256":"31378d158e0701a2ce373cff30d97037736798a2a0697996443a9769e2fb6812","bytes":42497,"approx_tokens":10625,"source":"pr#3339 334646674d8b79a1c7e326a343aa78d0bb129833 reversed; fixes #3337","defect":[{"file":"scripts/check_silicon_coverage.sh","line":65},{"file":"scripts/check_silicon_coverage.sh","line":88},{"file":"scripts/check_silicon_coverage.sh","line":89},{"file":"scripts/check_silicon_coverage.sh","line":90},{"file":"scripts/check_silicon_coverage.sh","line":91},{"file":"scripts/check_silicon_coverage.sh","line":94},{"file":"scripts/check_silicon_coverage.sh","line":158},{"file":"scripts/check_silicon_coverage.sh","line":212},{"file":"scripts/check_silicon_coverage.sh","line":213},{"file":"scripts/check_silicon_coverage.sh","line":217},{"file":"scripts/check_silicon_coverage.sh","line":218},{"file":"scripts/check_silicon_coverage.sh","line":219},{"file":"scripts/check_silicon_coverage.sh","line":220},{"file":"scripts/check_silicon_coverage.sh","line":221},{"file":"scripts/check_silicon_coverage.sh","line":222},{"file":"scripts/check_silicon_coverage.sh","line":230},{"file":"scripts/check_silicon_coverage.sh","line":231},{"file":"scripts/check_silicon_coverage.sh","line":232},{"file":"scripts/check_silicon_coverage.sh","line":234},{"file":"scripts/check_silicon_coverage.sh","line":235},{"file":"scripts/check_silicon_coverage.sh","line":236},{"file":"scripts/check_silicon_coverage.sh","line":237},{"file":"scripts/check_silicon_coverage.sh","line":238},{"file":"scripts/check_silicon_coverage.sh","line":239},{"file":"scripts/check_silicon_coverage.sh","line":240},{"file":"scripts/check_silicon_coverage.sh","line":241},{"file":"scripts/check_silicon_coverage.sh","line":242},{"file":"scripts/check_silicon_coverage.sh","line":243},{"file":"scripts/check_silicon_coverage.sh","line":244},{"file":"scripts/check_silicon_coverage.sh","line":245},{"file":"scripts/check_silicon_coverage.sh","line":246},{"file":"scripts/check_silicon_coverage.sh","line":247},{"file":"scripts/check_silicon_coverage.sh","line":248},{"file":"scripts/check_silicon_coverage.sh","line":249},{"file":"scripts/check_silicon_coverage.sh","line":250},{"file":"scripts/check_silicon_coverage.sh","line":251},{"file":"scripts/check_silicon_coverage.sh","line":252},{"file":"scripts/check_silicon_coverage.sh","line":253},{"file":"scripts/check_silicon_coverage.sh","line":254},{"file":"scripts/check_silicon_coverage.sh","line":255},{"file":"scripts/check_silicon_coverage.sh","line":256},{"file":"scripts/check_silicon_coverage.sh","line":257},{"file":"scripts/check_silicon_coverage.sh","line":259},{"file":"scripts/check_silicon_coverage.sh","line":260},{"file":"scripts/check_silicon_coverage.sh","line":261},{"file":"scripts/check_silicon_coverage.sh","line":267},{"file":"scripts/check_silicon_coverage.sh","line":269},{"file":"scripts/check_silicon_coverage.sh","line":272},{"file":"scripts/check_silicon_coverage.sh","line":273},{"file":"scripts/check_silicon_coverage.sh","line":274},{"file":"scripts/check_silicon_coverage.sh","line":285},{"file":"scripts/check_silicon_coverage.sh","line":297},{"file":"scripts/check_silicon_coverage.sh","line":313},{"file":"scripts/check_silicon_coverage.sh","line":315},{"file":"scripts/check_silicon_coverage.sh","line":316},{"file":"scripts/check_silicon_coverage.sh","line":337},{"file":"scripts/check_silicon_coverage.sh","line":338},{"file":"scripts/check_silicon_coverage.sh","line":340},{"file":"scripts/check_silicon_coverage.sh","line":346},{"file":"scripts/check_silicon_coverage.sh","line":347},{"file":"scripts/check_silicon_coverage.sh","line":362},{"file":"scripts/check_silicon_coverage.sh","line":363},{"file":"scripts/check_silicon_coverage.sh","line":364},{"file":"scripts/check_silicon_coverage.sh","line":365},{"file":"scripts/check_silicon_coverage.sh","line":366},{"file":"scripts/check_silicon_coverage.sh","line":367},{"file":"scripts/check_silicon_coverage.sh","line":368},{"file":"scripts/check_silicon_coverage.sh","line":369},{"file":"scripts/check_silicon_coverage.sh","line":370},{"file":"scripts/check_silicon_coverage.sh","line":371},{"file":"scripts/check_silicon_coverage.sh","line":372},{"file":"scripts/check_silicon_coverage.sh","line":373},{"file":"scripts/check_silicon_coverage.sh","line":374},{"file":"scripts/check_silicon_coverage.sh","line":375},{"file":"scripts/check_silicon_coverage.sh","line":376},{"file":"scripts/check_silicon_coverage.sh","line":377},{"file":"scripts/check_silicon_coverage.sh","line":378},{"file":"scripts/check_silicon_coverage.sh","line":381},{"file":"scripts/check_silicon_coverage.sh","line":382},{"file":"scripts/check_silicon_coverage.sh","line":383},{"file":"scripts/check_silicon_coverage.sh","line":384},{"file":"scripts/check_silicon_coverage.sh","line":386},{"file":"scripts/check_silicon_coverage.sh","line":440},{"file":"scripts/check_silicon_coverage.sh","line":447},{"file":"scripts/check_silicon_coverage.sh","line":467},{"file":"scripts/check_silicon_coverage.sh","line":483},{"file":"scripts/check_silicon_coverage.sh","line":484},{"file":"scripts/check_silicon_coverage.sh","line":508},{"file":"scripts/check_silicon_coverage.sh","line":523}]} +{"id":"R-pr3396","class":"R","stratum":"S","split":"Dev","diff_sha256":"dd13f723671a980a1fbd6b1330343bd001ba826fa30372a1c42bb67e04bfcd93","bytes":7039,"approx_tokens":1760,"source":"pr#3396 ce4f661ea8fb5c9e4e051b046cc82847ca16c50b reversed; fixes #3367","defect":[{"file":"crates/apr-cli/src/commands/chat_04.rs","line":12},{"file":"crates/apr-cli/src/commands/chat_generate_safetensors.rs","line":91},{"file":"crates/apr-cli/src/commands/chat_generate_session.rs","line":42},{"file":"crates/apr-cli/src/commands/chat_generate_session.rs","line":86},{"file":"crates/apr-cli/src/commands/chat_generate_session_02.rs","line":8},{"file":"crates/apr-cli/src/commands/chat_generate_session_02.rs","line":37},{"file":"crates/apr-cli/src/commands/chat_generate_session_02.rs","line":94},{"file":"crates/apr-cli/src/commands/chat_session.rs","line":50},{"file":"crates/apr-cli/src/commands/chat_session_02.rs","line":140}]} +{"id":"R-pr3405","class":"R","stratum":"M","split":"Test","diff_sha256":"9e6ef22b3d82a12aca58a6849123a39bb01af67c817f34f68585651f0bec5847","bytes":9553,"approx_tokens":2389,"source":"pr#3405 102a154c9bb457f0efd24d04c36a1c73d5d06d9a reversed; fixes #3341,#3368,#3369","defect":[{"file":"crates/aprender-serve/src/gguf/qwen3_moe_load.rs","line":37},{"file":"crates/aprender-serve/src/gguf/qwen3_moe_load.rs","line":70},{"file":"crates/aprender-serve/src/gguf/qwen3_moe_load.rs","line":76},{"file":"crates/aprender-serve/src/gguf/qwen3_moe_load.rs","line":94},{"file":"crates/aprender-serve/src/gguf/qwen3_moe_load.rs","line":115},{"file":"crates/aprender-serve/src/gguf/qwen3_moe_load.rs","line":311},{"file":"crates/aprender-serve/src/gguf/qwen3_moe_load.rs","line":312},{"file":"crates/aprender-serve/src/gguf/qwen3_moe_load.rs","line":317},{"file":"crates/aprender-serve/src/gguf/qwen3_moe_load.rs","line":318},{"file":"crates/aprender-serve/src/gguf/qwen3_moe_load.rs","line":319},{"file":"crates/aprender-serve/src/gguf/qwen3_moe_load.rs","line":320},{"file":"crates/aprender-serve/src/gguf/qwen3_moe_load.rs","line":321},{"file":"crates/aprender-serve/src/gguf/qwen3_moe_load.rs","line":322},{"file":"crates/aprender-serve/src/gguf/qwen3_moe_load.rs","line":323},{"file":"crates/aprender-serve/src/gguf/qwen3_moe_load.rs","line":324},{"file":"crates/aprender-serve/src/gguf/qwen3_moe_load.rs","line":809}]} +{"id":"R-pr3412","class":"R","stratum":"M","split":"Dev","diff_sha256":"8973b51129e6a50921bb673ba4a0ab7a169e6b250e26196a66d9720b64c995bb","bytes":18499,"approx_tokens":4625,"source":"pr#3412 4db5ab6aa86030790a23c805645a903da69083be reversed; fixes #3367","defect":[{"file":"/dev/null","line":0},{"file":"crates/apr-cli/src/commands/parity.rs","line":471},{"file":"crates/apr-cli/src/commands/parity_03.rs","line":40},{"file":"crates/apr-cli/src/commands/parity_03.rs","line":51},{"file":"scripts/check_model_parity.sh","line":73},{"file":"scripts/check_model_parity.sh","line":111},{"file":"scripts/check_model_parity.sh","line":144}]} +{"id":"R-pr3469","class":"R","stratum":"S","split":"Test","diff_sha256":"006cbf09fdb08a2b2074eabd6195675d798d416f64b23cc9cd798de1f35aa4f1","bytes":5811,"approx_tokens":1453,"source":"pr#3469 49c52cc052b7dac9310eddd05e66f94d2641bb29 reversed; fixes #3468","defect":[{"file":"scripts/check_publish_preflight.sh","line":172},{"file":"scripts/check_publish_preflight.sh","line":173},{"file":"scripts/check_publish_preflight.sh","line":174},{"file":"scripts/check_publish_preflight.sh","line":175},{"file":"scripts/check_publish_preflight.sh","line":182},{"file":"scripts/check_publish_preflight.sh","line":183},{"file":"scripts/check_publish_preflight.sh","line":184},{"file":"scripts/check_publish_preflight.sh","line":185},{"file":"scripts/check_publish_preflight.sh","line":186},{"file":"scripts/check_publish_preflight.sh","line":188},{"file":"scripts/check_publish_preflight.sh","line":189},{"file":"scripts/check_publish_preflight.sh","line":190},{"file":"scripts/check_publish_preflight.sh","line":195},{"file":"scripts/check_publish_preflight.sh","line":196},{"file":"scripts/check_publish_preflight.sh","line":250},{"file":"scripts/check_publish_preflight.sh","line":251},{"file":"scripts/check_publish_preflight.sh","line":350},{"file":"scripts/check_publish_preflight.sh","line":352}]} +{"id":"R-pr3481","class":"R","stratum":"S","split":"Test","diff_sha256":"158b3a19aec4fd5b17d87196981f3df79c1519b94d8d5c6109066ce3d06495ba","bytes":4302,"approx_tokens":1076,"source":"pr#3481 3409b29d22d8be7ae04334130ccc01877b967a1d reversed; fixes #3477","defect":[{"file":"scripts/check_model_ladder.sh","line":97},{"file":"scripts/model_ladder.sh","line":126},{"file":"scripts/model_ladder.sh","line":127},{"file":"scripts/model_ladder.sh","line":163},{"file":"scripts/model_ladder.sh","line":178}]} +{"id":"R-pr3497","class":"R","stratum":"S","split":"Dev","diff_sha256":"35037ece999ddd2e1ea46ddd84667896a931db51a96d4896e730e2d9be0b6436","bytes":5028,"approx_tokens":1257,"source":"pr#3497 e9962a9b743e444d6eb1fe818309289c23b75522 reversed; fixes #3400","defect":[{"file":"scripts/check_pr_closes_issue.sh","line":44},{"file":"scripts/check_pr_closes_issue.sh","line":119},{"file":"scripts/check_pr_closes_issue.sh","line":146},{"file":"scripts/check_pr_closes_issue.sh","line":148},{"file":"scripts/check_pr_closes_issue.sh","line":149},{"file":"scripts/check_pr_closes_issue.sh","line":153},{"file":"scripts/check_pr_closes_issue.sh","line":195},{"file":"scripts/check_pr_closes_issue.sh","line":196},{"file":"scripts/check_pr_closes_issue.sh","line":218},{"file":"scripts/check_pr_closes_issue.sh","line":219}]} +{"id":"R-pr3530","class":"R","stratum":"M","split":"Dev","diff_sha256":"ee157cd1b08359328d7c257a6c4f5b11c701ee2d87c91e648ff7dbcfab6df0b6","bytes":14096,"approx_tokens":3524,"source":"pr#3530 860b02814ffbd7f476e96aa522462e157975397b reversed; fixes #3529","defect":[{"file":"crates/aprender-contracts/src/ontology/extract/pv_contract.rs","line":74},{"file":"crates/aprender-contracts/src/ontology/extract/pv_contract.rs","line":96},{"file":"crates/aprender-contracts/src/ontology/extract/pv_contract.rs","line":137},{"file":"crates/aprender-contracts/src/ontology/shapes.rs","line":581},{"file":"crates/aprender-contracts/src/ontology/shapes.rs","line":585},{"file":"crates/aprender-contracts/src/ontology/shapes.rs","line":595},{"file":"crates/aprender-contracts/src/ontology/shapes.rs","line":605},{"file":"crates/aprender-contracts/src/ontology/shapes.rs","line":613},{"file":"crates/aprender-contracts/src/ontology/sigma.rs","line":48},{"file":"crates/aprender-contracts/src/ontology/sigma.rs","line":177},{"file":"crates/aprender-contracts/src/ontology/sigma.rs","line":186},{"file":"crates/aprender-contracts/src/ontology/sigma.rs","line":292},{"file":"crates/aprender-contracts/src/ontology/sigma.rs","line":356}]} +{"id":"R-pr3706","class":"R","stratum":"M","split":"Test","diff_sha256":"45670c30d023bff5378b5638b90e7e8ab609c1f2d9e062063fb449b56798b572","bytes":12092,"approx_tokens":3023,"source":"pr#3706 52f43da71745eedacde2f86e1903ea81d6fa34ad reversed; fixes #3704","defect":[{"file":"crates/aprender-contracts/src/lint/shapes_gate.rs","line":25},{"file":"crates/aprender-contracts/src/lint/shapes_gate.rs","line":26},{"file":"crates/aprender-contracts/src/lint/shapes_gate.rs","line":33},{"file":"crates/aprender-contracts/src/lint/shapes_gate.rs","line":381},{"file":"crates/aprender-contracts/src/lint/shapes_gate.rs","line":801},{"file":"crates/aprender-contracts/src/ontology/extract/json.rs","line":163},{"file":"crates/aprender-contracts/src/ontology/extract/json.rs","line":164},{"file":"crates/aprender-contracts/src/ontology/extract/json.rs","line":167},{"file":"crates/aprender-contracts/src/ontology/extract/json.rs","line":169},{"file":"crates/aprender-contracts/src/ontology/extract/json.rs","line":172},{"file":"crates/aprender-contracts/src/ontology/extract/json.rs","line":519},{"file":"crates/aprender-contracts/src/ontology/extract/parity_receipt.rs","line":271},{"file":"crates/aprender-contracts/src/ontology/extract/pv_contract.rs","line":109},{"file":"crates/aprender-contracts/src/ontology/extract/pv_contract.rs","line":295}]} +{"id":"R-pr410","class":"R","stratum":"S","split":"Test","diff_sha256":"dfd7fd3a4ae169eb2f771c286946890b0596183386e5b9f604d7b6ad3ec977f2","bytes":884,"approx_tokens":221,"source":"pr#410 41dd3cf035742989c6d492379b78eda3e08a10d4 reversed; fixes #363","defect":[{"file":"src/format/converter/safe_tensors_load_result.rs","line":257}]} +{"id":"R-pr411","class":"R","stratum":"L","split":"Test","diff_sha256":"e663897e386a4ff509930d23d8b185c1b9a17a6f0c6ee703c34b1fb2d6157aa0","bytes":94451,"approx_tokens":23613,"source":"pr#411 d6e989e73d1f533aa4c4e2dc3705d089d3934e5f reversed; fixes #409","defect":[{"file":"crates/apr-cli/src/commands/bench.rs","line":245},{"file":"crates/apr-cli/src/commands/bench.rs","line":261},{"file":"crates/apr-cli/src/commands/bench.rs","line":264},{"file":"crates/apr-cli/src/commands/bench.rs","line":265},{"file":"crates/apr-cli/src/commands/bench.rs","line":266},{"file":"crates/apr-cli/src/commands/bench.rs","line":267},{"file":"crates/apr-cli/src/commands/bench.rs","line":288},{"file":"crates/apr-cli/src/commands/bench.rs","line":289},{"file":"crates/apr-cli/src/commands/bench.rs","line":290},{"file":"crates/apr-cli/src/commands/bench.rs","line":291},{"file":"crates/apr-cli/src/commands/bench.rs","line":292},{"file":"crates/apr-cli/src/commands/bench.rs","line":293},{"file":"crates/apr-cli/src/commands/bench.rs","line":294},{"file":"crates/apr-cli/src/commands/bench.rs","line":295},{"file":"crates/apr-cli/src/commands/bench.rs","line":296},{"file":"crates/apr-cli/src/commands/bench.rs","line":297},{"file":"crates/apr-cli/src/commands/bench.rs","line":298},{"file":"crates/apr-cli/src/commands/bench.rs","line":299},{"file":"crates/apr-cli/src/commands/bench.rs","line":300},{"file":"crates/apr-cli/src/commands/bench.rs","line":301},{"file":"crates/apr-cli/src/commands/bench.rs","line":341},{"file":"crates/apr-cli/src/commands/bench.rs","line":342},{"file":"crates/apr-cli/src/commands/bench.rs","line":343},{"file":"crates/apr-cli/src/commands/bench.rs","line":344},{"file":"crates/apr-cli/src/commands/bench.rs","line":345},{"file":"crates/apr-cli/src/commands/bench.rs","line":346},{"file":"crates/apr-cli/src/commands/bench.rs","line":347},{"file":"crates/apr-cli/src/commands/bench.rs","line":348},{"file":"crates/apr-cli/src/commands/bench.rs","line":349},{"file":"crates/apr-cli/src/commands/bench.rs","line":350},{"file":"crates/apr-cli/src/commands/bench.rs","line":351},{"file":"crates/apr-cli/src/commands/bench.rs","line":352},{"file":"crates/apr-cli/src/commands/bench.rs","line":353},{"file":"crates/apr-cli/src/commands/bench.rs","line":354},{"file":"crates/apr-cli/src/commands/bench.rs","line":355},{"file":"crates/apr-cli/src/commands/bench.rs","line":356},{"file":"crates/apr-cli/src/commands/bench.rs","line":357},{"file":"crates/apr-cli/src/commands/bench.rs","line":358},{"file":"crates/apr-cli/src/commands/bench.rs","line":359},{"file":"crates/apr-cli/src/commands/bench.rs","line":360},{"file":"crates/apr-cli/src/commands/bench.rs","line":361},{"file":"crates/apr-cli/src/commands/bench.rs","line":362},{"file":"crates/apr-cli/src/commands/bench.rs","line":363},{"file":"crates/apr-cli/src/commands/bench.rs","line":364},{"file":"crates/apr-cli/src/commands/bench.rs","line":365},{"file":"crates/apr-cli/src/commands/bench.rs","line":366},{"file":"crates/apr-cli/src/commands/bench.rs","line":367},{"file":"crates/apr-cli/src/commands/bench.rs","line":368},{"file":"crates/apr-cli/src/commands/bench.rs","line":369},{"file":"crates/apr-cli/src/commands/bench.rs","line":370},{"file":"crates/apr-cli/src/commands/bench.rs","line":371},{"file":"crates/apr-cli/src/commands/bench.rs","line":372},{"file":"crates/apr-cli/src/commands/bench.rs","line":373},{"file":"crates/apr-cli/src/commands/bench.rs","line":374},{"file":"crates/apr-cli/src/commands/bench.rs","line":375},{"file":"crates/apr-cli/src/commands/bench.rs","line":376},{"file":"crates/apr-cli/src/commands/bench.rs","line":377},{"file":"crates/apr-cli/src/commands/bench.rs","line":378},{"file":"crates/apr-cli/src/commands/bench.rs","line":379},{"file":"crates/apr-cli/src/commands/bench.rs","line":380},{"file":"crates/apr-cli/src/commands/bench.rs","line":381},{"file":"crates/apr-cli/src/commands/bench.rs","line":382},{"file":"crates/apr-cli/src/commands/bench.rs","line":383},{"file":"crates/apr-cli/src/commands/bench.rs","line":384},{"file":"crates/apr-cli/src/commands/bench.rs","line":385},{"file":"crates/apr-cli/src/commands/bench.rs","line":386},{"file":"crates/apr-cli/src/commands/bench.rs","line":387},{"file":"crates/apr-cli/src/commands/bench.rs","line":388},{"file":"crates/apr-cli/src/commands/bench.rs","line":389},{"file":"crates/apr-cli/src/commands/bench.rs","line":390},{"file":"crates/apr-cli/src/commands/bench.rs","line":391},{"file":"crates/apr-cli/src/commands/bench.rs","line":392},{"file":"crates/apr-cli/src/commands/bench.rs","line":393},{"file":"crates/apr-cli/src/commands/bench.rs","line":394},{"file":"crates/apr-cli/src/commands/bench.rs","line":395},{"file":"crates/apr-cli/src/commands/bench.rs","line":396},{"file":"crates/apr-cli/src/commands/bench.rs","line":397},{"file":"crates/apr-cli/src/commands/bench.rs","line":398},{"file":"crates/apr-cli/src/commands/bench.rs","line":399},{"file":"crates/apr-cli/src/commands/bench.rs","line":400},{"file":"crates/apr-cli/src/commands/bench.rs","line":401},{"file":"crates/apr-cli/src/commands/bench.rs","line":402},{"file":"crates/apr-cli/src/commands/bench.rs","line":403},{"file":"crates/apr-cli/src/commands/bench.rs","line":404},{"file":"crates/apr-cli/src/commands/bench.rs","line":405},{"file":"crates/apr-cli/src/commands/bench.rs","line":406},{"file":"crates/apr-cli/src/commands/bench.rs","line":407},{"file":"crates/apr-cli/src/commands/bench.rs","line":408},{"file":"crates/apr-cli/src/commands/bench.rs","line":409},{"file":"crates/apr-cli/src/commands/bench.rs","line":410},{"file":"crates/apr-cli/src/commands/bench.rs","line":411},{"file":"crates/apr-cli/src/commands/bench.rs","line":412},{"file":"crates/apr-cli/src/commands/bench.rs","line":413},{"file":"crates/apr-cli/src/commands/bench.rs","line":414},{"file":"crates/apr-cli/src/commands/bench.rs","line":415},{"file":"crates/apr-cli/src/commands/bench.rs","line":416},{"file":"crates/apr-cli/src/commands/bench.rs","line":417},{"file":"crates/apr-cli/src/commands/bench.rs","line":418},{"file":"crates/apr-cli/src/commands/bench.rs","line":419},{"file":"crates/apr-cli/src/commands/bench.rs","line":420},{"file":"crates/apr-cli/src/commands/bench.rs","line":421},{"file":"crates/apr-cli/src/commands/bench.rs","line":422},{"file":"crates/apr-cli/src/commands/bench.rs","line":423},{"file":"crates/apr-cli/src/commands/bench.rs","line":424},{"file":"crates/apr-cli/src/commands/bench.rs","line":425},{"file":"crates/apr-cli/src/commands/bench.rs","line":426},{"file":"crates/apr-cli/src/commands/bench.rs","line":427},{"file":"crates/apr-cli/src/commands/bench.rs","line":428},{"file":"crates/apr-cli/src/commands/bench.rs","line":429},{"file":"crates/apr-cli/src/commands/bench.rs","line":430},{"file":"crates/apr-cli/src/commands/bench.rs","line":431},{"file":"crates/apr-cli/src/commands/bench.rs","line":432},{"file":"crates/apr-cli/src/commands/bench.rs","line":433},{"file":"crates/apr-cli/src/commands/bench.rs","line":434},{"file":"crates/apr-cli/src/commands/bench.rs","line":435},{"file":"crates/apr-cli/src/commands/bench.rs","line":436},{"file":"crates/apr-cli/src/commands/bench.rs","line":437},{"file":"crates/apr-cli/src/commands/bench.rs","line":438},{"file":"crates/apr-cli/src/commands/bench.rs","line":439},{"file":"crates/apr-cli/src/commands/bench.rs","line":440},{"file":"crates/apr-cli/src/commands/bench.rs","line":441},{"file":"crates/apr-cli/src/commands/bench.rs","line":442},{"file":"crates/apr-cli/src/commands/bench.rs","line":443},{"file":"crates/apr-cli/src/commands/bench.rs","line":444},{"file":"crates/apr-cli/src/commands/bench.rs","line":445},{"file":"crates/apr-cli/src/commands/bench.rs","line":446},{"file":"crates/apr-cli/src/commands/bench.rs","line":447},{"file":"crates/apr-cli/src/commands/bench.rs","line":448},{"file":"crates/apr-cli/src/commands/bench.rs","line":449},{"file":"crates/apr-cli/src/commands/bench.rs","line":450},{"file":"crates/apr-cli/src/commands/bench.rs","line":451},{"file":"crates/apr-cli/src/commands/bench.rs","line":452},{"file":"crates/apr-cli/src/commands/bench.rs","line":453},{"file":"crates/apr-cli/src/commands/bench.rs","line":454},{"file":"crates/apr-cli/src/commands/bench.rs","line":455},{"file":"crates/apr-cli/src/commands/bench.rs","line":456},{"file":"crates/apr-cli/src/commands/bench.rs","line":457},{"file":"crates/apr-cli/src/commands/bench.rs","line":458},{"file":"crates/apr-cli/src/commands/bench.rs","line":459},{"file":"crates/apr-cli/src/commands/bench.rs","line":460},{"file":"crates/apr-cli/src/commands/bench.rs","line":461},{"file":"crates/apr-cli/src/commands/bench.rs","line":462},{"file":"crates/apr-cli/src/commands/bench.rs","line":463},{"file":"crates/apr-cli/src/commands/bench.rs","line":464},{"file":"crates/apr-cli/src/commands/bench.rs","line":465},{"file":"crates/apr-cli/src/commands/bench.rs","line":466},{"file":"crates/apr-cli/src/commands/bench.rs","line":467},{"file":"crates/apr-cli/src/commands/bench.rs","line":468},{"file":"crates/apr-cli/src/commands/bench.rs","line":469},{"file":"crates/apr-cli/src/commands/bench.rs","line":470},{"file":"crates/apr-cli/src/commands/bench.rs","line":471},{"file":"crates/apr-cli/src/commands/bench.rs","line":472},{"file":"crates/apr-cli/src/commands/bench.rs","line":473},{"file":"crates/apr-cli/src/commands/bench.rs","line":474},{"file":"crates/apr-cli/src/commands/bench.rs","line":475},{"file":"crates/apr-cli/src/commands/bench.rs","line":476},{"file":"crates/apr-cli/src/commands/bench.rs","line":477},{"file":"crates/apr-cli/src/commands/bench.rs","line":478},{"file":"crates/apr-cli/src/commands/bench.rs","line":479},{"file":"crates/apr-cli/src/commands/bench.rs","line":480},{"file":"crates/apr-cli/src/commands/bench.rs","line":481},{"file":"crates/apr-cli/src/commands/bench.rs","line":482},{"file":"crates/apr-cli/src/commands/bench.rs","line":483},{"file":"crates/apr-cli/src/commands/bench.rs","line":484},{"file":"crates/apr-cli/src/commands/bench.rs","line":485},{"file":"crates/apr-cli/src/commands/bench.rs","line":486},{"file":"crates/apr-cli/src/commands/bench.rs","line":487},{"file":"crates/apr-cli/src/commands/bench.rs","line":488},{"file":"crates/apr-cli/src/commands/bench.rs","line":489},{"file":"crates/apr-cli/src/commands/bench.rs","line":490},{"file":"crates/apr-cli/src/commands/bench.rs","line":491},{"file":"crates/apr-cli/src/commands/bench.rs","line":492},{"file":"crates/apr-cli/src/commands/bench.rs","line":493},{"file":"crates/apr-cli/src/commands/bench.rs","line":494},{"file":"crates/apr-cli/src/commands/bench.rs","line":495},{"file":"crates/apr-cli/src/commands/bench.rs","line":496},{"file":"crates/apr-cli/src/commands/bench.rs","line":497},{"file":"crates/apr-cli/src/commands/bench.rs","line":498},{"file":"crates/apr-cli/src/commands/bench.rs","line":499},{"file":"crates/apr-cli/src/commands/bench.rs","line":500},{"file":"crates/apr-cli/src/commands/bench.rs","line":501},{"file":"crates/apr-cli/src/commands/bench.rs","line":502},{"file":"crates/apr-cli/src/commands/bench.rs","line":503},{"file":"crates/apr-cli/src/commands/bench.rs","line":504},{"file":"crates/apr-cli/src/commands/bench.rs","line":505},{"file":"crates/apr-cli/src/commands/bench.rs","line":506},{"file":"crates/apr-cli/src/commands/bench.rs","line":507},{"file":"crates/apr-cli/src/commands/bench.rs","line":508},{"file":"crates/apr-cli/src/commands/bench.rs","line":509},{"file":"crates/apr-cli/src/commands/bench.rs","line":510},{"file":"crates/apr-cli/src/commands/bench.rs","line":511},{"file":"crates/apr-cli/src/commands/bench.rs","line":512},{"file":"crates/apr-cli/src/commands/bench.rs","line":513},{"file":"crates/apr-cli/src/commands/bench.rs","line":514},{"file":"crates/apr-cli/src/commands/bench.rs","line":515},{"file":"crates/apr-cli/src/commands/bench.rs","line":516},{"file":"crates/apr-cli/src/commands/bench.rs","line":517},{"file":"crates/apr-cli/src/commands/bench.rs","line":518},{"file":"crates/apr-cli/src/commands/bench.rs","line":519},{"file":"crates/apr-cli/src/commands/bench.rs","line":520},{"file":"crates/apr-cli/src/commands/bench.rs","line":521},{"file":"crates/apr-cli/src/commands/bench.rs","line":522},{"file":"crates/apr-cli/src/commands/bench.rs","line":523},{"file":"crates/apr-cli/src/commands/bench.rs","line":524},{"file":"crates/apr-cli/src/commands/bench.rs","line":525},{"file":"crates/apr-cli/src/commands/bench.rs","line":526},{"file":"crates/apr-cli/src/commands/bench.rs","line":527},{"file":"crates/apr-cli/src/commands/bench.rs","line":528},{"file":"crates/apr-cli/src/commands/bench.rs","line":529},{"file":"crates/apr-cli/src/commands/bench.rs","line":530},{"file":"crates/apr-cli/src/commands/bench.rs","line":531},{"file":"crates/apr-cli/src/commands/bench.rs","line":532},{"file":"crates/apr-cli/src/commands/bench.rs","line":533},{"file":"crates/apr-cli/src/commands/bench.rs","line":534},{"file":"crates/apr-cli/src/commands/bench.rs","line":535},{"file":"crates/apr-cli/src/commands/bench.rs","line":536},{"file":"crates/apr-cli/src/commands/bench.rs","line":537},{"file":"crates/apr-cli/src/commands/bench.rs","line":538},{"file":"crates/apr-cli/src/commands/bench.rs","line":539},{"file":"crates/apr-cli/src/commands/bench.rs","line":540},{"file":"crates/apr-cli/src/commands/bench.rs","line":554},{"file":"crates/apr-cli/src/commands/data.rs","line":14},{"file":"crates/apr-cli/src/commands/data.rs","line":23},{"file":"crates/apr-cli/src/commands/data.rs","line":171},{"file":"crates/apr-cli/src/commands/data.rs","line":217},{"file":"crates/apr-cli/src/commands/data.rs","line":218},{"file":"crates/apr-cli/src/commands/data.rs","line":219},{"file":"crates/apr-cli/src/commands/data.rs","line":398},{"file":"crates/apr-cli/src/commands/data.rs","line":428},{"file":"crates/apr-cli/src/commands/data.rs","line":462},{"file":"crates/apr-cli/src/commands/distill_include_01.rs","line":31},{"file":"crates/apr-cli/src/commands/distill_include_01.rs","line":40},{"file":"crates/apr-cli/src/commands/distill_include_01.rs","line":50},{"file":"crates/apr-cli/src/commands/distill_include_01.rs","line":59},{"file":"crates/apr-cli/src/commands/distill_include_01.rs","line":72},{"file":"crates/apr-cli/src/commands/distill_include_01.rs","line":81},{"file":"crates/apr-cli/src/commands/distill_include_01.rs","line":95},{"file":"crates/apr-cli/src/commands/distill_include_01.rs","line":104},{"file":"crates/apr-cli/src/commands/distill_include_01.rs","line":120},{"file":"crates/apr-cli/src/commands/distill_include_01.rs","line":129},{"file":"crates/apr-cli/src/commands/distill_include_01.rs","line":161},{"file":"crates/apr-cli/src/commands/distill_include_01.rs","line":170},{"file":"crates/apr-cli/src/commands/distill_include_01.rs","line":185},{"file":"crates/apr-cli/src/commands/distill_include_01.rs","line":194},{"file":"crates/apr-cli/src/commands/distill_include_01.rs","line":203},{"file":"crates/apr-cli/src/commands/distill_include_01.rs","line":212},{"file":"crates/apr-cli/src/commands/distill_include_01.rs","line":223},{"file":"crates/apr-cli/src/commands/distill_include_01.rs","line":232},{"file":"crates/apr-cli/src/commands/finetune.rs","line":525},{"file":"crates/apr-cli/src/commands/finetune.rs","line":527},{"file":"crates/apr-cli/src/commands/finetune.rs","line":530},{"file":"crates/apr-cli/src/commands/finetune.rs","line":531},{"file":"crates/apr-cli/src/commands/finetune.rs","line":532},{"file":"crates/apr-cli/src/commands/finetune.rs","line":533},{"file":"crates/apr-cli/src/commands/finetune.rs","line":534},{"file":"crates/apr-cli/src/commands/finetune.rs","line":535},{"file":"crates/apr-cli/src/commands/finetune.rs","line":536},{"file":"crates/apr-cli/src/commands/finetune.rs","line":537},{"file":"crates/apr-cli/src/commands/finetune.rs","line":538},{"file":"crates/apr-cli/src/commands/finetune.rs","line":539},{"file":"crates/apr-cli/src/commands/finetune.rs","line":540},{"file":"crates/apr-cli/src/commands/finetune.rs","line":541},{"file":"crates/apr-cli/src/commands/finetune.rs","line":542},{"file":"crates/apr-cli/src/commands/finetune.rs","line":543},{"file":"crates/apr-cli/src/commands/finetune.rs","line":544},{"file":"crates/apr-cli/src/commands/finetune.rs","line":545},{"file":"crates/apr-cli/src/commands/finetune.rs","line":546},{"file":"crates/apr-cli/src/commands/finetune.rs","line":547},{"file":"crates/apr-cli/src/commands/finetune.rs","line":548},{"file":"crates/apr-cli/src/commands/finetune.rs","line":549},{"file":"crates/apr-cli/src/commands/finetune.rs","line":550},{"file":"crates/apr-cli/src/commands/finetune.rs","line":551},{"file":"crates/apr-cli/src/commands/finetune.rs","line":552},{"file":"crates/apr-cli/src/commands/finetune.rs","line":553},{"file":"crates/apr-cli/src/commands/finetune.rs","line":554},{"file":"crates/apr-cli/src/commands/finetune.rs","line":555},{"file":"crates/apr-cli/src/commands/finetune.rs","line":556},{"file":"crates/apr-cli/src/commands/finetune.rs","line":557},{"file":"crates/apr-cli/src/commands/finetune.rs","line":558},{"file":"crates/apr-cli/src/commands/finetune.rs","line":559},{"file":"crates/apr-cli/src/commands/finetune.rs","line":560},{"file":"crates/apr-cli/src/commands/finetune.rs","line":561},{"file":"crates/apr-cli/src/commands/finetune.rs","line":562},{"file":"crates/apr-cli/src/commands/finetune.rs","line":563},{"file":"crates/apr-cli/src/commands/finetune.rs","line":586},{"file":"crates/apr-cli/src/commands/finetune.rs","line":596},{"file":"crates/apr-cli/src/commands/finetune.rs","line":597},{"file":"crates/apr-cli/src/commands/finetune.rs","line":598},{"file":"crates/apr-cli/src/commands/finetune.rs","line":599},{"file":"crates/apr-cli/src/commands/finetune.rs","line":600},{"file":"crates/apr-cli/src/commands/finetune.rs","line":601},{"file":"crates/apr-cli/src/commands/finetune.rs","line":602},{"file":"crates/apr-cli/src/commands/finetune.rs","line":603},{"file":"crates/apr-cli/src/commands/finetune.rs","line":604},{"file":"crates/apr-cli/src/commands/finetune.rs","line":605},{"file":"crates/apr-cli/src/commands/finetune.rs","line":606},{"file":"crates/apr-cli/src/commands/finetune.rs","line":607},{"file":"crates/apr-cli/src/commands/finetune.rs","line":608},{"file":"crates/apr-cli/src/commands/finetune.rs","line":609},{"file":"crates/apr-cli/src/commands/finetune.rs","line":610},{"file":"crates/apr-cli/src/commands/finetune.rs","line":611},{"file":"crates/apr-cli/src/commands/finetune.rs","line":612},{"file":"crates/apr-cli/src/commands/finetune.rs","line":613},{"file":"crates/apr-cli/src/commands/finetune.rs","line":614},{"file":"crates/apr-cli/src/commands/finetune.rs","line":615},{"file":"crates/apr-cli/src/commands/finetune.rs","line":616},{"file":"crates/apr-cli/src/commands/finetune.rs","line":617},{"file":"crates/apr-cli/src/commands/finetune.rs","line":618},{"file":"crates/apr-cli/src/commands/finetune.rs","line":619},{"file":"crates/apr-cli/src/commands/finetune.rs","line":620},{"file":"crates/apr-cli/src/commands/finetune.rs","line":621},{"file":"crates/apr-cli/src/commands/finetune.rs","line":624},{"file":"crates/apr-cli/src/commands/finetune.rs","line":625},{"file":"crates/apr-cli/src/commands/finetune.rs","line":626},{"file":"crates/apr-cli/src/commands/finetune.rs","line":627},{"file":"crates/apr-cli/src/commands/finetune.rs","line":628},{"file":"crates/apr-cli/src/commands/finetune.rs","line":629},{"file":"crates/apr-cli/src/commands/finetune.rs","line":630},{"file":"crates/apr-cli/src/commands/finetune.rs","line":631},{"file":"crates/apr-cli/src/commands/finetune.rs","line":632},{"file":"crates/apr-cli/src/commands/finetune.rs","line":667},{"file":"crates/apr-cli/src/commands/finetune.rs","line":668},{"file":"crates/apr-cli/src/commands/finetune.rs","line":669},{"file":"crates/apr-cli/src/commands/finetune.rs","line":670},{"file":"crates/apr-cli/src/commands/finetune.rs","line":671},{"file":"crates/apr-cli/src/commands/finetune.rs","line":672},{"file":"crates/apr-cli/src/commands/finetune.rs","line":673},{"file":"crates/apr-cli/src/commands/finetune.rs","line":674},{"file":"crates/apr-cli/src/commands/finetune.rs","line":675},{"file":"crates/apr-cli/src/commands/finetune.rs","line":724},{"file":"crates/apr-cli/src/commands/finetune.rs","line":746},{"file":"crates/apr-cli/src/commands/finetune.rs","line":749},{"file":"crates/apr-cli/src/commands/finetune.rs","line":793},{"file":"crates/apr-cli/src/commands/finetune.rs","line":802},{"file":"crates/apr-cli/src/commands/finetune.rs","line":803},{"file":"crates/apr-cli/src/commands/finetune.rs","line":804},{"file":"crates/apr-cli/src/commands/finetune.rs","line":822},{"file":"crates/apr-cli/src/commands/finetune.rs","line":823},{"file":"crates/apr-cli/src/commands/finetune.rs","line":837},{"file":"crates/apr-cli/src/commands/finetune.rs","line":838},{"file":"crates/apr-cli/src/commands/finetune.rs","line":839},{"file":"crates/apr-cli/src/commands/finetune.rs","line":840},{"file":"crates/apr-cli/src/commands/finetune.rs","line":841},{"file":"crates/apr-cli/src/commands/finetune.rs","line":842},{"file":"crates/apr-cli/src/commands/finetune.rs","line":843},{"file":"crates/apr-cli/src/commands/finetune.rs","line":844},{"file":"crates/apr-cli/src/commands/finetune.rs","line":845},{"file":"crates/apr-cli/src/commands/finetune.rs","line":846},{"file":"crates/apr-cli/src/commands/finetune.rs","line":847},{"file":"crates/apr-cli/src/commands/finetune.rs","line":848},{"file":"crates/apr-cli/src/commands/finetune.rs","line":849},{"file":"crates/apr-cli/src/commands/finetune.rs","line":850},{"file":"crates/apr-cli/src/commands/finetune.rs","line":851},{"file":"crates/apr-cli/src/commands/finetune.rs","line":852},{"file":"crates/apr-cli/src/commands/finetune.rs","line":853},{"file":"crates/apr-cli/src/commands/finetune.rs","line":991},{"file":"crates/apr-cli/src/commands/finetune.rs","line":992},{"file":"crates/apr-cli/src/commands/finetune.rs","line":994},{"file":"crates/apr-cli/src/commands/finetune.rs","line":995},{"file":"crates/apr-cli/src/commands/finetune.rs","line":996},{"file":"crates/apr-cli/src/commands/finetune.rs","line":997},{"file":"crates/apr-cli/src/commands/finetune.rs","line":998},{"file":"crates/apr-cli/src/commands/finetune.rs","line":999},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1000},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1001},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1002},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1003},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1004},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1005},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1006},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1008},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1009},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1010},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1011},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1012},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1013},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1014},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1015},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1016},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1017},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1018},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1019},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1020},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1021},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1022},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1023},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1024},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1025},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1026},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1027},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1028},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1029},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1030},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1031},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1032},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1033},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1034},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1035},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1036},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1037},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1038},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1039},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1040},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1041},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1042},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1043},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1044},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1045},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1046},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1047},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1048},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1049},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1050},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1051},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1052},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1053},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1054},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1055},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1056},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1057},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1058},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1059},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1060},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1061},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1062},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1063},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1064},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1065},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1066},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1067},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1068},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1069},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1070},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1071},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1072},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1073},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1074},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1075},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1076},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1077},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1078},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1079},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1080},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1081},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1082},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1083},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1084},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1085},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1086},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1087},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1088},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1089},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1090},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1091},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1092},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1093},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1094},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1095},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1096},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1097},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1098},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1099},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1100},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1101},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1102},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1103},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1104},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1105},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1106},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1107},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1108},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1109},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1110},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1111},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1112},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1113},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1114},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1115},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1116},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1117},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1118},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1119},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1120},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1121},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1122},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1123},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1124},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1125},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1126},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1127},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1128},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1129},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1130},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1131},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1132},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1135},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1149},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1150},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1156},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1157},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1158},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1159},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1160},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1161},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1201},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1202},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1203},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1204},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1205},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1206},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1207},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1208},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1209},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1210},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1211},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1212},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1213},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1214},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1215},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1216},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1217},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1218},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1219},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1220},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1221},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1222},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1223},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1224},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1225},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1226},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1227},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1228},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1229},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1230},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1231},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1232},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1233},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1234},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1235},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1236},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1237},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1238},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1239},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1240},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1241},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1242},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1243},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1244},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1245},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1246},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1247},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1248},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1249},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1250},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1251},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1252},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1253},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1254},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1255},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1256},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1257},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1258},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1259},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1260},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1261},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1262},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1263},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1264},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1265},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1266},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1267},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1268},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1269},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1270},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1271},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1272},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1273},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1274},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1275},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1276},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1277},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1278},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1279},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1280},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1281},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1282},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1283},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1284},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1285},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1286},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1287},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1288},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1289},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1290},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1291},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1292},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1293},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1294},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1295},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1296},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1297},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1298},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1299},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1300},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1301},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1302},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1303},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1304},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1305},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1306},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1307},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1308},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1309},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1310},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1311},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1312},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1313},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1314},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1315},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1316},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1317},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1318},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1319},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1320},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1321},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1322},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1323},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1324},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1325},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1326},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1327},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1328},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1329},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1330},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1331},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1332},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1333},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1334},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1335},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1336},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1337},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1338},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1339},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1340},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1341},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1342},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1343},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1344},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1345},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1346},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1347},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1348},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1349},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1350},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1351},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1352},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1353},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1354},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1355},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1356},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1357},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1358},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1359},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1360},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1361},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1362},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1363},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1364},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1365},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1366},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1367},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1368},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1369},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1370},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1371},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1372},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1373},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1374},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1375},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1376},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1377},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1378},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1379},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1380},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1381},{"file":"crates/apr-cli/src/commands/finetune.rs","line":1382},{"file":"crates/apr-cli/src/commands/model_config.rs","line":37},{"file":"crates/apr-cli/src/commands/model_config.rs","line":114},{"file":"crates/apr-cli/src/commands/model_config.rs","line":133},{"file":"crates/apr-cli/src/commands/model_config.rs","line":140},{"file":"crates/apr-cli/src/commands/model_config.rs","line":141},{"file":"crates/apr-cli/src/commands/model_config.rs","line":146},{"file":"crates/apr-cli/src/commands/model_config.rs","line":147},{"file":"crates/apr-cli/src/commands/serve/handlers_include_01.rs","line":66},{"file":"crates/apr-cli/src/commands/serve/handlers_include_01.rs","line":67},{"file":"crates/apr-cli/src/commands/serve/handlers_include_01.rs","line":68},{"file":"crates/apr-cli/src/commands/serve/handlers_include_01.rs","line":69},{"file":"crates/apr-cli/src/commands/serve/handlers_include_01.rs","line":70},{"file":"crates/apr-cli/src/commands/serve/handlers_include_01.rs","line":71},{"file":"crates/apr-cli/src/commands/serve/handlers_include_01.rs","line":72},{"file":"crates/apr-cli/src/commands/serve/handlers_include_01.rs","line":73},{"file":"crates/apr-cli/src/commands/serve/handlers_include_01.rs","line":74},{"file":"crates/apr-cli/src/commands/serve/handlers_include_01.rs","line":151},{"file":"crates/apr-cli/src/commands/serve/handlers_include_01.rs","line":152},{"file":"crates/apr-cli/src/commands/serve/handlers_include_01.rs","line":153},{"file":"crates/apr-cli/src/commands/serve/handlers_include_01.rs","line":154},{"file":"crates/apr-cli/src/commands/serve/handlers_include_01.rs","line":155},{"file":"crates/apr-cli/src/commands/serve/handlers_include_01.rs","line":156},{"file":"crates/apr-cli/src/commands/serve/handlers_include_01.rs","line":157},{"file":"crates/apr-cli/src/commands/serve/handlers_include_01.rs","line":158},{"file":"crates/apr-cli/src/commands/train.rs","line":15},{"file":"crates/apr-cli/src/commands/train.rs","line":18},{"file":"crates/apr-cli/src/commands/train.rs","line":19},{"file":"crates/apr-cli/src/commands/train.rs","line":20},{"file":"crates/apr-cli/src/commands/train.rs","line":21},{"file":"crates/apr-cli/src/commands/train.rs","line":24},{"file":"crates/apr-cli/src/commands/train.rs","line":25},{"file":"crates/apr-cli/src/commands/train.rs","line":26},{"file":"crates/apr-cli/src/commands/train.rs","line":27},{"file":"crates/apr-cli/src/commands/train.rs","line":28},{"file":"crates/apr-cli/src/commands/train.rs","line":29},{"file":"crates/apr-cli/src/commands/train.rs","line":30},{"file":"crates/apr-cli/src/commands/train.rs","line":31},{"file":"crates/apr-cli/src/commands/train.rs","line":32},{"file":"crates/apr-cli/src/commands/train.rs","line":33},{"file":"crates/apr-cli/src/commands/train.rs","line":34},{"file":"crates/apr-cli/src/commands/train.rs","line":39},{"file":"crates/apr-cli/src/commands/train.rs","line":40},{"file":"crates/apr-cli/src/commands/train.rs","line":41},{"file":"crates/apr-cli/src/commands/train.rs","line":42},{"file":"crates/apr-cli/src/commands/train.rs","line":43},{"file":"crates/apr-cli/src/commands/train.rs","line":44},{"file":"crates/apr-cli/src/commands/train.rs","line":45},{"file":"crates/apr-cli/src/commands/train.rs","line":46},{"file":"crates/apr-cli/src/commands/train.rs","line":47},{"file":"crates/apr-cli/src/commands/train.rs","line":48},{"file":"crates/apr-cli/src/commands/train.rs","line":49},{"file":"crates/apr-cli/src/commands/train.rs","line":50},{"file":"crates/apr-cli/src/commands/train.rs","line":57},{"file":"crates/apr-cli/src/commands/train.rs","line":58},{"file":"crates/apr-cli/src/commands/train.rs","line":59},{"file":"crates/apr-cli/src/commands/train.rs","line":60},{"file":"crates/apr-cli/src/commands/train.rs","line":61},{"file":"crates/apr-cli/src/commands/train.rs","line":62},{"file":"crates/apr-cli/src/commands/train.rs","line":63},{"file":"crates/apr-cli/src/commands/train.rs","line":64},{"file":"crates/apr-cli/src/commands/train.rs","line":65},{"file":"crates/apr-cli/src/commands/train.rs","line":66},{"file":"crates/apr-cli/src/commands/train.rs","line":67},{"file":"crates/apr-cli/src/commands/train.rs","line":68},{"file":"crates/apr-cli/src/commands/train.rs","line":69},{"file":"crates/apr-cli/src/commands/train.rs","line":70},{"file":"crates/apr-cli/src/commands/train.rs","line":71},{"file":"crates/apr-cli/src/commands/train.rs","line":72},{"file":"crates/apr-cli/src/commands/train.rs","line":73},{"file":"crates/apr-cli/src/commands/train.rs","line":74},{"file":"crates/apr-cli/src/commands/train.rs","line":75},{"file":"crates/apr-cli/src/commands/train.rs","line":76},{"file":"crates/apr-cli/src/commands/train.rs","line":77},{"file":"crates/apr-cli/src/commands/train.rs","line":78},{"file":"crates/apr-cli/src/commands/train.rs","line":79},{"file":"crates/apr-cli/src/commands/train.rs","line":80},{"file":"crates/apr-cli/src/commands/train.rs","line":81},{"file":"crates/apr-cli/src/commands/train.rs","line":82},{"file":"crates/apr-cli/src/commands/train.rs","line":83},{"file":"crates/apr-cli/src/commands/train.rs","line":84},{"file":"crates/apr-cli/src/commands/train.rs","line":85},{"file":"crates/apr-cli/src/commands/train.rs","line":86},{"file":"crates/apr-cli/src/commands/train.rs","line":87},{"file":"crates/apr-cli/src/commands/train.rs","line":88},{"file":"crates/apr-cli/src/commands/train.rs","line":89},{"file":"crates/apr-cli/src/commands/train.rs","line":90},{"file":"crates/apr-cli/src/commands/train.rs","line":91},{"file":"crates/apr-cli/src/commands/train.rs","line":92},{"file":"crates/apr-cli/src/commands/train.rs","line":93},{"file":"crates/apr-cli/src/commands/train.rs","line":94},{"file":"crates/apr-cli/src/commands/train.rs","line":95},{"file":"crates/apr-cli/src/commands/train.rs","line":96},{"file":"crates/apr-cli/src/commands/train.rs","line":97},{"file":"crates/apr-cli/src/commands/train.rs","line":98},{"file":"crates/apr-cli/src/commands/train.rs","line":99},{"file":"crates/apr-cli/src/commands/train.rs","line":100},{"file":"crates/apr-cli/src/commands/train.rs","line":101},{"file":"crates/apr-cli/src/commands/train.rs","line":102},{"file":"crates/apr-cli/src/commands/train.rs","line":103},{"file":"crates/apr-cli/src/commands/train.rs","line":104},{"file":"crates/apr-cli/src/commands/train.rs","line":105},{"file":"crates/apr-cli/src/commands/train.rs","line":106},{"file":"crates/apr-cli/src/commands/train.rs","line":107},{"file":"crates/apr-cli/src/commands/train.rs","line":108},{"file":"crates/apr-cli/src/commands/train.rs","line":109},{"file":"crates/apr-cli/src/commands/train.rs","line":110},{"file":"crates/apr-cli/src/commands/train.rs","line":111},{"file":"crates/apr-cli/src/commands/train.rs","line":112},{"file":"crates/apr-cli/src/commands/train.rs","line":113},{"file":"crates/apr-cli/src/commands/train.rs","line":114},{"file":"crates/apr-cli/src/commands/train.rs","line":163},{"file":"crates/apr-cli/src/commands/train.rs","line":164},{"file":"crates/apr-cli/src/commands/train.rs","line":165},{"file":"crates/apr-cli/src/commands/train.rs","line":166},{"file":"crates/apr-cli/src/commands/train.rs","line":167},{"file":"crates/apr-cli/src/commands/train.rs","line":168},{"file":"crates/apr-cli/src/commands/train.rs","line":169},{"file":"crates/apr-cli/src/commands/train.rs","line":170},{"file":"crates/apr-cli/src/commands/train.rs","line":171},{"file":"crates/apr-cli/src/commands/train.rs","line":172},{"file":"crates/apr-cli/src/commands/train.rs","line":173},{"file":"crates/apr-cli/src/commands/train.rs","line":174},{"file":"crates/apr-cli/src/commands/train.rs","line":175},{"file":"crates/apr-cli/src/commands/train.rs","line":176},{"file":"crates/apr-cli/src/commands/train.rs","line":208},{"file":"crates/apr-cli/src/commands/train.rs","line":209},{"file":"crates/apr-cli/src/commands/train.rs","line":210},{"file":"crates/apr-cli/src/commands/train.rs","line":211},{"file":"crates/apr-cli/src/commands/train.rs","line":212},{"file":"crates/apr-cli/src/commands/train.rs","line":213},{"file":"crates/apr-cli/src/commands/train.rs","line":214},{"file":"crates/apr-cli/src/commands/train.rs","line":215},{"file":"crates/apr-cli/src/commands/train.rs","line":216},{"file":"crates/apr-cli/src/commands/train.rs","line":217},{"file":"crates/apr-cli/src/commands/train.rs","line":218},{"file":"crates/apr-cli/src/commands/train.rs","line":219},{"file":"crates/apr-cli/src/commands/train.rs","line":220},{"file":"crates/apr-cli/src/commands/train.rs","line":221},{"file":"crates/apr-cli/src/commands/train.rs","line":222},{"file":"crates/apr-cli/src/commands/train.rs","line":223},{"file":"crates/apr-cli/src/commands/train.rs","line":224},{"file":"crates/apr-cli/src/commands/train.rs","line":225},{"file":"crates/apr-cli/src/commands/train.rs","line":226},{"file":"crates/apr-cli/src/commands/train.rs","line":227},{"file":"crates/apr-cli/src/commands/train.rs","line":228},{"file":"crates/apr-cli/src/commands/train.rs","line":229},{"file":"crates/apr-cli/src/commands/train.rs","line":230},{"file":"crates/apr-cli/src/commands/train.rs","line":231},{"file":"crates/apr-cli/src/commands/train.rs","line":232},{"file":"crates/apr-cli/src/commands/train.rs","line":233},{"file":"crates/apr-cli/src/commands/train.rs","line":242},{"file":"crates/apr-cli/src/commands/train.rs","line":243},{"file":"crates/apr-cli/src/commands/train.rs","line":244},{"file":"crates/apr-cli/src/commands/train.rs","line":245},{"file":"crates/apr-cli/src/commands/train.rs","line":246},{"file":"crates/apr-cli/src/commands/train.rs","line":247},{"file":"crates/apr-cli/src/commands/train.rs","line":248},{"file":"crates/apr-cli/src/commands/train.rs","line":249},{"file":"crates/apr-cli/src/commands/train.rs","line":250},{"file":"crates/apr-cli/src/commands/train.rs","line":251},{"file":"crates/apr-cli/src/commands/train.rs","line":252},{"file":"crates/apr-cli/src/commands/train.rs","line":253},{"file":"crates/apr-cli/src/commands/train.rs","line":254},{"file":"crates/apr-cli/src/commands/train.rs","line":255},{"file":"crates/apr-cli/src/commands/train.rs","line":256},{"file":"crates/apr-cli/src/commands/train.rs","line":257},{"file":"crates/apr-cli/src/commands/train.rs","line":258},{"file":"crates/apr-cli/src/commands/train.rs","line":259},{"file":"crates/apr-cli/src/commands/train.rs","line":260},{"file":"crates/apr-cli/src/commands/train.rs","line":261},{"file":"crates/apr-cli/src/commands/train.rs","line":262},{"file":"crates/apr-cli/src/commands/train.rs","line":263},{"file":"crates/apr-cli/src/commands/train.rs","line":264},{"file":"crates/apr-cli/src/commands/train.rs","line":265},{"file":"crates/apr-cli/src/commands/train.rs","line":266},{"file":"crates/apr-cli/src/commands/train.rs","line":267},{"file":"crates/apr-cli/src/commands/train.rs","line":268},{"file":"crates/apr-cli/src/commands/train.rs","line":269},{"file":"crates/apr-cli/src/commands/train.rs","line":270},{"file":"crates/apr-cli/src/commands/train.rs","line":271},{"file":"crates/apr-cli/src/commands/train.rs","line":272},{"file":"crates/apr-cli/src/commands/train.rs","line":273},{"file":"crates/apr-cli/src/commands/train.rs","line":274},{"file":"crates/apr-cli/src/commands/train.rs","line":275},{"file":"crates/apr-cli/src/commands/train.rs","line":276},{"file":"crates/apr-cli/src/commands/train.rs","line":277},{"file":"crates/apr-cli/src/commands/train.rs","line":278},{"file":"crates/apr-cli/src/commands/train.rs","line":279},{"file":"crates/apr-cli/src/commands/train.rs","line":280},{"file":"crates/apr-cli/src/commands/train.rs","line":281},{"file":"crates/apr-cli/src/commands/train.rs","line":282},{"file":"crates/apr-cli/src/commands/train.rs","line":283},{"file":"crates/apr-cli/src/commands/train.rs","line":284},{"file":"crates/apr-cli/src/commands/train.rs","line":285},{"file":"crates/apr-cli/src/commands/train.rs","line":286},{"file":"crates/apr-cli/src/commands/train.rs","line":287},{"file":"crates/apr-cli/src/commands/train.rs","line":288},{"file":"crates/apr-cli/src/commands/train.rs","line":289},{"file":"crates/apr-cli/src/commands/train.rs","line":290},{"file":"crates/apr-cli/src/commands/train.rs","line":291},{"file":"crates/apr-cli/src/commands/train.rs","line":292},{"file":"crates/apr-cli/src/commands/train.rs","line":293},{"file":"crates/apr-cli/src/commands/train.rs","line":294},{"file":"crates/apr-cli/src/commands/train.rs","line":295},{"file":"crates/apr-cli/src/commands/train.rs","line":296},{"file":"crates/apr-cli/src/commands/train.rs","line":297},{"file":"crates/apr-cli/src/commands/train.rs","line":298},{"file":"crates/apr-cli/src/commands/train.rs","line":299},{"file":"crates/apr-cli/src/commands/train.rs","line":300},{"file":"crates/apr-cli/src/commands/train.rs","line":301},{"file":"crates/apr-cli/src/commands/train.rs","line":302},{"file":"crates/apr-cli/src/commands/train.rs","line":303},{"file":"crates/apr-cli/src/commands/train.rs","line":304},{"file":"crates/apr-cli/src/commands/train.rs","line":305},{"file":"crates/apr-cli/src/commands/train.rs","line":306},{"file":"crates/apr-cli/src/commands/train.rs","line":307},{"file":"crates/apr-cli/src/commands/train.rs","line":308},{"file":"crates/apr-cli/src/commands/train.rs","line":309},{"file":"crates/apr-cli/src/commands/train.rs","line":310},{"file":"crates/apr-cli/src/commands/train.rs","line":311},{"file":"crates/apr-cli/src/commands/train.rs","line":312},{"file":"crates/apr-cli/src/commands/train.rs","line":313},{"file":"crates/apr-cli/src/commands/train.rs","line":314},{"file":"crates/apr-cli/src/commands/train.rs","line":315},{"file":"crates/apr-cli/src/commands/train.rs","line":316},{"file":"crates/apr-cli/src/commands/train.rs","line":317},{"file":"crates/apr-cli/src/commands/train.rs","line":318},{"file":"crates/apr-cli/src/commands/train.rs","line":319},{"file":"crates/apr-cli/src/commands/train.rs","line":320},{"file":"crates/apr-cli/src/commands/train.rs","line":321},{"file":"crates/apr-cli/src/commands/train.rs","line":322},{"file":"crates/apr-cli/src/commands/train.rs","line":323},{"file":"crates/apr-cli/src/commands/train.rs","line":324},{"file":"crates/apr-cli/src/commands/train.rs","line":325},{"file":"crates/apr-cli/src/commands/train.rs","line":326},{"file":"crates/apr-cli/src/commands/train.rs","line":327},{"file":"crates/apr-cli/src/commands/train.rs","line":328},{"file":"crates/apr-cli/src/commands/train.rs","line":329},{"file":"crates/apr-cli/src/commands/train.rs","line":330},{"file":"crates/apr-cli/src/commands/train.rs","line":331},{"file":"crates/apr-cli/src/commands/train.rs","line":332},{"file":"crates/apr-cli/src/commands/train.rs","line":333},{"file":"crates/apr-cli/src/commands/train.rs","line":334},{"file":"crates/apr-cli/src/commands/train.rs","line":335},{"file":"crates/apr-cli/src/commands/train.rs","line":336},{"file":"crates/apr-cli/src/commands/train.rs","line":337},{"file":"crates/apr-cli/src/commands/train.rs","line":338},{"file":"crates/apr-cli/src/commands/train.rs","line":339},{"file":"crates/apr-cli/src/commands/train.rs","line":340},{"file":"crates/apr-cli/src/commands/train.rs","line":341},{"file":"crates/apr-cli/src/commands/train.rs","line":342},{"file":"crates/apr-cli/src/commands/train.rs","line":343},{"file":"crates/apr-cli/src/commands/train.rs","line":344},{"file":"crates/apr-cli/src/commands/train.rs","line":345},{"file":"crates/apr-cli/src/commands/train.rs","line":346},{"file":"crates/apr-cli/src/commands/train.rs","line":347},{"file":"crates/apr-cli/src/commands/train.rs","line":348},{"file":"crates/apr-cli/src/commands/train.rs","line":349},{"file":"crates/apr-cli/src/commands/train.rs","line":350},{"file":"crates/apr-cli/src/commands/train.rs","line":351},{"file":"crates/apr-cli/src/commands/train.rs","line":352},{"file":"crates/apr-cli/src/commands/train.rs","line":353},{"file":"crates/apr-cli/src/commands/train.rs","line":354},{"file":"crates/apr-cli/src/commands/train.rs","line":355},{"file":"crates/apr-cli/src/commands/train.rs","line":356},{"file":"crates/apr-cli/src/commands/train.rs","line":357},{"file":"crates/apr-cli/src/commands/train.rs","line":358},{"file":"crates/apr-cli/src/commands/train.rs","line":359},{"file":"crates/apr-cli/src/commands/train.rs","line":360},{"file":"crates/apr-cli/src/commands/train.rs","line":361},{"file":"crates/apr-cli/src/commands/train.rs","line":362},{"file":"crates/apr-cli/src/commands/train.rs","line":363},{"file":"crates/apr-cli/src/commands/train.rs","line":364},{"file":"crates/apr-cli/src/commands/train.rs","line":365},{"file":"crates/apr-cli/src/commands/train.rs","line":366},{"file":"crates/apr-cli/src/commands/train.rs","line":367},{"file":"crates/apr-cli/src/commands/train.rs","line":368},{"file":"crates/apr-cli/src/commands/train.rs","line":369},{"file":"crates/apr-cli/src/commands/train.rs","line":370},{"file":"crates/apr-cli/src/commands/train.rs","line":371},{"file":"crates/apr-cli/src/commands/train.rs","line":372},{"file":"crates/apr-cli/src/commands/train.rs","line":373},{"file":"crates/apr-cli/src/commands/train.rs","line":374},{"file":"crates/apr-cli/src/commands/train.rs","line":375},{"file":"crates/apr-cli/src/commands/train.rs","line":376},{"file":"crates/apr-cli/src/commands/train.rs","line":377},{"file":"crates/apr-cli/src/commands/train.rs","line":378},{"file":"crates/apr-cli/src/commands/train.rs","line":379},{"file":"crates/apr-cli/src/commands/train.rs","line":380},{"file":"crates/apr-cli/src/commands/train.rs","line":381},{"file":"crates/apr-cli/src/commands/train.rs","line":382},{"file":"crates/apr-cli/src/commands/train.rs","line":383},{"file":"crates/apr-cli/src/commands/train.rs","line":384},{"file":"crates/apr-cli/src/commands/train.rs","line":385},{"file":"crates/apr-cli/src/commands/train.rs","line":386},{"file":"crates/apr-cli/src/commands/train.rs","line":387},{"file":"crates/apr-cli/src/commands/train.rs","line":388},{"file":"crates/apr-cli/src/commands/train.rs","line":389},{"file":"crates/apr-cli/src/commands/train.rs","line":390},{"file":"crates/apr-cli/src/commands/train.rs","line":391},{"file":"crates/apr-cli/src/commands/train.rs","line":392},{"file":"crates/apr-cli/src/commands/train.rs","line":393},{"file":"crates/apr-cli/src/commands/train.rs","line":394},{"file":"crates/apr-cli/src/commands/train.rs","line":395},{"file":"crates/apr-cli/src/commands/train.rs","line":396},{"file":"crates/apr-cli/src/commands/train.rs","line":397},{"file":"crates/apr-cli/src/commands/train.rs","line":398},{"file":"crates/apr-cli/src/commands/train.rs","line":399},{"file":"crates/apr-cli/src/commands/train.rs","line":400},{"file":"crates/apr-cli/src/commands/train.rs","line":401},{"file":"crates/apr-cli/src/commands/train.rs","line":402},{"file":"crates/apr-cli/src/commands/train.rs","line":403},{"file":"crates/apr-cli/src/commands/train.rs","line":404},{"file":"crates/apr-cli/src/commands/train.rs","line":405},{"file":"crates/apr-cli/src/commands/train.rs","line":406},{"file":"crates/apr-cli/src/commands/train.rs","line":407},{"file":"crates/apr-cli/src/commands/train.rs","line":408},{"file":"crates/apr-cli/src/commands/train.rs","line":409},{"file":"crates/apr-cli/src/commands/train.rs","line":410},{"file":"crates/apr-cli/src/commands/train.rs","line":411},{"file":"crates/apr-cli/src/commands/train.rs","line":412},{"file":"crates/apr-cli/src/commands/train.rs","line":413},{"file":"crates/apr-cli/src/commands/train.rs","line":414},{"file":"crates/apr-cli/src/commands/train.rs","line":415},{"file":"crates/apr-cli/src/commands/train.rs","line":416},{"file":"crates/apr-cli/src/commands/train.rs","line":417},{"file":"crates/apr-cli/src/commands/train.rs","line":418},{"file":"crates/apr-cli/src/commands/train.rs","line":419},{"file":"crates/apr-cli/src/commands/train.rs","line":420},{"file":"crates/apr-cli/src/commands/train.rs","line":421},{"file":"crates/apr-cli/src/commands/train.rs","line":422},{"file":"crates/apr-cli/src/commands/train.rs","line":423},{"file":"crates/apr-cli/src/commands/train.rs","line":424},{"file":"crates/apr-cli/src/commands/train.rs","line":425},{"file":"crates/apr-cli/src/commands/train.rs","line":426},{"file":"crates/apr-cli/src/commands/train.rs","line":427},{"file":"crates/apr-cli/src/commands/train.rs","line":428},{"file":"crates/apr-cli/src/commands/train.rs","line":429},{"file":"crates/apr-cli/src/commands/train.rs","line":430},{"file":"crates/apr-cli/src/commands/train.rs","line":431},{"file":"crates/apr-cli/src/commands/train.rs","line":432},{"file":"crates/apr-cli/src/commands/train.rs","line":433},{"file":"crates/apr-cli/src/commands/train.rs","line":434},{"file":"crates/apr-cli/src/commands/train.rs","line":435},{"file":"crates/apr-cli/src/commands/train.rs","line":436},{"file":"crates/apr-cli/src/commands/train.rs","line":440},{"file":"crates/apr-cli/src/commands/train.rs","line":443},{"file":"crates/apr-cli/src/commands/train.rs","line":444},{"file":"crates/apr-cli/src/commands/train.rs","line":445},{"file":"crates/apr-cli/src/commands/train.rs","line":446},{"file":"crates/apr-cli/src/commands/train.rs","line":447},{"file":"crates/apr-cli/src/commands/train.rs","line":448},{"file":"crates/apr-cli/src/commands/train.rs","line":449},{"file":"crates/apr-cli/src/commands/train.rs","line":450},{"file":"crates/apr-cli/src/commands/train.rs","line":451},{"file":"crates/apr-cli/src/commands/train.rs","line":452},{"file":"crates/apr-cli/src/commands/train.rs","line":453},{"file":"crates/apr-cli/src/commands/train.rs","line":454},{"file":"crates/apr-cli/src/commands/train.rs","line":474},{"file":"crates/apr-cli/src/commands/train.rs","line":475},{"file":"crates/apr-cli/src/commands/train.rs","line":476},{"file":"crates/apr-cli/src/commands/train.rs","line":477},{"file":"crates/apr-cli/src/commands/train.rs","line":478},{"file":"crates/apr-cli/src/commands/train.rs","line":485},{"file":"crates/apr-cli/src/commands/train.rs","line":486},{"file":"crates/apr-cli/src/commands/train.rs","line":487},{"file":"crates/apr-cli/src/commands/train.rs","line":488},{"file":"crates/apr-cli/src/commands/train.rs","line":489},{"file":"crates/apr-cli/src/commands/train.rs","line":490},{"file":"crates/apr-cli/src/commands/train.rs","line":491},{"file":"crates/apr-cli/src/commands/train.rs","line":492},{"file":"crates/apr-cli/src/commands/train.rs","line":493},{"file":"crates/apr-cli/src/commands/train.rs","line":494},{"file":"crates/apr-cli/src/commands/train.rs","line":495},{"file":"crates/apr-cli/src/commands/train.rs","line":496},{"file":"crates/apr-cli/src/commands/train.rs","line":497},{"file":"crates/apr-cli/src/commands/train.rs","line":498},{"file":"crates/apr-cli/src/commands/train.rs","line":499},{"file":"crates/apr-cli/src/commands/train.rs","line":500},{"file":"crates/apr-cli/src/commands/train.rs","line":501},{"file":"crates/apr-cli/src/commands/train.rs","line":502},{"file":"crates/apr-cli/src/commands/train.rs","line":503},{"file":"crates/apr-cli/src/commands/train.rs","line":504},{"file":"crates/apr-cli/src/commands/train.rs","line":505},{"file":"crates/apr-cli/src/commands/train.rs","line":506},{"file":"crates/apr-cli/src/commands/train.rs","line":507},{"file":"crates/apr-cli/src/commands/train.rs","line":508},{"file":"crates/apr-cli/src/commands/train.rs","line":509},{"file":"crates/apr-cli/src/commands/train.rs","line":510},{"file":"crates/apr-cli/src/commands/train.rs","line":511},{"file":"crates/apr-cli/src/commands/train.rs","line":512},{"file":"crates/apr-cli/src/commands/train.rs","line":513},{"file":"crates/apr-cli/src/commands/train.rs","line":514},{"file":"crates/apr-cli/src/commands/train.rs","line":515},{"file":"crates/apr-cli/src/commands/train.rs","line":516},{"file":"crates/apr-cli/src/commands/train.rs","line":517},{"file":"crates/apr-cli/src/commands/train.rs","line":518},{"file":"crates/apr-cli/src/commands/train.rs","line":519},{"file":"crates/apr-cli/src/commands/train.rs","line":520},{"file":"crates/apr-cli/src/commands/train.rs","line":521},{"file":"crates/apr-cli/src/commands/train.rs","line":522},{"file":"crates/apr-cli/src/commands/train.rs","line":523},{"file":"crates/apr-cli/src/commands/train.rs","line":524},{"file":"crates/apr-cli/src/commands/train.rs","line":525},{"file":"crates/apr-cli/src/commands/train.rs","line":526},{"file":"crates/apr-cli/src/commands/train.rs","line":527},{"file":"crates/apr-cli/src/commands/train.rs","line":528},{"file":"crates/apr-cli/src/commands/train.rs","line":529},{"file":"crates/apr-cli/src/commands/train.rs","line":530},{"file":"crates/apr-cli/src/commands/train.rs","line":531},{"file":"crates/apr-cli/src/commands/train.rs","line":532},{"file":"crates/apr-cli/src/commands/train.rs","line":533},{"file":"crates/apr-cli/src/commands/train.rs","line":534},{"file":"crates/apr-cli/src/commands/train.rs","line":535},{"file":"crates/apr-cli/src/commands/train.rs","line":536},{"file":"crates/apr-cli/src/commands/train.rs","line":537},{"file":"crates/apr-cli/src/commands/train.rs","line":538},{"file":"crates/apr-cli/src/commands/train.rs","line":539},{"file":"crates/apr-cli/src/commands/train.rs","line":540},{"file":"crates/apr-cli/src/commands/train.rs","line":541},{"file":"crates/apr-cli/src/commands/train.rs","line":542},{"file":"crates/apr-cli/src/commands/train.rs","line":543},{"file":"crates/apr-cli/src/commands/train.rs","line":544},{"file":"crates/apr-cli/src/commands/train.rs","line":545},{"file":"crates/apr-cli/src/commands/train.rs","line":546},{"file":"crates/apr-cli/src/commands/train.rs","line":547},{"file":"crates/apr-cli/src/commands/train.rs","line":548},{"file":"crates/apr-cli/src/commands/train.rs","line":549},{"file":"crates/apr-cli/src/commands/train.rs","line":550},{"file":"crates/apr-cli/src/commands/train.rs","line":551},{"file":"crates/apr-cli/src/commands/train.rs","line":552},{"file":"crates/apr-cli/src/commands/train.rs","line":553},{"file":"crates/apr-cli/src/commands/train.rs","line":554},{"file":"crates/apr-cli/src/commands/train.rs","line":555},{"file":"crates/apr-cli/src/commands/train.rs","line":556},{"file":"crates/apr-cli/src/commands/train.rs","line":557},{"file":"crates/apr-cli/src/commands/train.rs","line":558},{"file":"crates/apr-cli/src/commands/train.rs","line":559},{"file":"crates/apr-cli/src/commands/train.rs","line":560},{"file":"crates/apr-cli/src/commands/train.rs","line":561},{"file":"crates/apr-cli/src/commands/train.rs","line":562},{"file":"crates/apr-cli/src/commands/train.rs","line":563},{"file":"crates/apr-cli/src/commands/train.rs","line":564},{"file":"crates/apr-cli/src/commands/train.rs","line":565},{"file":"crates/apr-cli/src/commands/train.rs","line":566},{"file":"crates/apr-cli/src/commands/train.rs","line":567},{"file":"crates/apr-cli/src/commands/train.rs","line":568},{"file":"crates/apr-cli/src/commands/train.rs","line":569},{"file":"crates/apr-cli/src/commands/train.rs","line":570},{"file":"crates/apr-cli/src/commands/train.rs","line":571},{"file":"crates/apr-cli/src/commands/train.rs","line":572},{"file":"crates/apr-cli/src/commands/train.rs","line":745},{"file":"crates/apr-cli/src/commands/train.rs","line":746},{"file":"crates/apr-cli/src/commands/train.rs","line":747},{"file":"crates/apr-cli/src/commands/train.rs","line":748},{"file":"crates/apr-cli/src/commands/train.rs","line":749},{"file":"crates/apr-cli/src/commands/train.rs","line":750},{"file":"crates/apr-cli/src/commands/train.rs","line":751},{"file":"crates/apr-cli/src/commands/train.rs","line":752},{"file":"crates/apr-cli/src/commands/train.rs","line":753},{"file":"crates/apr-cli/src/commands/train.rs","line":754},{"file":"crates/apr-cli/src/commands/train.rs","line":755},{"file":"crates/apr-cli/src/commands/train.rs","line":756},{"file":"crates/apr-cli/src/commands/train.rs","line":757},{"file":"crates/apr-cli/src/commands/train.rs","line":758},{"file":"crates/apr-cli/src/commands/train.rs","line":759},{"file":"crates/apr-cli/src/commands/train.rs","line":760},{"file":"crates/apr-cli/src/commands/train.rs","line":761},{"file":"crates/apr-cli/src/commands/train.rs","line":762},{"file":"crates/apr-cli/src/commands/train.rs","line":763},{"file":"crates/apr-cli/src/commands/train.rs","line":764},{"file":"crates/apr-cli/src/commands/train.rs","line":765},{"file":"crates/apr-cli/src/commands/train.rs","line":766},{"file":"crates/apr-cli/src/commands/train_output.rs","line":1},{"file":"crates/apr-cli/src/commands/train_output.rs","line":2},{"file":"crates/apr-cli/src/commands/train_output.rs","line":3},{"file":"crates/apr-cli/src/commands/train_output.rs","line":4},{"file":"crates/apr-cli/src/commands/train_output.rs","line":5},{"file":"crates/apr-cli/src/commands/train_output.rs","line":6},{"file":"crates/apr-cli/src/commands/train_output.rs","line":7},{"file":"crates/apr-cli/src/commands/train_output.rs","line":8},{"file":"crates/apr-cli/src/commands/train_output.rs","line":9},{"file":"crates/apr-cli/src/commands/train_output.rs","line":10},{"file":"crates/apr-cli/src/commands/train_output.rs","line":11},{"file":"crates/apr-cli/src/commands/train_output.rs","line":12},{"file":"crates/apr-cli/src/commands/train_output.rs","line":13},{"file":"crates/apr-cli/src/commands/train_output.rs","line":14},{"file":"crates/apr-cli/src/commands/train_output.rs","line":15},{"file":"crates/apr-cli/src/commands/train_output.rs","line":16},{"file":"crates/apr-cli/src/commands/train_output.rs","line":17},{"file":"crates/apr-cli/src/commands/train_output.rs","line":18},{"file":"crates/apr-cli/src/commands/train_output.rs","line":19},{"file":"crates/apr-cli/src/commands/train_output.rs","line":20},{"file":"crates/apr-cli/src/commands/train_output.rs","line":21},{"file":"crates/apr-cli/src/commands/train_output.rs","line":22},{"file":"crates/apr-cli/src/commands/train_output.rs","line":23},{"file":"crates/apr-cli/src/commands/train_output.rs","line":24},{"file":"crates/apr-cli/src/commands/train_output.rs","line":25},{"file":"crates/apr-cli/src/commands/train_output.rs","line":26},{"file":"crates/apr-cli/src/commands/train_output.rs","line":27},{"file":"crates/apr-cli/src/commands/train_output.rs","line":28},{"file":"crates/apr-cli/src/commands/train_output.rs","line":29},{"file":"crates/apr-cli/src/commands/train_output.rs","line":30},{"file":"crates/apr-cli/src/commands/train_output.rs","line":31},{"file":"crates/apr-cli/src/commands/train_output.rs","line":32},{"file":"crates/apr-cli/src/commands/train_output.rs","line":33},{"file":"crates/apr-cli/src/commands/train_output.rs","line":34},{"file":"crates/apr-cli/src/commands/train_output.rs","line":35},{"file":"crates/apr-cli/src/commands/train_output.rs","line":36},{"file":"crates/apr-cli/src/commands/train_output.rs","line":37},{"file":"crates/apr-cli/src/commands/train_output.rs","line":38},{"file":"crates/apr-cli/src/commands/train_output.rs","line":39},{"file":"crates/apr-cli/src/commands/train_output.rs","line":40},{"file":"crates/apr-cli/src/commands/train_output.rs","line":41},{"file":"crates/apr-cli/src/commands/train_output.rs","line":42},{"file":"crates/apr-cli/src/commands/train_output.rs","line":43},{"file":"crates/apr-cli/src/commands/train_output.rs","line":44},{"file":"crates/apr-cli/src/commands/train_output.rs","line":45},{"file":"crates/apr-cli/src/commands/train_output.rs","line":46},{"file":"crates/apr-cli/src/commands/train_output.rs","line":47},{"file":"crates/apr-cli/src/commands/train_output.rs","line":48},{"file":"crates/apr-cli/src/commands/train_output.rs","line":49},{"file":"crates/apr-cli/src/commands/train_output.rs","line":50},{"file":"crates/apr-cli/src/commands/train_output.rs","line":51},{"file":"crates/apr-cli/src/commands/train_output.rs","line":52},{"file":"crates/apr-cli/src/commands/train_output.rs","line":53},{"file":"crates/apr-cli/src/commands/train_output.rs","line":54},{"file":"crates/apr-cli/src/commands/train_output.rs","line":55},{"file":"crates/apr-cli/src/commands/train_output.rs","line":56},{"file":"crates/apr-cli/src/commands/train_output.rs","line":57},{"file":"crates/apr-cli/src/commands/train_output.rs","line":58},{"file":"crates/apr-cli/src/commands/train_output.rs","line":59},{"file":"crates/apr-cli/src/commands/train_output.rs","line":60},{"file":"crates/apr-cli/src/commands/train_output.rs","line":61},{"file":"crates/apr-cli/src/commands/train_output.rs","line":62},{"file":"crates/apr-cli/src/commands/train_output.rs","line":63},{"file":"crates/apr-cli/src/commands/train_output.rs","line":64},{"file":"crates/apr-cli/src/commands/train_output.rs","line":65},{"file":"crates/apr-cli/src/commands/train_output.rs","line":66},{"file":"crates/apr-cli/src/commands/train_output.rs","line":67},{"file":"crates/apr-cli/src/commands/train_output.rs","line":68},{"file":"crates/apr-cli/src/commands/train_output.rs","line":69},{"file":"crates/apr-cli/src/commands/train_output.rs","line":70},{"file":"crates/apr-cli/src/commands/train_output.rs","line":71},{"file":"crates/apr-cli/src/commands/train_output.rs","line":72},{"file":"crates/apr-cli/src/commands/train_output.rs","line":73},{"file":"crates/apr-cli/src/commands/train_output.rs","line":74},{"file":"crates/apr-cli/src/commands/train_output.rs","line":75},{"file":"crates/apr-cli/src/commands/train_output.rs","line":76},{"file":"crates/apr-cli/src/commands/train_output.rs","line":77},{"file":"crates/apr-cli/src/commands/train_output.rs","line":78},{"file":"crates/apr-cli/src/commands/train_output.rs","line":79},{"file":"crates/apr-cli/src/commands/train_output.rs","line":80},{"file":"crates/apr-cli/src/commands/train_output.rs","line":81},{"file":"crates/apr-cli/src/commands/train_output.rs","line":82},{"file":"crates/apr-cli/src/commands/train_output.rs","line":83},{"file":"crates/apr-cli/src/commands/train_output.rs","line":84},{"file":"crates/apr-cli/src/commands/train_output.rs","line":85},{"file":"crates/apr-cli/src/commands/train_output.rs","line":86},{"file":"crates/apr-cli/src/commands/train_output.rs","line":87},{"file":"crates/apr-cli/src/commands/train_output.rs","line":88},{"file":"crates/apr-cli/src/commands/train_output.rs","line":89},{"file":"crates/apr-cli/src/commands/train_output.rs","line":90},{"file":"crates/apr-cli/src/commands/train_output.rs","line":91},{"file":"crates/apr-cli/src/commands/train_output.rs","line":92},{"file":"crates/apr-cli/src/commands/train_output.rs","line":93},{"file":"crates/apr-cli/src/commands/train_output.rs","line":94},{"file":"crates/apr-cli/src/commands/train_output.rs","line":95},{"file":"crates/apr-cli/src/commands/train_output.rs","line":96},{"file":"crates/apr-cli/src/commands/train_output.rs","line":97},{"file":"crates/apr-cli/src/commands/train_output.rs","line":98},{"file":"crates/apr-cli/src/commands/train_output.rs","line":99},{"file":"crates/apr-cli/src/commands/train_output.rs","line":100},{"file":"crates/apr-cli/src/commands/train_output.rs","line":101},{"file":"crates/apr-cli/src/commands/train_output.rs","line":102},{"file":"crates/apr-cli/src/commands/train_output.rs","line":103},{"file":"crates/apr-cli/src/commands/train_output.rs","line":104},{"file":"crates/apr-cli/src/commands/train_output.rs","line":105},{"file":"crates/apr-cli/src/commands/train_output.rs","line":106},{"file":"crates/apr-cli/src/commands/train_output.rs","line":107},{"file":"crates/apr-cli/src/commands/train_output.rs","line":108},{"file":"crates/apr-cli/src/commands/train_output.rs","line":109},{"file":"crates/apr-cli/src/commands/train_output.rs","line":110},{"file":"crates/apr-cli/src/commands/train_output.rs","line":111},{"file":"crates/apr-cli/src/commands/train_output.rs","line":112},{"file":"crates/apr-cli/src/commands/train_output.rs","line":113},{"file":"crates/apr-cli/src/commands/train_output.rs","line":114},{"file":"crates/apr-cli/src/commands/train_output.rs","line":115},{"file":"crates/apr-cli/src/commands/train_output.rs","line":116},{"file":"crates/apr-cli/src/commands/train_output.rs","line":117},{"file":"crates/apr-cli/src/commands/train_output.rs","line":118},{"file":"crates/apr-cli/src/commands/train_output.rs","line":119},{"file":"crates/apr-cli/src/commands/train_output.rs","line":120},{"file":"crates/apr-cli/src/commands/train_output.rs","line":121},{"file":"crates/apr-cli/src/commands/train_output.rs","line":122},{"file":"crates/apr-cli/src/commands/train_output.rs","line":123},{"file":"crates/apr-cli/src/commands/train_output.rs","line":124},{"file":"crates/apr-cli/src/commands/train_output.rs","line":125},{"file":"crates/apr-cli/src/commands/train_output.rs","line":126},{"file":"crates/apr-cli/src/commands/train_output.rs","line":127},{"file":"crates/apr-cli/src/commands/train_output.rs","line":128},{"file":"crates/apr-cli/src/commands/train_output.rs","line":129},{"file":"crates/apr-cli/src/commands/train_output.rs","line":130},{"file":"crates/apr-cli/src/commands/train_output.rs","line":131},{"file":"crates/apr-cli/src/commands/train_output.rs","line":132},{"file":"crates/apr-cli/src/commands/train_output.rs","line":133},{"file":"crates/apr-cli/src/commands/train_output.rs","line":134},{"file":"crates/apr-cli/src/commands/train_output.rs","line":135},{"file":"crates/apr-cli/src/commands/train_output.rs","line":136},{"file":"crates/apr-cli/src/commands/train_output.rs","line":137},{"file":"crates/apr-cli/src/commands/train_output.rs","line":138},{"file":"crates/apr-cli/src/commands/train_output.rs","line":139},{"file":"crates/apr-cli/src/commands/train_output.rs","line":140},{"file":"crates/apr-cli/src/commands/train_output.rs","line":141},{"file":"crates/apr-cli/src/commands/train_output.rs","line":142},{"file":"crates/apr-cli/src/commands/train_output.rs","line":143},{"file":"crates/apr-cli/src/commands/train_output.rs","line":144},{"file":"crates/apr-cli/src/commands/train_output.rs","line":145},{"file":"crates/apr-cli/src/commands/train_output.rs","line":146},{"file":"crates/apr-cli/src/commands/train_output.rs","line":147},{"file":"crates/apr-cli/src/commands/train_output.rs","line":148},{"file":"crates/apr-cli/src/commands/train_output.rs","line":149},{"file":"crates/apr-cli/src/commands/train_output.rs","line":150},{"file":"crates/apr-cli/src/commands/train_output.rs","line":151},{"file":"crates/apr-cli/src/commands/train_output.rs","line":152},{"file":"crates/apr-cli/src/commands/train_output.rs","line":153},{"file":"crates/apr-cli/src/commands/train_output.rs","line":154},{"file":"crates/apr-cli/src/commands/train_output.rs","line":155},{"file":"crates/apr-cli/src/commands/train_output.rs","line":156},{"file":"crates/apr-cli/src/commands/train_output.rs","line":157},{"file":"crates/apr-cli/src/commands/train_output.rs","line":158},{"file":"crates/apr-cli/src/commands/train_output.rs","line":159},{"file":"crates/apr-cli/src/commands/train_output.rs","line":160},{"file":"crates/apr-cli/src/commands/train_output.rs","line":161},{"file":"crates/apr-cli/src/commands/train_output.rs","line":162},{"file":"crates/apr-cli/src/commands/train_output.rs","line":163},{"file":"crates/apr-cli/src/commands/train_output.rs","line":164},{"file":"crates/apr-cli/src/commands/train_output.rs","line":165},{"file":"crates/apr-cli/src/commands/train_output.rs","line":166},{"file":"crates/apr-cli/src/commands/train_output.rs","line":167},{"file":"crates/apr-cli/src/commands/train_output.rs","line":168},{"file":"crates/apr-cli/src/commands/train_output.rs","line":169},{"file":"crates/apr-cli/src/commands/train_output.rs","line":170},{"file":"crates/apr-cli/src/commands/train_output.rs","line":171},{"file":"crates/apr-cli/src/commands/train_output.rs","line":172},{"file":"crates/apr-cli/src/commands/train_output.rs","line":173},{"file":"crates/apr-cli/src/commands/train_output.rs","line":174},{"file":"crates/apr-cli/src/commands/train_output.rs","line":175},{"file":"crates/apr-cli/src/commands/train_output.rs","line":176},{"file":"crates/apr-cli/src/commands/train_output.rs","line":177},{"file":"crates/apr-cli/src/commands/train_output.rs","line":178},{"file":"crates/apr-cli/src/commands/train_output.rs","line":179},{"file":"crates/apr-cli/src/commands/train_output.rs","line":180},{"file":"crates/apr-cli/src/commands/train_output.rs","line":181},{"file":"crates/apr-cli/src/commands/train_output.rs","line":182},{"file":"crates/apr-cli/src/commands/train_output.rs","line":183},{"file":"crates/apr-cli/src/commands/train_output.rs","line":184},{"file":"crates/apr-cli/src/commands/train_output.rs","line":185},{"file":"crates/apr-cli/src/commands/train_output.rs","line":186},{"file":"crates/apr-cli/src/commands/train_output.rs","line":187},{"file":"crates/apr-cli/src/commands/train_output.rs","line":188},{"file":"crates/apr-cli/src/commands/train_output.rs","line":189},{"file":"crates/apr-cli/src/commands/train_output.rs","line":190},{"file":"crates/apr-cli/src/commands/train_output.rs","line":191},{"file":"crates/apr-cli/src/commands/train_output.rs","line":192},{"file":"crates/apr-cli/src/commands/train_output.rs","line":193},{"file":"crates/apr-cli/src/commands/train_output.rs","line":194},{"file":"crates/apr-cli/src/commands/train_output.rs","line":195},{"file":"crates/apr-cli/src/commands/train_output.rs","line":196},{"file":"crates/apr-cli/src/commands/train_output.rs","line":197},{"file":"crates/apr-cli/src/commands/train_output.rs","line":198},{"file":"crates/apr-cli/src/commands/train_output.rs","line":199},{"file":"crates/apr-cli/src/commands/train_output.rs","line":200},{"file":"crates/apr-cli/src/commands/train_output.rs","line":201},{"file":"crates/apr-cli/src/commands/train_output.rs","line":202},{"file":"crates/apr-cli/src/commands/train_output.rs","line":203},{"file":"crates/apr-cli/src/commands/train_output.rs","line":204},{"file":"crates/apr-cli/src/commands/train_output.rs","line":205},{"file":"crates/apr-cli/src/commands/train_output.rs","line":206},{"file":"crates/apr-cli/src/commands/train_output.rs","line":207},{"file":"crates/apr-cli/src/commands/train_output.rs","line":208},{"file":"crates/apr-cli/src/commands/train_output.rs","line":209},{"file":"crates/apr-cli/src/commands/train_output.rs","line":210},{"file":"crates/apr-cli/src/commands/train_output.rs","line":211},{"file":"crates/apr-cli/src/commands/train_output.rs","line":212},{"file":"crates/apr-cli/src/commands/train_output.rs","line":213},{"file":"crates/apr-cli/src/commands/train_output.rs","line":214},{"file":"crates/apr-cli/src/commands/train_output.rs","line":215},{"file":"crates/apr-cli/src/commands/train_output.rs","line":216},{"file":"crates/apr-cli/src/commands/train_output.rs","line":217},{"file":"crates/apr-cli/src/commands/train_output.rs","line":218},{"file":"crates/apr-cli/src/commands/train_output.rs","line":219},{"file":"crates/apr-cli/src/commands/train_output.rs","line":220},{"file":"crates/apr-cli/src/commands/train_output.rs","line":221},{"file":"crates/apr-cli/src/commands/train_output.rs","line":222},{"file":"crates/apr-cli/src/commands/train_output.rs","line":223},{"file":"crates/apr-cli/src/commands/train_output.rs","line":224},{"file":"crates/apr-cli/src/commands/train_output.rs","line":225},{"file":"crates/apr-cli/src/commands/train_output.rs","line":226},{"file":"crates/apr-cli/src/commands/train_output.rs","line":227},{"file":"crates/apr-cli/src/commands/train_output.rs","line":228},{"file":"crates/apr-cli/src/commands/train_output.rs","line":229},{"file":"crates/apr-cli/src/commands/train_output.rs","line":230},{"file":"crates/apr-cli/src/commands/train_output.rs","line":231},{"file":"crates/apr-cli/src/commands/train_output.rs","line":232},{"file":"crates/apr-cli/src/commands/train_output.rs","line":233},{"file":"crates/apr-cli/src/commands/train_output.rs","line":234},{"file":"crates/apr-cli/src/commands/train_output.rs","line":235},{"file":"crates/apr-cli/src/commands/train_output.rs","line":236},{"file":"crates/apr-cli/src/commands/train_output.rs","line":237},{"file":"crates/apr-cli/src/commands/train_output.rs","line":238},{"file":"crates/apr-cli/src/commands/train_output.rs","line":239},{"file":"crates/apr-cli/src/commands/train_output.rs","line":240},{"file":"crates/apr-cli/src/commands/train_output.rs","line":241},{"file":"crates/apr-cli/src/commands/train_output.rs","line":242},{"file":"crates/apr-cli/src/commands/train_output.rs","line":243},{"file":"crates/apr-cli/src/commands/train_output.rs","line":244},{"file":"crates/apr-cli/src/commands/train_output.rs","line":245},{"file":"crates/apr-cli/src/commands/train_output.rs","line":246},{"file":"crates/apr-cli/src/commands/train_output.rs","line":247},{"file":"crates/apr-cli/src/commands/train_output.rs","line":248},{"file":"crates/apr-cli/src/commands/train_output.rs","line":249},{"file":"crates/apr-cli/src/commands/train_output.rs","line":250},{"file":"crates/apr-cli/src/commands/train_output.rs","line":251},{"file":"crates/apr-cli/src/commands/train_output.rs","line":252},{"file":"crates/apr-cli/src/commands/train_output.rs","line":253},{"file":"crates/apr-cli/src/commands/train_output.rs","line":254},{"file":"crates/apr-cli/src/commands/train_output.rs","line":255},{"file":"crates/apr-cli/src/commands/train_output.rs","line":256},{"file":"crates/apr-cli/src/commands/train_output.rs","line":257},{"file":"crates/apr-cli/src/commands/train_output.rs","line":258},{"file":"crates/apr-cli/src/commands/train_output.rs","line":259},{"file":"crates/apr-cli/src/commands/train_output.rs","line":260},{"file":"crates/apr-cli/src/commands/train_output.rs","line":261},{"file":"crates/apr-cli/src/commands/train_output.rs","line":262},{"file":"crates/apr-cli/src/commands/train_output.rs","line":263},{"file":"crates/apr-cli/src/commands/train_output.rs","line":264},{"file":"crates/apr-cli/src/commands/train_output.rs","line":265},{"file":"crates/apr-cli/src/commands/train_output.rs","line":266},{"file":"crates/apr-cli/src/commands/train_output.rs","line":267},{"file":"crates/apr-cli/src/commands/train_output.rs","line":268},{"file":"crates/apr-cli/src/commands/train_output.rs","line":269},{"file":"crates/apr-cli/src/commands/train_output.rs","line":270},{"file":"crates/apr-cli/src/commands/train_output.rs","line":271},{"file":"crates/apr-cli/src/commands/train_output.rs","line":272},{"file":"crates/apr-cli/src/commands/train_output.rs","line":273},{"file":"crates/apr-cli/src/commands/train_output.rs","line":274},{"file":"crates/apr-cli/src/commands/train_output.rs","line":275},{"file":"crates/apr-cli/src/commands/train_output.rs","line":276},{"file":"crates/apr-cli/src/commands/train_output.rs","line":277},{"file":"crates/apr-cli/src/commands/train_output.rs","line":278},{"file":"crates/apr-cli/src/commands/train_output.rs","line":279},{"file":"crates/apr-cli/src/commands/train_output.rs","line":280},{"file":"crates/apr-cli/src/commands/train_output.rs","line":281},{"file":"crates/apr-cli/src/commands/train_output.rs","line":282},{"file":"crates/apr-cli/src/commands/train_output.rs","line":283},{"file":"crates/apr-cli/src/commands/train_output.rs","line":284},{"file":"crates/apr-cli/src/commands/train_output.rs","line":285},{"file":"crates/apr-cli/src/commands/train_output.rs","line":286},{"file":"crates/apr-cli/src/commands/train_output.rs","line":287},{"file":"crates/apr-cli/src/commands/train_output.rs","line":288},{"file":"crates/apr-cli/src/commands/train_output.rs","line":289},{"file":"crates/apr-cli/src/commands/train_output.rs","line":290},{"file":"crates/apr-cli/src/commands/train_output.rs","line":291},{"file":"crates/apr-cli/src/commands/train_output.rs","line":292},{"file":"crates/apr-cli/src/commands/train_output.rs","line":293},{"file":"crates/apr-cli/src/commands/train_output.rs","line":294},{"file":"crates/apr-cli/src/commands/train_output.rs","line":295},{"file":"crates/apr-cli/src/commands/train_output.rs","line":296},{"file":"crates/apr-cli/src/commands/train_output.rs","line":297},{"file":"crates/apr-cli/src/commands/train_output.rs","line":298},{"file":"crates/apr-cli/src/commands/train_output.rs","line":299},{"file":"crates/apr-cli/src/commands/train_output.rs","line":300},{"file":"crates/apr-cli/src/commands/train_output.rs","line":301},{"file":"crates/apr-cli/src/commands/train_output.rs","line":302},{"file":"crates/apr-cli/src/commands/train_output.rs","line":303},{"file":"crates/apr-cli/src/commands/train_output.rs","line":304},{"file":"crates/apr-cli/src/commands/train_output.rs","line":305},{"file":"crates/apr-cli/src/commands/train_output.rs","line":306},{"file":"crates/apr-cli/src/commands/train_output.rs","line":307},{"file":"crates/apr-cli/src/commands/train_output.rs","line":308},{"file":"crates/apr-cli/src/commands/train_output.rs","line":309},{"file":"crates/apr-cli/src/commands/train_output.rs","line":310},{"file":"crates/apr-cli/src/commands/train_output.rs","line":311},{"file":"crates/apr-cli/src/commands/train_output.rs","line":312},{"file":"crates/apr-cli/src/commands/train_output.rs","line":313},{"file":"crates/apr-cli/src/commands/train_output.rs","line":314},{"file":"crates/apr-cli/src/commands/train_output.rs","line":315},{"file":"crates/apr-cli/src/commands/train_output.rs","line":316},{"file":"crates/apr-cli/src/commands/train_output.rs","line":317},{"file":"crates/apr-cli/src/commands/train_output.rs","line":318},{"file":"crates/apr-cli/src/commands/train_output.rs","line":319},{"file":"crates/apr-cli/src/commands/train_output.rs","line":320},{"file":"crates/apr-cli/src/commands/train_output.rs","line":321},{"file":"crates/apr-cli/src/commands/train_output.rs","line":322},{"file":"crates/apr-cli/src/commands/train_output.rs","line":323},{"file":"crates/apr-cli/src/commands/train_output.rs","line":324},{"file":"crates/apr-cli/src/commands/train_output.rs","line":325},{"file":"crates/apr-cli/src/commands/train_output.rs","line":326},{"file":"crates/apr-cli/src/commands/train_output.rs","line":327},{"file":"crates/apr-cli/src/commands/train_output.rs","line":328},{"file":"crates/apr-cli/src/commands/train_output.rs","line":329},{"file":"crates/apr-cli/src/commands/train_output.rs","line":330},{"file":"crates/apr-cli/src/commands/train_output.rs","line":331},{"file":"crates/apr-cli/src/commands/train_output.rs","line":332},{"file":"crates/apr-cli/src/commands/train_output.rs","line":333},{"file":"crates/apr-cli/src/commands/train_output.rs","line":334},{"file":"crates/apr-cli/src/commands/train_output.rs","line":335},{"file":"crates/apr-cli/src/commands/train_output.rs","line":336},{"file":"crates/apr-cli/src/commands/train_output.rs","line":337},{"file":"crates/apr-cli/src/commands/train_output.rs","line":338},{"file":"crates/apr-cli/src/commands/train_output.rs","line":339},{"file":"crates/apr-cli/src/commands/train_output.rs","line":340},{"file":"crates/apr-cli/src/commands/train_output.rs","line":341},{"file":"crates/apr-cli/src/commands/train_output.rs","line":342},{"file":"crates/apr-cli/src/commands/train_output.rs","line":343},{"file":"crates/apr-cli/src/commands/train_output.rs","line":344},{"file":"crates/apr-cli/src/commands/train_output.rs","line":345},{"file":"crates/apr-cli/src/commands/train_output.rs","line":346},{"file":"crates/apr-cli/src/commands/train_output.rs","line":347},{"file":"crates/apr-cli/src/commands/train_output.rs","line":348},{"file":"crates/apr-cli/src/commands/train_output.rs","line":349},{"file":"crates/apr-cli/src/commands/train_output.rs","line":350},{"file":"crates/apr-cli/src/commands/train_output.rs","line":351},{"file":"crates/apr-cli/src/commands/train_output.rs","line":352},{"file":"crates/apr-cli/src/commands/train_output.rs","line":353},{"file":"crates/apr-cli/src/commands/train_output.rs","line":354},{"file":"crates/apr-cli/src/commands/train_output.rs","line":355},{"file":"crates/apr-cli/src/commands/train_output.rs","line":356},{"file":"crates/apr-cli/src/commands/train_output.rs","line":357},{"file":"crates/apr-cli/src/commands/train_output.rs","line":358},{"file":"crates/apr-cli/src/commands/train_output.rs","line":359},{"file":"crates/apr-cli/src/commands/train_output.rs","line":360}]} +{"id":"R-pr750","class":"R","stratum":"M","split":"Test","diff_sha256":"1c288258c6724f6d343c02fb5288fcf62e76264e3c163b6dd459856f334abebf","bytes":20088,"approx_tokens":5022,"source":"pr#750 7f74543b2db381a632447da62533c53fff614b82 reversed; fixes #478","defect":[{"file":"crates/aprender-serve/src/gguf/inference/fused_matmul_into.rs","line":544},{"file":"crates/aprender-serve/src/gguf/inference/matmul.rs","line":8},{"file":"crates/aprender-serve/src/gguf/inference/matmul.rs","line":9},{"file":"crates/aprender-serve/src/gguf/inference/matmul_fused.rs","line":154},{"file":"crates/aprender-serve/src/gguf/loader_apr_quantized.rs","line":15},{"file":"crates/aprender-serve/src/gguf/loader_apr_quantized.rs","line":34},{"file":"crates/aprender-serve/src/gguf/loader_apr_quantized.rs","line":35},{"file":"crates/aprender-serve/src/gguf/loader_apr_quantized.rs","line":36},{"file":"crates/aprender-serve/src/gguf/loader_apr_quantized.rs","line":37},{"file":"crates/aprender-serve/src/gguf/loader_apr_quantized.rs","line":38},{"file":"crates/aprender-serve/src/gguf/loader_apr_quantized.rs","line":39},{"file":"crates/aprender-serve/src/gguf/loader_apr_quantized.rs","line":40},{"file":"crates/aprender-serve/src/gguf/loader_apr_quantized.rs","line":41},{"file":"crates/aprender-serve/src/gguf/loader_apr_quantized.rs","line":43},{"file":"crates/aprender-serve/src/gguf/loader_apr_quantized.rs","line":44},{"file":"crates/aprender-serve/src/gguf/loader_apr_quantized.rs","line":45},{"file":"crates/aprender-serve/src/gguf/loader_apr_quantized.rs","line":55},{"file":"crates/aprender-serve/src/gguf/loader_apr_quantized.rs","line":56},{"file":"crates/aprender-serve/src/gguf/loader_apr_quantized.rs","line":57},{"file":"crates/aprender-serve/src/gguf/loader_apr_quantized.rs","line":58},{"file":"crates/aprender-serve/src/gguf/loader_apr_quantized.rs","line":59},{"file":"crates/aprender-serve/src/gguf/loader_apr_quantized.rs","line":60},{"file":"crates/aprender-serve/src/gguf/loader_apr_quantized.rs","line":61},{"file":"crates/aprender-serve/src/gguf/loader_apr_quantized.rs","line":63},{"file":"crates/aprender-serve/src/gguf/loader_apr_quantized.rs","line":64},{"file":"crates/aprender-serve/src/gguf/loader_apr_quantized.rs","line":65},{"file":"crates/aprender-serve/src/gguf/loader_apr_quantized.rs","line":450},{"file":"crates/aprender-serve/src/gguf/types.rs","line":68},{"file":"crates/aprender-serve/src/gpu/adapters/wgpu_adapter.rs","line":229},{"file":"crates/aprender-serve/src/gpu/adapters/wgpu_adapter.rs","line":247}]} diff --git a/docs/audits/review-corpus/prompts/v1.txt b/docs/audits/review-corpus/prompts/v1.txt new file mode 100644 index 0000000000..e6c510727b --- /dev/null +++ b/docs/audits/review-corpus/prompts/v1.txt @@ -0,0 +1,2 @@ +You are a code reviewer. Review this diff. Reply with VERDICT: PASS or FAIL on the first line, then at most 5 short bullet findings (bugs, risks). Be concrete. + diff --git a/docs/audits/review-corpus/test-manifest-v1.txt b/docs/audits/review-corpus/test-manifest-v1.txt new file mode 100644 index 0000000000..e3c409620a --- /dev/null +++ b/docs/audits/review-corpus/test-manifest-v1.txt @@ -0,0 +1,106 @@ +review-corpus-v1 test-manifest +G-pr1026 dbf2e76f07bdf31bde4a9c92e2d8698c46b80224ad101876525711f71afca173 a6ab9abe8b4f72f4ed32ab104b0e7d204dacfb8a0dac1b708b5356a1646b5ac9,e514fa602721d06b27de466ae073a5c68dfc6b07f3415407dcf4cb0d14c42746 +G-pr1052 5e72ec8a21cc3687e1bc84eb3b99b58fe737f7fabe7788546348dbddbed1bc95 055745d46bfb18604834f85906136c26b5c8aa2c2aaafa77837470a3307a7b41,0bc386ff8e00fa7aba0891d12d8c5890d5a720a4a7bed5c0ec9147583f0a8e5f,290e51555532d80841c95fb5a78debca12495b685937067fd08c7230198e3ebf,453ed76bc7d81690ddf02a42abef7585ec4544e92e3cefec3694e310f0bd4b79,759e671b0b150fa4c0e97a05660234b92fd03193f6840b2d9b107220a38598b7,7a39f07a083e76c3bcad33f6f09e5bec6aa795e142c36346d68df6cc4e3c191b,7a3eddde3abe5e1e2f26845a2dfd2e7609c6afda692d347f4af1b0f0eaab7272,87ed621aaec09c5eb6421e34ae8d4f69cb53b91aa8049134de37efbafe40b547,88780b9fa72bb6020f95deeb7c5b76080630beeeec665b4ee04f880cb2a84e96,89c40f0de4974cc092f939b12076dbff1842d7ad92b335d207623d78d691dc9e,963744d8cb60b6d7e9aadbc7c54a174ac13b9b3e3a7e8c9b97f6b5d59de36e16,a8ded673c4360c01359c509f0e692d934fe23441a76d2486659a9482cb3cd7f8,af6713997f17801210341f195031ef2ba5b1c1c4a6c1acbe9b9e8f39ed613c79,afff0e356bbd8e3e360a25f647822582204b47ca9644aad40e17b4230ceae016,b1430d785846907e69432853d8121be2170481a96f756d1f76d22cc4c0a14971,b5cbfa8a5ca7b6904c41e86149f948ba465d873cbd96f9aa1df4c9de8c42424f,c4ab9f2bdf11fdba478698cb12ca5cf5e25ecf0a902e63779a7344c24a66846f,d24d366878f43802f96de2095088a80f6bceed9892004d43e0c2e26390dc9d7a +G-pr1058 52b05d908f3e82ae8c8a0a2fd25a91b4a1a8fbc035892ca41a35b779ec7c5296 e0c0de77ea035c6fc248ee737382944f81dcad8cd8427d6266ff5e5ed92d3509 +G-pr1081 d2091f65f7ed18515023b8905f660ccda8ae9bc26cf72d9b398e2a70ef12fdec 07e1ea66e481e0dfd92a0139c94b82ce7f22f33287d1c84edcab7fb4934fb1c9,1cc8374217577fbc76d748417c4a15b337eb7747c217e295518f250b3b0163a3,60979f4a130018670ddf07b06b3d0479505c92f716b22294202745ebb93a75aa,998d8cf3d3033290d11a5a121174192e7db4c39689a17f62dbd180257128e677,e2a1311232733f92aacb6e49d5d1cfe518a1b3f91b371f868089d3d7afd9d040 +G-pr1136 9631361a48f554f6dddd39803ed12ac599663ef3bc18458b4e64273589e0eeec 040cffaea49bd21c81ebbcc1081cd25c5e9ef46d7b1f9c2ab7f391e367ddbfe3,fd3f37a793b9e9ba7709317f7c4c87ad0a053e65cf8c80e7c612208c8c71c509 +G-pr1408 b5a1581d712f18d2f11c32d33ed588a358f79fcc706f9441705e66fe75fa0bfb 4eedcb38ab8bd6b705c5aaa8569f92e408a4eb2ed8a1ae42bb746840dc922f96,e316f27198cc1189ffef7bd7cb1f2543567644398fd3f4971b7295daebaa486c +G-pr1516 8a64769fd7a2021c6559ed5334f4b8519b71e30a656c0e688c985fdcb47d857f 183f98c8b619e119458cd7ff4dc081264c4c2dd65162601a31e3614fab3d8993,50cc7ca494661f9ed3fce5a2e73f251bdb8216ec5e012bbfd25e15bd04b9c992,871d7d856b84293e7025c399fd95268fff2781b78ddbd1b26d9590bce9c87628 +G-pr1536 ede8268135e2f9e3d03eefe0237a494ab458eebea31bd518f1e6d3b14bb28fbc 159d24fdb261516c06fba0834a40fbc5ce5f8fdffe0e061f90a58d7e973b756e +G-pr1543 fbbfea4dfbb011561f92c791a1399818cccc4951580defb1fe217c20a37291a4 022749e3e236150cd41cce3c52c9858d91731a8a1533a5b56595e16a7f8b9c86 +G-pr1555 f0dc753d991a0fc4f2649e6efe4d31c259f510e4b620080f7b74e950f02167eb 109abcb7c280287a4b9a17088e28a22836aa0a48408491742829ff0acb0c009a,1789cc0894aca19d666192bae4ad5dd4787178b3c1a531ac061029d4cd345588,1c6cddc1f5ac7c44345316bd87a87cf453964f9dc1b81644ab274607b75c3396,88fe604b882d135308f0c823caa9d94079c41d0dff7932dca99e60faf33fc5a5,e909aa603cfb24eb2a9166044042abf910b8fc3a6e7328bc78c8f1c36fcbbe9c +G-pr1568 9826cd5a6fcc1cbac765a87322a66dffbabe7149ac76e7a095642a9482d9d9cb 52227199a387b86f578a391edf5e1bc9b0db3cf2d0dc91f2bdfdfe86dd74db8a,5b618edc76df2721985316cbe53f9714192014e0ea550690a2e6a056c1081231,fb2c903f779dc6cd8727ea05e0879c5f17119353e907cd73b41aa04f670156b5 +G-pr1602 0e8e87d8a2b062f04e24c651677a13e50c75ab032f1891388dffd83fc2118287 2d646476c6478cfb3a6b389fe9997f1b3c2f075fe6e277c04f6a281eed7a295f,356946668fe2bb11a056c568e9a3bdb1d795dbe3ab40a39fb5358fb770095460,382b5aa065eee0a0a5686ad9e035629feab8120048c70a9a535824b7b366b727,388d4a31fd38f951f2ad679158f1e8e1a6b358ef5266bb2261277aad561bde2b,424a497bed8321b8ea49c0c6c57f1a4385001e60aab72e453e721b76a0cde053,578be6c034e399f984e805aeb7ae4b87c65277402caf9bbe8ed7e74371bcd883,5fb762d62b23b435c894b46236314cea6411f12ce07ab0f0e32af601aad5b28e,87c764757887ecdd96236c31fd846886a928f91676886427931fb50cc7b5fbe8,8f2386e0c0726b4f6c1ffd3eb2416031348ab29afa050b9b082d783f92f5f82c,958258ce3e506188d09a4e6fe71072547826209109b96fae5aff3717396e8796,a36522f935ee2ecd7de656baf95b6e1a77c6a3cc3de2edd1b5fd692af1ea5021,ad40cc0fd00cacd5e3b814437cfd886296338dfaa828d332c5e9d8a6de1805ba,da128ceec6f8f764087bc749f4a978a1fe42264d014c7a2d4a19f72ddca795d0,f03d1de58c2845804be265049f81f92394ebd741b8bd6a4aa8bd2c1fa7d90ddc +G-pr1664 03fe31a5ed2150c50a8f2ed444803f49bacf6dd3edd304d38bd61dc9f5ba370b 00f157ba14778d4bd56e635b9d477725fee46b36837f50d45d445ae468400ddc,042e325aa478ac34c8c54986c3d9e659ced9ffba549c553b81a07a7cfaf9cb65,65f9bb699191f9bbfc35b5d848f62c458a7ac04732bbd6ca3b667c5262a9aa2a,935f17f80560e1286e94b18154fd901aaaafd8dc63639f58bb147e575746f2de,a141ca8091af7efec1d1c6853f427c2b36e0923a8fecfa669538d20b40b5faef,c1c38d1940d6a7e9839d9dcb4c35ccc4dddae76171f0f15ca6e97f132a168fb6 +G-pr1694 565c6d9d2ac74ddb3174bd31894a5ecbbcee13d5d227a0cd451cc16c6a6b14ee 1ee03d9132c2674ba4c8e5b06cb3f76e06f19dab1c7b99315c3ce6f9fbbde7c6,26780fe819ea0fc5a5cd48260d9890e3e15ab050471d2ec0bae299f9840493bd,55cdd795ea10b63c332d28048fe4c11d514f8e691d17c56e169da569ed8ef818,778030396b345ddf2b1173f87f70e6136486939631ac4e703d90e116888f1d66,a0d9be96254bd7f31d60dcdfc3d7186f25834e7493f1b610015cb889f94c215a,d926c6057772c2f86a00565c52cbfdcb107a66c27342e3fc6d9b6d75443ae872 +G-pr1779 0220fcea591f9c9bb66c9784dbb17d321ff41420790014b628838d6322694de5 12118f559c83e6d70058fa939a2a106a17f6f83dac604dc7a257be816971a9e9,1cfb8bfca4fc40bbb55e9dc5a4602cf357a7ea0b1715094672a69f74652c6961,47b68d571b6a42d2220ca96302c7f9cf51918efbab413e4b41ff79868d65ebdb,54b8fb9bae31de0c24d0ac386f7e65c752c9bea7e47c78ac513ed0fb5b4c8a47,56796cad5d11f393a01f63dc169bf2232c0af281ccd9a165c1b8359a9595e7f6,568f267ff970c05314997675f12796d40a88436abf8dc304dce02784aa8f6222,6d6327e1f70210275cbebb8fb6122bf8fd39ab0c9d83336fe6ccddc3d0b4b53d,84e7503e0540edebd785d7a7ee691f8018576ef15ad32a3b214646c4997939f6,9adb45340d78b424fb788ae5ac4297eb9995cf0a5afd772e69a33a85c4718321,9b5f15a1bbbcfd5aa9051687de24161a377629cbcb7a1d67b13febdabfc97389,b3d8b393eb9a28a78038a8f17c78e2589a3acf03483ac1fb1f545b40a03e1284,bc3402123cb679f4aa6da71f5e8c72a1fb6ef3a1bf345851c6aa2377eb1ded72,e9fbe89e12df331eb40ad4296f3de3d469b5229307b136b0dd39fd3cfa7d9200 +G-pr1804 e31677b8f712468175040fec4f3ae1be70f7973d62aa83884a38a307c425c9da 3d46d5ab45e11630d6569de70435c80730014ee435649f327604f34a154aec92 +G-pr1919 fffd8169289f9084d6b09fd2ec19c49b18592f2d4d1e4d0fc09aaeee78936ad4 43683cd57ffdefa713a98ba88b898c32b3d83bdc9b472b476e69fd86b0a9de65,77bea2b3e8b11489363c2aa6ad614d33ff8bd7fe12405919c72409ee4f2a9eff,7a743e36dc56a0eb64a9d9182d6218a3faee74dfb4fbd88d5907f0e161e80bf3,a0b014631663a5847b6f082a5a183f4c862210abaa8576b3e5c0faeba7c6033e,c5986d748d9d485e7779896c5dd9ff0a37dd39f23bb0fec1893a435d574f1d2e,cf8e3c668874ea7cc26f047b6c0c21376f11fa4a49f4f84553f186d24c4d8ece,d93b367bd1498d6bdf3aceeea522cd6a3e83192a5ab23fb25d32fea587bee576 +G-pr1962 f5422e3d66d0dce51be91b3c04f513962c9a2727bfc33fc38e772818debbebc1 7f320fa80f83110ca7215b6e1524dd8c511ce13096fa93f3df1545b8c2550790,c75c264635722f11d22278f89231d4261b699a73bc23e2c76aa5a10c638f8207 +G-pr2064 c74635eb50fc1949e0a49941b83f15f36b9422c516e2ac25d42b861d9f9bb4ce 00430ae39f5caa1256c0e1199c2172762abca97fd4d11ad52f708a040e935b8e,08699374dfe776575fd013a67c922b7e8862979b4b0f0fda63bbf519c9582dcc,185b8719cb76021915cf19039b5331f48140bb996e5b97a876a12842ccd6ae8e,1b47f4f1f6ef4ab08db9f5f8296cdbb829408bc41dbe68bb2ee7d4573f8b0757,2ee4fdafbd03ce75825546feb88bc52c7fe89fd0cd974ca79a23254b888febe1,33c28c95f6ddbb735f63a0d1b0518757cee53cf4d9f9a5c3520ec1539c920cac,349af1afef6869addc719b9c1f70fa576936d76d851e0e065e5e049b81719230,401e9c7e2f2b326209d63bfbff0753f856bcf4db284068c02ded198b28c198e7,463debd2e9ab97dcfbf95464e1f726597c2edc0542375ac928a7b4e8d584411f,46ff7a0e0d881994d88b78822067888e8af17ad774f69b7b80d4441bf71853db,54d012b2fbb9efce755610418f3510b419d7d6fd2a82a68f9a5b7d23123cb7e5,553760dad2e34bc3c823a5744356a7ab22448507da2f59c41f7784408345e4a2,5bde34a55125bdd4bef92650e4fd501eeb2e376505a3ef7e1eef0f5f47552e6b,64712a413aa855f16e4322410c5b4605e822b0ee64a15c9cd1f2f16a65d46ef0,6692e9c69d8ae69e714ed9544bc51d77ba3a8ae0655069b28cf2715236d3b93d,6d74a9c0686f8a38b52279509eaef328d8de848e87e6d64b619780e34abf4b0d,7481bd880056858c8a985a577bf4cb69bbadf7fcb50de6b553a77152c6264081,7694a1543ffadf0a681e5c0ca30e3e8ec8d2899eb969e78686c830e4a3f99be6,7a1513262730e93f279d6f3374b9fd7f7f043fd290be47b3c5ca05a1c2d4f092,844e189b8585a0dd0f0a8923d509d35108661e1c5b4cafc12a2e6941c6dd55c5,84adc79afb774b3564c6216f838d2f89732c21784b1c2f9b112bc00d7f7b08fc,8938873c3b3e7b90cf7574a3177c04b261695c1c00caea2411e2caae29e8f9d4,8be77df859a220521202e3fd983f21c931adb83c8af11fdd6c08861c916c14c8,906eb1f63d72e95c13d5dafc85812c89fe3e5cd2a999c850a6a0adc770ee4226,922ff9aa98748eac0513f9fbdddd261dea25847978eba791c5fb30c67907a7fc,a48ffd31aa7e8251d7454a4dda4e45ca87e176daab6a194f13c7b425f7ac4c61,a61e4a62e902a34d3b3c58a6a181cd87d7ad2bf0c952956fa45a23226d92ed58,a737942f690fbe3f64fa4355b4d8ce6aefc6202886d7cd3743eabf5c6d376277,ad206e476d9a8953cc02c7ff8c423e94aa3a7722a3d5b7c1de25571023c1850a,adcf4ef6d9ad982d20be7983398218751596b47aa32a9a93449247c1aa731a2e,af916ff4198a2e0cd2a36880470beb038e4df019c992fa5f169b383b4d88c2c2,b2aca96aaa445b61602e4268f6f67efa9557202aff1a49a19a0d68d439bcc099,b4934c1f8b2a38eae8f12349ab4b2ec57545fb250fedea1d9b62835ecef94161,b9b29c566970552755ea84f0c5d027e9d7189146d3192127418aa6fa254cd428,bdbef2bc8c0c8e61b68c33c228716bc3106700fcb935a9a3a7b426bb931c9330,c63dc610fc50b4bb299a5a2694682f65aa1e771c993c205bf3ffd71be86276c2,cae23056aaf5bc60bc97cd49e0b7068b9331d4bfa8bad832df0892d70e89271f,e85a04c5485a348295ac1f8614f3582db725745d348d2025f8b0d5f6141843a2,ee08e992a075abcebd7ffa955a7f1b56006ffc9b9e1c870ae1ccfbf08ec27449 +G-pr2169 41532670a29989bfb28f4a7dbc5fbd8a5e1a3c378661a981f0ba7338a3e4edfb 44672bd61e60b41f7ef25314b4e3ee8604523d33867d7bf012b0fd0a47834d23 +G-pr2184 ee39081c2974f1623c5b6a0613e237659e972f03c5cfd4df1f9d24774cf3f318 a86f3d58de571b13b8b26c7540980260095d4642775a3ff181c5d2985213c6d3 +G-pr2216 bc3064ff29427cb268689b862153cc6fffff6898f8f3f850b828e5625e010ce4 0d91f9d92aa253bcbdfad44af4261d5d3193a6f1da43670142c38c932ed2905c,14ea12983eb154032f67c065ec2c905f74e6fc10cf0212493a0506061603ba57,39c220be166db2808b8931eb3a386879aac762db3ca58a54bcfae5b60ec4c4c7,3a77710dce4ce6e909059443f2793efc30a8c0dc32aa6915cd28d3c6667090e0,53964e15a1611ed5cf217aae81fead71b9c310c598a404ef4b2f56530be47330,5eda6c9750f4f0dcb7c2234ac57a33bd01161f8bd64a34bc275796d575e87993,5ff71aba2de42425d67021c673ed96d28318773f71188e1cb469ab8d50bf1212,6ba9c526bb300fbbb638c8be6d3cadb14175638d482c3287e7a20f57de867c7b,7548ee97373d9eef2a003b44aff7e21d7ed8e47d5b87057d0cdb844f5cd95988,936306137519092c89d77fa0e0059e8a8ca2de9f96a976d3344ae35910e0b4a8,9af3defe335166a38c394c46d352730b48d13aa097dee3e30813d358287dc323,9ea831dfcb586560e547b382dbfdad1ef610dc8947949c68c175b97bbea1bc46,b30eb7a906ea97cc36b2962c773ae88f54bcc660e3f773d7600341335c722369,c35a3609af67a297f62d8abf353c6d276a4516e9fb889d135bfcb246a3b93ed5,da1ada31cc4fc2252cfe3cbe3009549cee9d1c45c12b8b37c4175acb98bbe1d7,dfb761b2cb5d419b78c5864b18ca1f20058ab6b4436542e1ec690758b3e3738e,edad2be7263742a0fa50e92e38c8a671a979d819577156e7b00a77af3016a66a +G-pr2229 53480f5468a2203ed2ab24d5e5a170e883a167d8f2d145e90f4c54ad045180b4 6d3b49375209e60896d4dde19b8113a6a0f84f54c7be29b4c603517954cd72e8,7ed5b88dbf4d8a42a4093eb2953bdd28ab6d1950e51d4820a882fcc37be28bb4,ce902a00310839ce7eaa3096e50e8fcbf7e77fa53d77edf422eae4c589c7cb73,d5639cc63ff192ffc87b444e739230aee36875d567c2e93ff1ec0da4353a7cfb +G-pr2230 8bbccaaf96791623b2611e5d9bdbae2e98dcffbc765c7ef57b88388403cbb30a 04c4abed9cf90850675c6df449350bba312bf6804167764cf13c4c48a4fbfda4,2553297270f8aacc6e077a890a3655c89d265d31ccebf26d900b57e361500658,419237e0eb93f0dc32296b0a381a03bc747d271715089c03d54903d62de2a253,564d5ad6ca3cfc6afd3635f449e51bdaa85a1980fc46086edee13b97325ce227 +G-pr2233 923635105eb240551f34c4cbea2770d05bc980a24c407f91858d8bc635965b53 23fa045929bb7ffb884cfb4c3748a0fc6e652bc7244c192c9d8862944c3331c0,3dd3419d2a26edd6e3def4426345e5dbc6a6634bd228eba4fece89ab45d5cfa5,4a10ccb566a19918c362da8edd85693eb283cca4b4dc368c411df7441ab841b0,5481e0a983bb59e0b85f0db5af53c80082d7ea0761550b22606af799b15d4691,647a4b96440be2d6a6d579c0258f0308b9d0a5cc5f6c8a3515a179349fbaeb25,689d8f66d7bc97d636af8bb9af3cd97854cecf0a774791b0b0f6bb21e5b8a784,6ab59923c5207c7a674075a43eba7bf788c2fc2eba0d1f478d482ed0b11b84f7,6ca45b9064ed0d24ff2266a6c37ebdeed51dff0033fc32910a975bf26e14da28,6ffbc5658aab5deaa9012a94430a781d058fc66987cae60ba5da67d63f31224f,705dc290569df4b0be8e9517a0daffd59dfe7d0bf940e7722956301052f0ec01,72a8469f2cd7995e30303918042f9efe18926c2aff3448090f472a10d44edef7,77302184fd66b67cf9e82c2304e941829a98f375c51b2dd6529d82da206497c2,7858881ebb7df99f4dc4040e164873773c7fbb524d92ccfa318eaef1f15db6f7,7d7acea317224c60b985f110780d85048384255cf4304e2f0a02a91ba9201eb8,80844c1b43ac200873c46b4824a66c06f2638516514869fa651cb897ff957780,884c86b5d6cd5679cc42f7b4937c0b4cda25c83abfc11182205313dc010a5f1a,8a7aa1e6024f7955c9cc149eab068e86dcc82bd72ffbdf4553e5c4fd8178b5ed,8b3d35a1a6a8533b70701d62de4d1f2c5814cd5c330a135f0bcd7a8c79185289,90fe3ee45e0461859adc1640b24f51c6be8394923e6446898ec6e21faab2a2bd,98e37cdd751ac13408ffc0e5104ecee08a0d228d0b72db5244f6680d61d3c019,9b39b768e1f5f604e8fabc9353ae8ee25b8ad2a32f5960bdf938bddaa5542d31,a54cec1c824163e40c13b42395d029c8d3db5154d348625fdc298266271cfcc4,c376be857c330dec708a3b56bec57c5536b769766a1a25ce7dad57b06d31109c,ce4bb1e72f752b1fc0ebccf61edec92e600a350fbfebbaaad73cbdf446c1f652,d101a9e884600df8ae012171789978ffe450ce9c1b5df2659634dce6edb13087,d4ec7bb0b9c558ab7913757702861992322c302e788eddbcffd890b0c6664c22,f2ef05ae0aa6fdbe82ea76b2aed983061d16e34570d2e1439a2c7d5d7b066010,f6d8d8ca94c695d88ae89a1f2b0194188a19dd0104dfed5f7055f3b0ed4e0f6f,f776267b64ae50a930a3ed8033c3fc84a4c7c7970b4e86da46f0b90712fda537,fb69e284b4eda696411e21739245185458641d56595945d4ac76b31084c6f828 +G-pr2266 c42ca8723991273d0b3d44d0be4b57e1c8484bedfb2e0df7b8add569cebe39e4 6c554086d1054b4b1e31d2bfb19ed7ef3d4eaa2564c7fef923079ebc20ffb526 +G-pr2337 750778c9ef2c8c44a41665133999173cb0ba0ed78831c0414bb19e03df3b4426 16471bd9c708ccd817546238d20f84648231ed21c70fad49fb63627293ab5bad,27f58fae988a1f5e13fecad7cd42ca8c4e0a9163802ba0ed9e73b3640389751d,317ac72dec4fc245cb23386117a1d54b116ac068debcac02027565cdcace41d4,841c5638aabd3302724fdf265ce5a924564d39690f78fd9de271b99b58d0035b,98b341554bb68c74de7c996fe95a21ba4bb2088f1b27b15861d64fe7db3d8ea6,99e6c5def900076d4413e03239afe3afb5ae534c6f71ce709f6e726d751af85c,bc7661cf82c61a6222d4fe45ca2530f17b2e6bc1c4249f1817f53783272ebffc,c8fe5bbd9044f94e3af684c4d4e15a85cc40b8229fe6c39b2d2b3ae890dee138,cf715e2a1312a338a3555daee8dc94cffd50d28842395d240e35a41546098cc1 +G-pr2710 16cc93cf4c3c0e1d724f49231b643265c836da1b70e14bad351f71be3e8b571d 08d9a53db61e6f73c278d68a4d09fffda66dc1810aade795e0d84c4b3354edc1,107fcc953c0db847b8552048b2e08f892e2d0a4be109b427264446f19c0e5042,225d2a64a69a1cc20bf01e2fbece40d67046be5369fc93c487dfea839c3e3172,2bc7dbc2f7ddfe1c0fdbe2d3014996f5c27c2c37516109cc6bc9e0dd64dff0a0,306a251828d805a78eb0e6d620bc4ad783fd12ae9940788c03a9339355b7a0c7,37123c9fa491df0d1d406330ebfafce49c7ef869c48f15ea9d3e3cdb116a0a94,4d4d04ad799c096a85b633e606c4184f66e3271ffd170d7c8b1b10aa48c92194,618b4383fd3783042fd5f6542f9eea1752e5cee315f86726064c79419d04e2a5,6987fc48df0816876b5fde69eecd1bac36afe9f642085b90833184b9845b5a25,7ca345dcf0100956da97659f485519e86f7d0ce4efb2403d757cd8897484cd2b,7ef6cf1605389ca456572579921d5ee6bcda5ee835514b8e4d5cc80bcb2dd3d0,8d948cb25557a0e330f7319178354a7e4ba30d24b83a5d1c43bb58b809d6a5f5,98f14a0fba907289ffd3a69b4289be705e4c753cffa6723ef927ccf99c0ed75a,ad49305140b6c5d435157072ce826c12729cfdd25f9e123e65dc3509905b2ac7,b3546f71103aa1bb90620a1341da0c221df0633b8e749358a71cb696579bdb66,b997f3f18f0a43efd502c2d03abacc728f3ddbfcba0f92994ba42216bbf58a9c,b99a51f58ed3125d9c9d98172378f78a577f34cfee27d98f7b46390bbf64e309,d828ae0a435a0c8b233cf8ad7d46360f28b7e3a751aad020b37bf6eab59d7827,de8ab7294d63316548d0af008572b1d74a707a732ad0ab60b2eb395104b17c26,f172752a620d3d24395b14c11f418c62ab23923ac4974e64c9df6f16d3a89fc7,ff1a59f866a57885e9e74ae4be9cd4f2f5946b3453e08427a8b1d7ed124824e9 +G-pr2746 b47aa64a2b06f68b521fb7d447608cbdd1bd4baab87e22142440f5c638f03ee3 0f6f64a22886e978edc28a579f3d8c27478a6d482484116d217b24e671f040af,1213072422c9b4a2ca52b651f9149468e3d92db36330337964adbbcd0bba6e03,140142ab2aa6331f6e40423a412eeb6740a41b5fa17a3b98300cdbd2e5fb0fb1,14d94e7570a6bfb464e406218ec14c3ea1ffb228cac277bf2c1c0a162192aacb,1515a82b5bb1d4ef1843c4e596dae73de2c6591e5add393111f7cf7d4d169fc0,1924499d9c88af6b1de8b47ddc031f81efd4de84a84c842c1eff281a846124f9,1f8583dd9dcdccdc2d0d6e4c11ad99ef524c1b628f5fbc2f35264b9233272a13,22b57a3ba808e92276238b1087bcc97bd163a9861bfbf18ce7bcd6bc82d5b7f6,2515bab9120a352235fc7f8a4aa440da63c1713eb1e10f4a82b94338e6ea6189,29333144e06b3bd1a772c6900b53e70cc53f6e912dd0d72c4989d412ee3ac32b,298947688d3b41478701fabc6f9140beefe00c8bb66fc190237114b2df3f3c78,2b203b6c6a145a0f12c7627d7064ecb2f1fd5fd1304ca391fc960eca83545d7f,2dfc796b6762b6712c5993742e8bb56353480d2e67d584cee5ad613d0e2321b8,3a04430e02f01a4a41c4aaa4841ffa10f2b2ea58e5b63b100b94d36928e2bb5d,3a3d3cc2636d4ee7bf875e0d6402c2d330872e3b3fd32abbdc1e56566a1750a8,444af8c8b30482ebf5122017a5e9948e483c5ed75d12fd05cffd65f302fd1d44,49986963239e22ea452be1a91df08816a5ed97e588b4970c68f80c4dda2955c4,49f2152122359b1a6694054075009f38089bd1df6a81dd1c5c7e92a9076454be,4dfd4dcb086cb60142506efac966b6c1f08efcc8525a5e8a14998974a421d165,536e1d77f033caf0b31260f9626879a33e760f9c7fa00650b6f51d28f6686453,569cc6a820cba64a8d15ace23069962daebbf5a1c159600bdad85dd6a53d920a,5b61b67c00b12d57eb00b2f47ea2c46cb82e367dfc82d280c43bb80f7d9f5c97,5df4843073989244392fee4e0891d34864bf89f0adc37bf2e93e41f24b1c40b8,60ce6f131dbb161d6393330f533447a2453cd3b470d0a94310001dad1e3bd050,74348887bdf429073022751988d484fd79660643198ae7bce882bee800f6e1d0,7ad4f24911509b5793bd931d1558564d0c192f5f226883c9f6acbe3d4afebe7f,81522a7abfff8dc3ef01f24210c0b530b97c15879e03144cfae34a3207a82672,9901026cea7b08f5d9a47d94e1b410f511e41dc66bdc6e72c45af98aab36bdd6,9984bc07bf27cc0670dad548668005ef14a6bebf0449bb946865d3a5b5a276a5,9b059a90097c24a73ecabbe6f1adcc32e36807e30011bf8a849076af98475311,9cbb05f446366f579f51afc1b0e76b5c38520a0177f700864439e822ab7a39a1,9d168fd5de43c25252642632c96ac862dcb6bad9c455d2bca45ddcff69a00119,9db892d0ef319dcba9a8f9e925e3890365c92b778f5770a3e32f029d59917fa4,a0758e499150a1c17ec338bb996ccdfc37d6fc68f1eae4de4da36bb876d656ec,aff0e5d3a259d34b6ac27fe25be8b360d81bad71dbdd7ee0f532ccf7fb0a81df,b1fd21af0191d8752863b2624125ecc9f50d5f8e2cc606547faa64e562199a12,b2464c71cbcb3b55c6ff61d099c72922ab2ecfaec45b06d4aa9e3712908d2a4f,b44d8682223cbf3938f6ba7397ddd7fe7e41ad7368265a35d18e6b49e3e3ac79,bc3df644c12bc7f7de95c14286be374230b92fc007de64cafd70706903d0f94c,d0f9a9c9ed4d5c6c8ca46e0219c0180bfb413001f23f4d8b7063ab23b6d1eac2,d63eb199aed58db0a92f0b86e55ea0023e15bdd2d7b732534d311e8ee244aada,d754a634a8c486051612c522c1f64852de830ccb7ac5b1ab83c10e3ac8700c46,dbc5a3b7236d3826142c35747a247a9811ebe274d44379fd49af17e841868f35,e01f3587b1a6b1483982ae5358c0229a799f37389f6cc3cc9ac56b01a359550a,e695390c356c5fdedbab2adc5292f6f1704cbcf9f1fd9aa2c047b6664717fcd7,e735d01f82bacec6784221950210e75be8ccde9b20a98d0a0e2cf30262887886,e91dd1374b7b4a09f7bd546955d27aaebaaf3058660acf605c60ebd3a880d2fe,fec0419a117733c935cee4ca3bca3d5c7ffe0e07efac5e02bbd6598aca9f03b4 +G-pr741 acb894cb62d2b837954f9ec2c20c38a4f8535a1b6ff95daf5c56109c3281eb53 04990dfef8936b79b94f1e70d6981e1d89bdbc0fa64b4b0bfaa23d5a1fe7185a,192b4d8becd82462cec9f5bad15891e16fb19b9857d0969383f407b45a2856f8,2d930d00bf6251d77d02faeabba08169755d8e9367b41378768e285cdb1b7ad0,47c5ec13439b6e16307bd297625803644510c63a5f7b681956d1fa035f86ffe9,8db79f1d1047e678ed1fae6fb24727468cb738a94c2090f8b1efe68775c2a80a,c5fd50fc034161d2119ac9af103773ecb6fd08f6b527b7ad7ed9d146ebe2140d,c6d177f0c72e6573f932f2c7243340d03e5b2694dd80f8be8973b3cb19c23ed6,d62d7e96c777274f67990c50b4076512525ad4f26887ff450ee786e511a39123 +G-pr753 79bba234bf32fe754fcab5ab41ae01ea4ef5d2ef4088aae490cefeae1155b193 0e798acfcc6a4df6f03e81eb5ed6baa87c6ee5014a77210ce722567ea2dfb96b,5fa4337982750f88f79a4fd9f395ce06acbfe80a50ee9d846de0ccbfb4ece23b,6fc9aece6d5a80bad78af22cf51eafe89f13c89d2d745bc1fcf4b9cc2e7a18c4,7f1d19cb837a7fee149139fb28600c51c74dcc5318937c96e0bf40ce73cf4958,8249a5c85c6f3c9047e5b70874fd8081412c86f36d7b41842e5ecb3dd14f71d9,a91e0b6855681c6f20cae1b975a0e43a687fbc1b81ce039a27a509052769c8c6,bda5939c0d23af89af6e201486a0d91af9a3b8152f779bb0a921b256f21e6eb2,dca6d76cfe0ce855315ef932ffbb0fc7a03a15975632dfbd1ea7e5cf4e9dffeb,e026a3140bbcff26332926946e1589c394655701644e9b46148e248a005eb5d3,e46daa766cbff2da04fa5450becc18a204d58cd6af18bad7d31d1b0253f4bc0d,ec65a84464c801c44149a6266f39ee43b9c56815975d2f1fd273d34b7bef62af,fb572ac84739f989802a45bf644753beb7ba7843a7b18051a2d4ca8eb2523afb +G-pr888 688c27431c3a9733f4936aefd7edb1eed93bbd3875f35e8704a8da0da1d808f1 0551abf79326f990e3ede9c59f8ab5411205333c9f885e153218049f11714468,0cde7a2ea6b7cc11a64f986cb3b9999597a42b5623310c9bb8555b93f5c4e085,1370c3d9fff2ab4d5ea5d27a6e5f813ffab3540cbae09601c31461fbe627c68c,218d7a91e43fb607c84261c03465c1c89567d6b836b399aa932fb1cbd5211894,24a95ef283ce376128051862bb4fd6a0759994cfe2bb733d50ef5d4eaed505d7,25d4dcac046e55dbc2a8ea87e25ccc29f306163f35e72504d59f7ff7cc0ede10,2c6eea0050bff3f4638719fb478851630b3b17348e48e95033e09df9a48ab7e4,343e9dff36e2065f49f0884d7f67d597d9ba5a226abaf27facae1fd66bcda316,3554682ece9cef1f48edde224639470d2160e37231eee5c8434d3d4739a03500,35764a26e7d455bb5b2c46b0138055e2a180ef911cedb032368cbd6b362a10b3,3a8477626773f7eda3c7f53b60447c2425700b56339e8f9bb016820118c00230,3ea182e545df6c0d7a48e17761a4ec918235f3064c7043946b0f60e23cfffa26,4b51822c23c7d901c8419c20778a74e53e5f7cfbce443c9b418d08bc27188e7e,4bef00be2d30008474b5b90f2cdd7c0cee2776d05087a1dfa4bf38e8545e3a93,530779f3997a0823a0bfc26858e094ebc1774246f231ec125f48c6725078c78f,5b9fe90d681d04438472af5e187a1df50584eec9980b1b57e7826c2df7d42d6e,5d56560d48343c7bc9103ac554f466847811bb9d2c57a7789a3d9b976f38c21c,5dd8ab74a752dd779eb0955021d9ae65421caf863bc2bdbedcc72b652a5a597a,621d016d4c39c282c5dc41f023b25e212a77c835a8944d2f7b032faadbe42f58,6763d485b97538a19e67f0104774882527560ca29b55043f0260e4d508895e17,6c4e0b783740af4e1b9944e89d41466ff59f582ad240b41ea743fef9cbf83ef4,6c7630ce3f9144d744d5131d859b1bac9b3914320cf3467515fe27da86fe7f19,717958c8302a8c44398be5b83376b00997fb8ee112483b475d6eabc34c0c4098,79ba088064aeff0b0c774a9ccdec5c45a8ea9b9fb9d23c1c68f6803150a34188,7e7a1210bb890647fd8eb2e16a26dc4cfb288e9cf4ac843da888ae24e3020463,824f3470a350c88c471537552d6f3a96faa874634af724de5c31c649b0846b43,8365430c94a4644d10b8d52cb06675b620e4fd39795f5b2eba405740dd113a5c,86c1109c68639603d631c580a4323fc8b87b47c5428e1c60958f4b4eff42270c,8a5eb5d23f69b7679ba96e18860e5c5e2d2f657df8e2b2c845956c14b5781c2c,931b34db513223ae7073b58a97d84feda6a542ca615f257656f6b4f1af977e66,936219fadf1e19ee03f43471d1364359e4b592e882c40c131469f97f2c3e1ec6,9389cf66ec34e648967038ec49933f0254557979ada25d96135478534179754b,9861d378fafeb6ae16054d3d349a4d37fb5bdefcd157191547bd4c2f5dd2e43a,a9338efa90ecde459a3944acf7049ed03b572263b9966c0c8c302f4a579a30ca,aae27673430624e384af708c63f57f3b933e03716f6b70fa7adf194d6028aa83,b0f220235fbc2c332237fd92dbc720fe8f37fde4e71a5746361929ce9c5cf7d9,b902ae0a509b4fd12cb8ae62246c251984aee30e4c967144a895738ad7dcb7e9,c67de35f9cf5877b36cf01536a2b11b1b840950e56525c237c70393371d7ceb7,cbf5354fb4cb49ef6d10d0966544eecac09b8872bfa9e508076c4e219d4f6941,ccfdf78ff2a1057129acf0675cd71f43a7af108c4bd9e3a7f27252c3755fcf7f,dbaa615052afa5a4db91ccb8db6562b101987ee833f2048d66dcc9855dc58d9f,e0af756fb78cebfa6aba858f1b5ad673b3bf9a408031293c63504473c910ab48,e8b91e15b4aa5a70fe05af8b07e311c937495348e3d9b4badc5763d7a813d82b,e8eadc1669497d2599014cbda493de1ba421211cc8cee5d8453bcb20600397e7,f0c0da298b16f32f226d3d9b6e5227fba0708357223e997908e81a54c401b76c,f31fb9e315d88c97b53de0b3ef692db1cc67461c122934971f140e75e12fdf5b,fa7ec6cd6ff41466be1de099b62988d410dffa4333857621483b35b93ef15c09,fc4f510381a61a83d1a164a47bd5117695d7d0f2f3ee51b08df79e020c7ad3f9,ff5349cda73c7670bdfaaa0b29606e75050689083dffbe9cea903159f0dca7a8 +G-pr940 a3e65146b9110135d9a93050791429a09dd5fde35cd527e6f0848e0bd3d55a48 08c4a87aecd802615a2b819128401eb1057a42796f8c495fcb75001278ce6b30,331edb07d70536f3108d1d599d4ef85e42a5263e066af80811012842baea6e7c,74072627abd4a5c01935ed5edaa105c077195c9362828f32ed21a4210c10c915,76c6f7e067dd367f128563a5755801c55d7f35984acf1b8fd87b2766008d5baf,781409567f1633dbb5bc2f9d5cbd2b9e96efc2b59f2447fec200cc17c6b4b742,890ccd2678415b8416e11a79268727b8857f14fc89ed1e5a2f90ed2136088fdb,97b2e65e24f8709099357193ed637e665fdf0d6c3eb431e6133e9f5a2e2a3ef2,9f5e0fad911e8fc9a36631db63e19009336e1cf900352e598faceb998decd014,b9ccf58e5e683d398975d004a4a7fa1be46a17cc89459e558d81ab0de6c1ebbe,ba2f751ec7beebfdb9c3420235c75ccd568f1c2ad06537ebc7609282a4aeb2dc,c68b9f4ed527b3b40e14e85f2dc079f3bef896345aa278c975acf88309dfc2de,e1dd3019014d6db66b554681579c9748274d993da73555093c1807f8579e5ab7 +G-pr941 5810531966d5c87102685bfb0afd7525795c1867c9dc8029995d155dbcd247cd 09d7146479ce3e418bc0663cdcdd67a1db99e56b69daa5caa733ebe7858f2e8b,15b4c8d339683981d8772b314c32b127f0a13486a64fa81b15dea3de9ad9f017,46cecbe320554cce689b0bd94440d381f074cf51f4b08fdceed30a5d25ed91ff,7430bf8dd33756de663be32e68667258842a6820a23b7dd24e8be7e7ffe82800,7a7793002cf85f4eda38c6bda1a912ec686d95aa2dae0a702af8d4d3471e00e6,7ba154af1e9c94dd2894fca339bfabd8f90458fbe1eaf71959a608f7ff74540c,8dc01bc228004c699b0c51cc0e6f8d117cd713da3c1895ff5570998b69cbf0f4,91acd885dc802e649990f17bd66fbe48bd834037e6d82fd2de48a4147f1cbf3b,93ddb26305507c05f7c7dd7f92ed8336138fc54e4de6d67fbb8704927be6880e,d18324f1aa68ef1a066cc0e5670639fee2cee4fee886dfd27c162702e17be567,dbd5284ae2bf0e5ad55851f9a52853ab47385abedb0ca8c4668a5f2e0ef7f18f,f1022e726234f30ba674f6754a56f40779196b24b6375125eac5d35b43facc10,f979ae11d23d4925219862c28f8c5a625235c4e666287d4161d832f251cbc190 +G-pr953 7172f520ed7187e1ef725d7c17a0ecbf59e9a9c03622bffd5f20ea4b67545d84 3c41ab67885434cc38456dc176005c77c18c70d945ea21ec2f239dee477bd06e,d16c289795d6cb964f04a4717a4f490854cd01bf4cf2fa0bcb1920b28a292cfc +P-033caf9c30ca 83858a70fc8c0a0b603dcde3fdf466d2dfc08007a5b79debb7f505177265aecd e175c6666e1fd2e1d3d7e8ca0250fd28ee0577fd60e59c88579e801d4d85b2b0 +P-06a15fbc27d1 fcef599862241eb390e8fe36b0e3f679964945f59155eaaf385b24d42d718a48 fb93823b25b2125c194020832f83cf7899e19a17bcf090a7416b6c13e4e9693d +P-0afb0412cc4c 8984842123c0eb7fa001f563e26261f23c7cdcf87eef1f66f3ac01f2b7a3631a 9ab3f7bfeaf2865acfb9d9fffe2c12cd38918121d02992ac621dec01abee538b +P-0cfe3b858c03 ce09d1364459e580b6ef521f1ca5e7ba651e06ae7e27ec02b6f87dd94d402580 4186a3eabf3e6aaff88ebb0d90ba83e02df978839e9f9f0a6fb7a9560533ed37 +P-101aa47005a0 463d1922155417d9d37b1a2cf50d861df5b388207f94b8096d271369d181e8ff 607880325c9b39fd05355b5ca0a453f302e1be7b48e67cafe312a4bd36946594 +P-1486e2af4c6e e24f2020c4e1bdb913c863774ef3bfb0c4584a77e45cf1692d2597d2b6b947b0 036e87107cbb3abcfdc6d4cb60e0ca7e9726eab1b435ad34fc7db164fc347b9e +P-16a3b1d9bab2 77d23ece37e4eb1bbd486b7314a1be845e7feac3a3b104b27100fb33ae6d8e90 2004a3107033397b3cdd2f7839d554daf6ac7c272e1bdf854558b9023ed963ec +P-19fde4fa1f1f 284183be921ac0288241353520b358288e76a568016621dbe71ef1d911921567 d1f2f3a73bf7db2dc6c2a09b34f318a21e9f17507de33b09b0ee8c28c7d47a39 +P-1e8aa7eff08c f32bd6add6876b9ad0a7af0ce38b1d5fc86b0f3852d9fb0c05194a3cc8bf44bb 988a6405fcb64f6679297c146ea598a37555c41ca668d0d9ba25f894ffd0238d +P-2f194b384bba 4b29e387d50c0bdd8547b35df3e0c6c31b7792a9a27031e146eaee6d68cf71a5 97e551efb40dca364468e4a6c93af8ffe8463cdb56a35da93f5979e30e24b6b2 +P-32457b5e3477 88ed951903fcb3f92837a43c8fd304ac5dc6ea7133a219c7d4fa33d4e575aeee 518b8b9b675075dbe0197524e99c6505e78d813dba8357db127a00945b35ae11 +P-344775d57610 c6f90a22e788c9af6e6ba222717f809ac6287793477888fc41ed342f389e40d0 61df2d7d0b3877d90dae034a609cc5ff05cc6a0394396dc8179cfceb90a4aa78 +P-3b35f5a6934a 0935fefc74ee00843f623371b9b3f51087616f566c0c53581da5712bff5dc29f b818a5eebecd91df735854ed8372d08a2a72dd7168d0e14ae5209eb152741c59 +P-46e545d0621f 9ff8d7e9045c8668ab4a3ed75e5127d05a3b9cf617f151fb165043ed6704a0f1 442f98a4835e73f41cea7b990b6b7fbfbd294f9a747a389c206dc2e2a8f092db +P-4cd71741f944 e524bbfade51e4abe56825c03a610a50429ab306d495a0f24970999c67a2a185 6b09b07bbf391acaa9a3254366b2d7209bc39af1d1086fa9ec8fa9ec1d4eaff3 +P-50edd5a3123c 7edb9adc685a5a261f3e1c4395f6a8f937f84b13e8b4950b275c64e62b0692e0 bf442ffd598137afd325f8ec630cdbf797a5c718a92bd73ca424bf22f2569eb5 +P-553007c9399d 82c536482e6de6fbdd1e0a56a0531fab17588b7548331b5e2274b19c90d01bf9 7ec5bedb01aca28360fd82e2b91ebb6f49c995c1c85a1940d8fd13a29d3c3e6b +P-5a4122fffeae 48a6fa82013b64ad0eb92a45cee80aaa8581a3c989cc4cee33a1384009fb0cc2 9be68c913946cca8db35838028d2f1e111af2aac405896e6749fcdd7c3340605 +P-62b51b8e169b 0de6fa7f2fe125f79fda2d85b9e5ca6a8c628d7becb73a778275624aa279fdc1 c6f6163a8b38853e3b8c9259f89af1e20ac784686811fe6f40f600c6b3e839d3 +P-69fb0db38747 3150a112054ddb2389c40c00ec980a9be406a9266f8d11db69658679d2051642 e0f7d5bc0bad4723ca8c93ff37afc5e02d341f6103d0bed59bea21edb26df334 +P-6ec735d37bbd 1cf0c0b17710680c383d17a531328ab1d32a6cf24632a507c69d6037bc21b1a7 3f0cf482d44cfe0beebe8c9c68aee7d93801a85d83291caaf21e264d444bdfe9 +P-7a1b8a7554c4 8ab6d120527caa33beb77cdda306ec2b6cc9d1a68d70f58b86dbaf8934c9ca6c ffd7c85ffe800c79fa02ce89f7008001522f85bf88da556bb756ae9346c1ce8a +P-7c3e4fac1fd7 eb9f8c994e3e62c7ea3405fef08cd0e06143e0f6cc0106f80951daf1f4cabd62 b265e848b4450dbcd471a9f6f95966df7358b2a65c3f6f9736afa303f5629e78 +P-7fffab1fddb8 6341ad9d17c9fbf3152ee4d0d579898857ee3b478354c4caa8345ad59623e1ff 28293229615e8a18797c1caad304410587aa0827e88fb500e227913c744c35f0 +P-8453b248d79b 120f798def12e412b4e7981f7b63487a046f13f083c2fe85bfa004df46c58e9c 9afaeb752aa10e0f25643c494f5dadb58fffa023a5568afa5d60d31211e29591 +P-9e598f519091 11d89533d6556fae22cf6e612937588389d83886ce2b2025d9faae72d19e050c e027ada304428ffbca83d9e24ff4f130db9777332ca46313c9cb2e8163a5ec3e +P-ab41daaacc1b 07845a5850d0ed44224cccb076905bd2f789ee30fbde06d17d3c2632aa081306 3d03f7c2ee6dc5294cb948fd2365e8cdb58d63760743e2762c3cde77d30ed76f +P-b18dc9a4b9d9 04deb279b9e2be63caec0c27b3bef546bb8e28787ae0d23ab9faf779a4fbff5d 6e993c5bb752decc3b09894f29c7ec230058c5a161d799be98bc630f4715d899 +P-c3a0a45804a7 846365c3aa4228036e2c9427b18e67bacfa4c8ce59e2684ee7a6baf3af9c7947 e47cff2d5c9e6e075d3253267854326c98b8f86036ca39a054ba17fc09f9bb4e +P-cc6f48e56473 c4d56bfc1ce39b4587159f37473dc0cbbdf6432e3e1e1ba6ab5878f01bf039f0 8813eef9b523208d38f817e654b5c9b899c598ce6e8ae6b6c5e8e58829e10ff9 +P-d624d8619161 228655f98951077c9a600509a64ad444117461802443f79d223e595bdf8d6164 63576e238bf70687235b8c92a8b6bc03b1a3f5c34d9ec4d9d6c742036c19001d +P-da4f5964dfb1 a1c8a0b962078c2d15f14ee08cca992f6b463b5812d8d25cbcccf104c5619714 11d22c86dc4c8399e81c22a3c2a81b5921f859f5ca801b83549cf015ffda286a +P-edb10eb7e5ed 315abb1e168f2b48e75a3c932a388772d2a8d5478c1fcf5132bd2b4dea0b3772 cfff4d937d7cbc152960a3ae5b38be0239d877d4c4753bb1681c2c68fbd3e5ed +P-f3abd4438999 f91ef43c36cb1a1275fe24210314a6d1d365b8942fe1169af9e36fe1e1224ad8 9e21c016f2b7f2c38fe6180088ff5582efa28dd823d53d60231434f26287a27f +P-f72984d91c0d 74babb9ba8a27a89df0a0092691c08e076f50ab0792f273365eca699d5766825 7209fe1a83332bcf61b38d214016dae502217db361406408787a1cb0e18fe86a +R-pr1411 7e10fa511b0956191e864560ef66a623fd0fa78dd03c06c39079a69d4c61959a 5301e3dd66163aefebb08be4b7c9776b931b1d0fadd70e8186fd474ebd588a5c,76d9b36dfff8359d7e3894de18e6f2618f6133153a6929a57763a0607be8db9b,c7fa4a33a798d0cdce763178d0a056765e839cfd8af001765be0ebe121cbf4cd,cdf552438509aec3c2b6b6b15fce86ac3566fa90e1ee1c33099363afe94a9229,cf64d36335d755ac40d0764e513a52d56c94ef1627e5abd66889aae562e4481d,fc9929ff4de64ab8dfbbe70b8b5c81eeb678384913878c891405694d41208760 +R-pr1714 b5feb051c449a939190491b37058a534d03f3522b8484cd1bc2eff342e7e7fc1 34507c3ed34e9232cf29e7e310dde02a9f6d0802b4111f210772b818a8b2d27c,f377e11049959fc9c35abad20faf62422993c947abcb522444cf02c2c8aa440d +R-pr1782 5b778f76ef9b15da17d442e3ac2f190ec8409acaf86d68f4a5554669fe04da8f 1e5d2933ed568c6d4db62ba6e47b454b6250185a13d1e628cdc0cc14bfbcde76,2ddce983cf135b86ad27c09f5bc50d756b10275f984713f651feaa9797ce7faf,41fa6165b0866a404ec01961210fe1871a6450cfb3c9cb55857f9005a202d89b,902d837910795e61fa93d206d7d79cf4d90b4020532506944665c8441fd05282,c77b3e9334a72adfdbb136e0b15ea652a67951f38012e092dd860b866ca109d9 +R-pr1806 115e56567f24ae649f56e126a7cadcf0b9ca5b33e976bac2f414f1472a1735c4 13d98a8074501e6e1560a172b32de053a435eb9b71e6879f9ac9df71cc9d8992,3e6ddb0879db70714000113bd452b090458560ae14d5f6ca2a613a55f95aaf98,6680472eaf4bdf28aee317c3fd6d232b19d86ee4e3160f3ad80ad03602b08720,83aff99948d129e3308c0c1123087a0a23ed2b102092b271e2198ad4d7a3c6af,a5a833d493152c1df6ba254142ce931a16958cd2ea77584938f398b7eb11727c,baee6cf25260146342113f91cc64032031b962365854159f275ad564d5c578e5,c92658ff49c169ac411150feacddf44c32629695ba5821abd233b358c21726b7,d7ca61e65d6946e39925fa538a468b9e257283aa62f7957ee3e8249dd99e1790,fa615a8354a19ec46661377a4a5251449dfcce18604e3707f90b907a3c4ba4df,fda0b372d1a7a533c5891f0b3908afe7ef80034ee4cf9c17e5f013c9b4dd7029 +R-pr1808 4804683d7ff88ca019d882458ebdc858b8adb9795e34fe225d5de08122529f1d 08a962e7729272c0754045671f83e51f7af8245034a98c9e845259c52759eaac,17530d5f214cf8317c3301e715e5fe61be5762ab5504f01d44941f120a297708,c329c281464ab35f81462acef1ce7f4d64936bd8170fdd22ccb6dbbab741a375 +R-pr1867 49078e8fef4d0f7615a627297e1b0d5ae898a249974021ce25b8c550778ef589 623d730d0de7f66b73d2a9f9d2172f6f6a72245f8f1f406c0e2fa678fe89e53a,a8c86a0bc9803ae8ee304a5db0e9bd056762db958777542d370ec34a23f3e8bf +R-pr1870 1e1aefec535ed6a62176ae9f15d538a764a4e1983cb4c3a7c0a768d225721f50 6143d4c274c45cb92d186c6934ebd8e9004605416a6d0a35b29e1601785e9db7,6bf6caf52cff83e1add3436c86b05d2fb74a0283a5d56404faec791d41ffdc17,7c9b449b140064c58dcdc9f67bbae055e1b605180040426b9c5990be5e8cf3f6 +R-pr2092 157b1b19f40d99af3fc7d80beb5e99998aa9697795dc08b7d91145ce5179b0f8 0e78fb5e5399ce28c168f116eef22545ebd6e10a66368720f21a8e5a6d353324,3bb1ec9eb9b6a07209615da0bb4cf5e31ff75f58d3627b5448e06815ee810595,5be806530f0f4732af65c1d90f74485596d840d22cfe0afeaf8890c727d90fe1,7acd8ec3cbb8b240852052b40a067ce773cf05cf0f29a877a920d4b5af178a8a +R-pr2321 13812c7d8dd7e8352d0f24081fea89a771d1387febb09604dbcf43afcc82ea36 15a8709adc453bdf106eb8bbc231ae93758c6d322dc539c6a7e2111adcae6cae +R-pr2365 a30a4b965b369d1be55d808c80db3e1714a42dfa0e47226d9afa2dd88f77cc10 9167f2edb98bbe6804c8f2f156630c2d48642cd9dcf47e89adf15b3f2b1c1e70 +R-pr2368 0405b05a7de60d1f738c7991fdcafbf4f22f965ac3d006133fcfc0cdccc51d58 2f82651af168b9309292a01aa659ba1be879a63f8b9b3a29a92731ed98e6a955,565b044ed3a1c435266ac7412f114defe2f7a7a0af1935d775b6527621a77b8d,74a53657018f774a056f3ff9aadb91b7f4cfb1b41eff44a7a3ca458df503c152,9bd5c22ae61f0a4d7eab0b9a33d06b92bc135ed9042f4b6d0bd9891e9688161d,ab001d76075500766005bc65c441300df9b3002fdb3a635a0553e5b33c38751c,d9a57a7862f3ff37b5cfd1441c2fcfb06b40181e523b5c6f7dd9b89525b7bdfc +R-pr2415 a309341f6d6514af7f480f30574f7e86ffbc51f1e0b9d61264c4f77beee09811 94eeb9dc2b582f56cce3b95472a432e9d304bac8f4286339b4c125b080f5f099,9f5316b9cd5a2703467e61f6bdd1233571b2e6b6819e0a753174b9a5649f8bef,d9d1ba60213b8648232faa8f146515fbfbc65ac49156b6fb3e7ab1b932c88323,f35bf33499e498b3800bc2399d81e42e086267546c3f8394ed065a550a8b672c +R-pr2416 bbcf4ef31aa3cced18e102156f8f861b5052ee15902adcc4640324847adaef59 0354e566b916b5f1e52d532089ac8c6c42b939e45eb5994758d90182bd1a6fe1,0feb02fee5a699ae3315162cf33501fc819b35dec2cfacfcd7eafef3319b2e16,2860172426c8aab6c75b2acba9e110c14e5bee7376c426db205509c3bc9f24c4,2861819c808505993772526bc51745c221c5c94ec145af8012b6747182fe4ffa,51c102c483bc60d2e6fd92a50b485b23844a91bdb2cc8ee882d9051614835a50,52d9f9bcab863eb38923c7ecc6e651ca5388960ca9c42d2db7f989c6d01d7c3c,536350af806c11b1cbd94ede80e7ed294b22d08429f027bcfa32cbf2812db577,566be6d1772908f4a749bf86445f1223c2aad3af30807924df8a5371b16cbe0d,5820b79aefe7c1c87c9d44a0b37237ff981ed51145fe0c068bb6b526ce6a53da,6a54e488ef26368a6e1c893b5183af8f9a1a79d7b0cf58fd6ad30401ff9d091b,6b9148e56e7cdbbf56f9c9cbafaa194eed6bc359d8035ca0d2f1a934bbcbf444,6ecc0c1b7bad12014ecfb8351d8172d87fa50b835696618be8cc4e8ffde5c3f1,7dd2d0e1412a1df5425be660f4fc6e1f91956a7e92f9f62cd1e7d3684c23b61f,7fc8e0cc028268c50faec8e0454eca715310e8001c649f9540d546e1c4522d2c,94c8f79b6027e6464526aa0e95bc7b99f1e607d46ddf7a83ae051416dde4d5f0,9d65f2ebc1acde44d45c470a57b6bf833057c5452ac75cde9fd4c635872f1abd,9fda51a744f02e6ad8ba36aff5126cfe88777f80f513630db889e82c2dc34eba,cd73e8b78ae8a1b669920924245abfc1e110035390f403b45362be36f2ea41b3,d8c83caac55081a3722c71544a1df0501cf3d720a1e6a8951c3dcd0b3178779e,ed91c9bc387556808c1a9110cbf900a1e007baea51ba0b864e32501ac45d763e +R-pr2420 ebec8a3614796e4d5bc48e28b225910f40ee717e1a80bdfd11ccf87c5d01bb2d 02e4c9e5deb63e9f79134307ebc2747a9b8a73cfad99d81d73d980d63e10853e,1f9c7d41d687b84b6def56bb5d27ee4cb7bacb431186b588889c5700eca75b20,242ada2cec4ba656b0f1579f94afbbb1e47598e47d5e20de9f5ea22984e3e8bd,2447c1716326eeb9bf3c81f8d15ce690a8964e3df22139a3f22a6da86a774472,3339bfc3463c8f46cfb434d149313992015e7cd4efb85fbfb94995e127b77934,403718e99e94f09484005c355a40d1d0dde9e316e76e2a2f249f1d6b0e054a92,471939bc3435ddc0a126199802d1447b44787082c86d373a7bd2f1d189d8e07d,4b2cd7c22d4f2d6916a6c707be2ee34af9719e06a8bdc2986c1921a3fba2d3d9,53743f68f668cc528548c640cd8059aaee6e269cbe9f118612a36cc53704f2a7,558c48360136f318c45dd22aec3494937f914471535e087f7ac15a57afbe0ad9,64abeaa75fa974e7adacfa99bdfbd4e8d36cbb63b40c3bb98e24441b99b59147,656f0469383f0b8cee24cdd272643f9de51c4d4a4a596f0e3819406a7328108f,67c6e0b508eaad3f2675862f533dc580a353dc7190ac4cf5be9e06f3695a2093,67cb838d45b42b0f21e7b9d9f4d2569c521b80a6a490bd1646c7509ab9a9110c,6a82e42a78d75c12e877412ee7e9617f0d2f142fc029848b9a85db6404a86463,705fcc3187d3525b35ef6d21228674949652b14feee54428f527f030e36587cb,722d2930059849b69b7c20b7f308ee5feaed8612f671fb6227bcc26ea3849328,784cdb01a7af831fb92da1f0a2fe78cdabdd2eac651ab8ecad5348cc84fd20ce,83c476851c85ee3aa4e19785ccc5b1a3339c0e362dc90dfd5809d69470cf0c70,84ed8abc0b3214366b4c6e25a03fdec6788856256276ad576473d8e5d4d5d677,8d2c5311cf8e8e722a1e628107e80c61a3e08cd9e50e4e3f2fd6dab87fad429d,8dd1531ff2f99b4fc73ca0f19c853618023fe5232e8c0593f8b60f077a91b7a0,8e22fd9d89d9aff418edf88be7a74d10ca2f0bc01832d854a609a1fdb7dc5c9d,9075e10892ab2c827afafa4d07feb035ebf57d69da73713c0f42ca13d9481414,93b6d8279727b8eb48cedaf582fcb017b29e4393405d50617c7ef9d9c924866b,9b11abcbd67542b8964c794343a32e03e98cdaf21684320c2dbdd951e4c208fc,a18db2f7d4c6076fd5f972289b2f75363f9af7c4d0c8951c5e88ff0ebd265ba7,b4716b873a8334eddd6dbec06db0f84a930543ced4fa278ef6b84a01e1ecfacc,bb8204defa68ba2ab4f48ade17f62c06889949e7fd15b89102aae5c90f39f4f2,c7fe17fa71fd730353098db9d8c1cde4f290ea90c69e0c2cc02ff52b77e554c0,d3b38a47a476374ab97206a3ac8f49964d528b3ae8ef99e5531ad68c462eb9d4,d74562a35aa85708150d361aacec3af8b410601f859e6103562bdddcb2a46360,d812955e722c6163cbc453780ceaebd23561c082d80a65a6e408bac6cd7d88b8,d92a36452be13906f9bb703110f95e45ccd6be2677a2708f74f482e33635ede0,e18b24e8afe0e0dbad95dfed4d919726c948198196a6cbdec9b995da30c4757e,e62a86a497223688e7feb638daa343d08be6d543882faf710f76e819b5e20cf4,f819ae203b92985a923991c0fd2c061b8a89a727eca834608d539b53f02ceaf6 +R-pr2423 02e1a9c17700e11751b85177f1efd90e2d23c49b62d65f2b0b538931db534958 013abf627c1dcb2c02dc9e85c28ee80f9daf83ed9309d1fb528543b80253ffef,08f51f2ca0fb872b3a99847c413481ecf24e86256b6b80c6823cd10c5f129cd5,220620537893dbd90c9c06013a09dcb5bb4f95f392f3c20129981989508d79bf,2d7b2f23f26d961e079f749c8c3258355c69249ebd21cde95ea17fe47391a7e6,465750ecb983e3279b41d6a08488d941a9bbf2ef2018668cbdc29fbe2e97c0b8,4c83af7d68b4665ca21e90a3ca16924ad6838abd7772b393a8e24a035ddd9c5f,53ef98eea101dca8203ddd6e3fd0f0220d9dab311548a1aa0bdb8e43841a30c8,55841bd776ca6a9e441561d1180796d808709400517eb35eba12fb0b57bdd906,588622336b817333721e652754f9a9e376e192a2bff29d119686913fb748491c,6863310ded537c599a906fb6e4abcd8e08554cbf9542f5385a1086bb24ea6bc1,72314d519ca5ac5a59c475b2783adf258e11dc83b042568e4e294e8b4e068401,7f04dee2bae4782769d094eb8bb0926186b67dcfe0782dfd5c09ab86835fde56,83e889c6fd5ec70f6cfd48c447e314edf0bd06f7ffdbf9633114fa04a06acccb,9ddd7635511f291b7272f2d8cf653691ca83dd37e100228f7f6bfb2587c4a97a,a1c879042848cd27db90ec694c66043c83181ce117d03b7c13a54372966a7912,abb0f2c2c02ac5d70f7f612a95f3d8059fbc8ecadba210c07b8073dcb687a2a6,acfd06390244503652320be53056123fafb90b78340e63df588bdeb3ccbb01f3,b09314a7dd89076cf5b99f2437a7a279a34010d7ce2d1cfc0121f896c7320d0c,b094251868ec4b30ff625345b4d8a4ed8c30c20f403c9647770f8aa5bbd19658,b17f4f630988826d6e3a291ea328c4606fe3ae59a50dae58107a3ff6fa60c30c,c3fc5d01d9b7d747f0ee095adc0c91921f554fd0645217991e7c265b8bea225a,c742962abc6d84ee6e6f81b95f191bfe8cf33b9f3feb9556aa249383de245898,cfdca0158cfce53405ea65207933bb8496e54ca00312b355ebf1d17c3972b1dd,d01978c7a981f90c59680db5a813b0fc36bf9ac8ae1da6bbf93301a49b3ac099,d5ef4d8fd9eb2eca8537f985747b725738108962bc12ba63d8a9d15d9682bf15,dde4e77c5f87d39b7c7bb641f29c31bf0befefef86fca556c57ed7f137b8b4e0,df979d7233f7e94d8438489a1ed08317233613712163e584b64a8eb4c2cdb020,e6c37d761b2328ccc36732f3d193f3853905ca78ca3035090ac66bfa7db1f354,fb77720a17c1967340411c678adc5c6b410440b0df812fa68b376c136bc712c1 +R-pr2424 31f870a337b8604745500add7627a81463a19482860848cd6df6f56127ec2367 11ff51586fb320e56e440bde3280f5aabe813148448c4bc117d19b9f9523f4a8,1bd9a6e4cd34c16b83ad5823bf94f82d98c5aa128624c0aacc9c750dc6d9eda3,283f40aceb06443c2014c1afc9cc620e544dd004625f7d720bd1360e73591a94,3a39916b20567f322304c401a760fc6fb41c47d02886dfb33f3648ac64805772,42659b966e39ab97735dd3e82c100f48e6c16aa524521670f200a03ffada8f02,4689058eafd5d10bf7d8d3651538fcba14f9f154002bc0db60f1bc3954275a6b,555cca97f7668364dd89521c798644fb50fb730c0febb02ec898398f458e5d7b,afa650201cbf58c44b8973e5b5dbbf7fe33e6f2e389adac0623dde99d87d3bea,bab9a083386428e1d47926b3e31b230a8d13cb8f4e57972ffecc71b2f6fe1b5b,c0700dd24a606a95efd229071180fd85b86ac6950e19398584c69905d114c09c,cec85232205dfddda4a6e6dacf35ca76eb3053919393dffff38aa73b4f18ae8a,d177455ac2563500f1b172264ad4527e4d4322814d84188f57d097f99002e1d7,e4a8173c31a39c0baf0d079ea5e168cef919986d94b81c94cebc569eea659273,eccf1b222cdb5253d37805dc9d092ac59d947b3ebf10fab6b55002a4973e23ce,fc90fc1dce77cdae04fc84e6588f0ed649f70318c79c63a7da35e414a4fad2b0 +R-pr2617 66daca2dab9222f96edd5b06c2bd9c080bde3bb81e11b4dbf21aa82d018713a5 005693e7207ea0236c32089d16f3d8569fd693e98691e0fff5673e2479eb6519,00948be41d40ac5be963f1b9838c5791625b719e6caca431d258eec8b699a032,01e102ea44813dad38686986e09b4241a3628de9e267fe88968220f421bdfaf4,37b822b08af7a6fd17b45ccc07c9e5275ff91de598f067c080da69972f6ac49d,3a51ab10aa58e2768f3f11dfae08948fdcf07fb642028578563387630bdf5210,6d9fba63a105b2372f1498e652081b55cb533dfa35e1efc8583480dc3f832c89,75ee3dc3ade9a8fdf1e6065c9f8ee6f65ed898c8677f685d2542a22f1b1c61ec,a59fac20c99b94b0865da35cc5d85dda44ad4ce4c0868686f58b21cd3832ddf1,aa21135b188293f17790f7ad4abcdf962156bd73243eb7af2c1165c6723f566a,c1fd00c7bdffce91394b18f65bd5eacade3c787ca4f8826a06e169d1950c237f,d1d96527679b95f0f4746c11e9221c731ad8dc745b2a5bc24eee3340f8a370ff,e232a3fdce95b00c9adaffa512fa8b1b8fa1800a164421374f322f9a33f1e137,eaf01ba120a040e6ef2e59c0072805f81e4345daacc7f1c96d8de9005479574c +R-pr2707 58b4e337288c066ea76f8707cdf92a61dabda178420be0ef719ea8b56ce52cd4 1e7ab00ffeef37aaff5a6ad6c81d0cf05bfa80c1b687b57ddcc02abd24e4b4e3,292036811cd71959bff07b854d5319ea046c704bf6c2256b7dc139711b5584e4,414bf64dd14b04a3e5bce79e5e2e02da15e155a91bc3c603ff4ad7a0dc153ba9,62db7db04a3b42ba026d96331d7c7373e6a38d7d3268b31f849e286ed6388fa9,9454e382adf2ba5dba3dd71847a15110b9104530452cc666b42cc4159ac540d1,cc03c7e9ab3dffe1dddde78b7c20636203deee0e35e54138dce3914faba8bc9d,cd69f21dd640d8c26be4daad7892b30080088327a4d5cb828448c382c2b2b0f0,ef2d9c68529d35b73f11bfa88b295dbe1419fcaeac964a395965c92efe07fc90,ef38317303dfe3a5cb56e04e392f7765cb98ddbf4f04b742037e508b32f062e9,f751a9d1e06cddaf8b13dee285e142123a94c6ab04127d11585c314e254baabd,fb22e251d1ab7945a84d7f531d08a9748c7fe0f2ac395ce139e84b427ab00ee3 +R-pr2809 3dca02a5446ceb7d237d47bd31915d6b0682dfd3fe306da7eda39f4c2a6ece60 0971d8e362dc726d1d47ed5b793aa338aee21e7c0039e172f9ee416914d4bb45,45813068a29537fadd414f0cb4aae7b2fe784e3315e1a4f3c8fcb2fd8993fa64,509435c4d80425c76d92197da3f328997aaef33d97063b57b06f944f6f4fac51,58fbebf04d3e7def603dfc2b6d91abf69441bf8607c6c75adef9f823b2e90380,660ea960eaae43c58048eaa6950638f61966ecb9dbfb9588a10db84b57a46507,68d02bfb1160602d85014c24a4d38f420df69bb13644ec62698c5024f78e7daf,772ac3d8b6f56f9767246b70516a684ac10a57b8b32e1162ff49a60208f81d71,90a8a46009fc07f28b288ee7304a08f97149118d99b80830db298ffffb528ca1 +R-pr3008 22290559256405944315157b0e6d611a4415f6052c21d18a8c5387116305a1c1 161f93764d2ac7b0d378b1fe1cf37878964ba58e3c0fc0f4a35403773ddd7949,245665f14a8553f7f0d892a6844f613cfa7746eda36d24a20eb3f9fa8557d764,2b9925bd5577f1e7a018ea6fdbba1c72b16e99260d5f956e30effef36bade36e,821ce9f5670a44243a5b5d24766abb01a95ac4cec5fc2933e77105bc63ab6986,845b779a21af263fce4b1ccbac7ee985f36d998d1c9881691e73921c6f0927d3,8c58cde1325eb5f6a5711859211405e8b20e10a2dc6e4e8454ff4de2fd56a050,8d2496842c18a3f7aa2df85acd6662c166a054731f81e673c6cfbab639c7392a,c2423b35891382d5c33148aa8423108a7edeba5c600cb7561b623dd64a1a3014 +R-pr3030 93b5a5a36ddef241e29303b75109556e96476c13e674899cba7e98b2556b5b1c bf7f6b32a0bc1ec04956d9ef0bd8fc8f636d1f7e0fb95edfae554445312b978f,fd1d83fd595c69c429874e417d3209de696188fc04b0170e859da833d1d3fc3d +R-pr3048 ed3ae4adb04eb328661567498dd6f184c8aff2315d10efcd5b3ced5521462200 23b512006833f1f4f1ddfd7d78af45b28a891e3324ad2cc93e62f8bff57a353c,8093a7b45736a3b8ee5c14daad8fc884ca6a373b072415e4a4130b7767d481ad +R-pr3106 14e23cce1103258285ba91ff13086eb0b557225a965c60475ebf5bc7312f54c6 2c82df612bbcee99f53c5062bb4f67e96ded9cdafe5e6fe5ddc4d6916c6ae5da,6fa4f75a30b3bc85fa91e3760a0b5f6eb9062d1f8941a5765849f4f94f392179,7c6670b77df071d05e1f03da924aeed29765c8309b4c3a193154a7d9fe3a3028,d2cae08fea7e169496144a4bbfc85dc12c44beaf846f635feb69850436f4b4b3 +R-pr3216 8040b8896518ee29c7f4b6b1803f23d4ff12bb1185a950eb93aac5d52a9049f0 025150cdb7e27ac77cf1e93e6c4c33e26a5e92ea93c62ee98ecedd0e3068e074,040241d05b0c78fba5138d08cc63230d3aa8ca12b7278b22f39b500d73b22782,070dd7554d6858ae6d8b20433aa0f5529eefd311dffdc392da2026ed089db16a,1895134b0cb360373b3bb3205320111f9a11e22f26da457c70b85cc768bf5b2a,66df4896346785b8f488c6f1a787e7cff7138fa38a35e36c478d4c7e609c3f0d,7103135b083d34b94fba66f1c35ab6da529cbdeee217facfe8a81a854ba1d257,7b80c608c7049299914c1f75e14eaf2dfa0433c6e1abf3639963b6d46cc6b1e0,8281a9663d4d82308c76eb25908205397daf2704da747f88026886fdadcd1754,98d8857af435aaa81e555a47811d7eac9074185c00be0bbcbe93e4d1ec7ec537,c99a6d02038b9c8fcb1a5ad76dd775b8ec573ac3161ba820743d64fb9d286ce0,dd12a014b80ec12df862b1561011c23e23dab725c41e08a5dd09266fbd8ab506,ee532d8d06e4f6be762d21e9ec457200f6a2f8e5ca8a4e0eb986d87473626106,f8763735da0aac2027df496586ca654aa90615faba35f69576d74e1e3cdfc7f3 +R-pr3307 2a572c8c6487e7636101bc8c774d3ef8a68d883bd76667f6cbc9a4e06dac7931 df7c6b4011f7b296f31153fbd6b8261a2f1be26494d45cb36f5a7e89ab25ae5f,f41dbf11057d2e02eabaf5b3fa396c5f9ea384acb01857743506e0a881796d3a +R-pr3309 94f1d098ccbd15f33f168d6eda10f9615ccf7750128b8d163e6cb9f15cea9978 00b6af5c18dd01a29cc3fc90819160dd5d645374682365d1b1a2d40bce9e3ec8,34f70542dd0b68abaf89e998d9c75b8749b11438db340cd474212e5d436090c8,4dcf63bf62009e381c363f8685cd47dbd4a4df8df0e905aa9f63d844fb0b89c0,b08c6de8ca466e56e35f967adf7f7ddacedb84c821a0d36be2d4a155ad06daad,ebedbb71eeb4296888bc405dae1811c1023970d0cc5d3d7b1f4732163d0110a1,f7609d5ab1d6b44cd53a72168977548cd45be4abf80b71bae5e0feccebe6a306 +R-pr3319 fbb3dd43cf1fbe16ce485c6938a30d3c354ac4dd64f176ab3bd3e88ca6a14a8f 55444d412eec07798f4c97a347687f69fd4b5ca2fcd95409fe8a9b4f06b07afc,6fa5886f089df384f239f6ee991415af15031d6f1e1e963e9d7c10a2b858c8fc,80c4d54f43f5e16f091d719c62f954034710abbb10017af50d6d17f17cc2909a,8b05f37f9b1068a9c6eb9d7c08d6cc504fede5e2ebf3240162b7e9a9a9fa8536,cc32db73d9f9a4c36bda7e1e48dff782886c871ea2925862752622c65ebb5203,cf311c43e219c228d226b12bf29953098b063882298f9904936279821d70a0c5,d296d82cc0911ba8384e9ee5e9d9a6895511d10de70789757293f8d0258cff2c,def31e1ffeab38494b7fae62df0e4d0c2d82877e602bc66e3447e9d90ad00224 +R-pr3339 31378d158e0701a2ce373cff30d97037736798a2a0697996443a9769e2fb6812 100d98021e03943ebb8fdcffc04220d956643f8e84c2c97bb53e9d783502d007,11935d6233357adcda28c016835cc7ff70d075fde534d07879d912b99166acae,1acb90b27a9905001d31520fd947bc35864fb05055f61eb9b126cfd0e264415f,1f2013767120cc48e9a802cf064c132c38bbb6fc4845c07815a96a355286ac31,4e9f7a8bf9d1f75449984360c389cb2d5883c329ec13e6f6957b919343c4b03c,5700293685c333dc46515698597cac7de587d69e3c9737cde84d12fbab12a54e,5c4a8ce479c2f3d8cc9af05d76f977f23386b806f093c83f181aeaa812b16194,860dd6a30c56fa6c3433063f1315af44fe2929841aa3186521deca95f370890c,9098ca0abf02d69aabb8d96179741e10e765e02e6b8738ba07c0f889c5c835ba,b12a02a039c13a1f3b3598a1b52c685631566a7684fc26d2238555f6482b0f95,cb3d14d20c9fd37309ad03599425d602e4e07f576253030d228171fb57670b69,d152d88f7e69a3e5b13357b521900b8b6334ee2079bdebcdfc6ec04d128fb7f2,df6115680227e77f0d1be6b9eee819117139a589813657e4051abf2780ca1cae,ed5251d9c4b55e3f049d2a70aa1be20cf238bdbd3a32ace8cafdfc75b624b2d2 +R-pr3405 9e6ef22b3d82a12aca58a6849123a39bb01af67c817f34f68585651f0bec5847 057761196a9015b40f530aff4e9922e43c7d62ce6cbe5785fb5cd68ee33be4eb,32524851c8f4e62b2aaa92de02f002c5218ae417c1137462ae9ac2afce14c41b,442e308069e6062efd9c251b5d25a11c157f97cc7ed16448cd03f9835285fa7c,474d16b5a28530986cffe912dbc7f2054694e662d05d7b8d0cb228b43be9ea9c,b122145946aa4711ec35d662116411fb737808f4854644acb4e4cc426ba19df6,e7070fc020385c1eb327ac1810e29febaba35a0a87c2f5f7e9a688d020be89ff +R-pr3469 006cbf09fdb08a2b2074eabd6195675d798d416f64b23cc9cd798de1f35aa4f1 2a86b5a754788925b0b019973a226666ab5d12c30e79f5f5b84e45cb1416a6da,dfc55a8cf11785d5d8630792adc9ba2bcf6b278b8152c300196bb3e0c5070968,e3153e48d3f256b61c93e0815426f1c9bf76684c4afbf8a2fe5bb7d6e640dd70 +R-pr3481 158b3a19aec4fd5b17d87196981f3df79c1519b94d8d5c6109066ce3d06495ba 36a4a45a83bd6689520c5fd121114c88ee147ea776c6f2c41c450fd595c9ca2b,3ac473affde2c29d74fdae2949cb5d3e7ab0dffeddd981b635ae150d9fd43dbf,c0f594761bab59154c7b21b260f8b9862604da2c57bd8ca52932b3ab4e6d25a2,ee9b786581307ca999b21267ee95038ededd640c2c3d2486a8e29b45433d0e27 +R-pr3706 45670c30d023bff5378b5638b90e7e8ab609c1f2d9e062063fb449b56798b572 07f5305fbd6c4c684eb129c4ba6c6e81da1dbdc82886af9bdadeb2e981e0035c,1e9c63cb876214f1155330b71e809f2247ecfad61b82a68f6a3b3240d710aea5,5cdc0aa89bd95fdebbbb14cfb8d120e5aeeb33dd009b53afd160736ccc635f3a,84e6c90fa2bab256184ecb36fab253b4ee0657f4b3cceaaa873b141c3eddef1d,a667fe14bf718eada4835d23708349953672da032b467d767d33743276001fff,b341aa167db1733db7ef926bca1bd2a1765e19e7f972a53f22692650df39dda1,cbba944c03c59c59d30a763a505da36ad1bebd11c44e217f1f818ead329b3c2e,feed4e51de57bccc4872733821272ad7238330cc87fa7dd734a314e2c38a5ac1 +R-pr410 dfd7fd3a4ae169eb2f771c286946890b0596183386e5b9f604d7b6ad3ec977f2 da3efc5161d61e4125ce919a042e16fa630ae97c762c69ed3d438e37ee4f300c +R-pr411 e663897e386a4ff509930d23d8b185c1b9a17a6f0c6ee703c34b1fb2d6157aa0 09a6fee4d75357d1252ed042c47b80a52d2a24d210936e01dae9465fa0826c46,0a70dca0974a30260c181a9d291b272d41e473aaa07cf9f58bf68e87de725afd,133a951e931c4756c959f5ae78304b196aaa2f740493204777454f493e72d932,14db2a8499db3769b434fa7ff0a292179a1b6e3af58bf1099d075de4a543008a,15c2bf3aab96175e52cfd11db27e79d34914417e0c0254913276df76fd75178e,1975fe6afe3d1977e42a0792f229909ebbb71d7c4973921c70c5a76c85eedd63,2051aea24ba40ebe4c6d66f71848478d5484ca060bb40edf6547c6952c3cfbec,2de5466c7e2f64ea8204f4bf0c27fccbc260585d45875c3877314d4480efe430,2f671cc3fa6a31bfd9f9f51bce03f9d6436f78c48eaa43142c72ef389100b827,332d46455c4419f9c0f1b4d9318108ada894f1316d5ff77de72b4cdf13ee9316,3c6c7758dbccb0669c2b29a9cf6c9f93d7a3c2baaebe6406b7f2054a3c9fc015,478cfccfe85e793a8b6eea476672f525aec2e444dadc8fa33816a6149764c9f6,484503bc13a7c688ce2753037733e51e7080d67b7a6007b150c1c95acc3ac228,4cc86cae91563ed28e993b268cebaa3cb37ca35f089c37bc7a293a83e772421b,4e3a6e0e6d94bd69a4a38b964226c218d451444a68da917ddfa0ce63f5413f1e,4ee2ed57501e212e36016e9df100a3fa51f040441b36d1e5d0ac487d2c722b89,5c81b3413c85c7f699460bc30aa7af5ea6cd41d9ba27e6253ac8cdefaad17985,6885cd06f4ddac9940469d3574d59f031fba439e93462c04c912f4068c9c97db,6901720a1015a1e78a480195b3373ab068939b83d41a84dfde193dc517c23924,6fc719e4d2ba20e5976268f28bcb125d0dc5c0cdc7a037f5fe6c8b660a358dcf,7052157d857af4cf919c0ea61efb8d9f4909954105c0ef5186359ee363579bdc,79053d7fdfe77ca44dfde1587d70f658e4421431c0846cad953db30235e0e5d8,81ce21572b271722d7ad7be38e42c5eb213547785449818b7e5967cfed567aa7,857cc6d883013c2e6f965943bb101c8da4fd821139b17ab2316ac5bbb050385d,8629fe66a7a2cba7bf43f44489304b2bb17167536f9850ee6b8b31f0a9ca5264,887103e40e19838dfac6485f45ab3de3132a469ace2fcb14b659581404b25671,88f40ac136f61f65749f72e7ed49f34318b634fa66c6d0e641f41943badcab80,8eda9a9c0ba134f26ffc9ea2622f23551a8f9fed298534ff0ff1172c4171b240,8f7de56e69fa91bc3189d99bfe50082ff182e6f3c2323d5574cd186adc6bc4a3,9408a281e3953120620a9341d838298af4885de301d826eaa52bbdc5e305510a,974508edf0c303cbe2a45e254ad9269efc8c8345db2ed12ec3a625ce5fe2f409,9b43f0c2fb72cd7f9ff9e794c16bd6eaa2ef45a8d585f38a7e3c12f65084ab08,9ba220f309323b43c446fdaa80de83a771d266921d06c8553156fd551b247639,9e18831a71447da53931f05c7347a1982271d47df83f08222ad4ed9576a5f4a3,a32605b245742179dbe7b5d9538b17545bb1fa5023580f332f305882227941ba,a64e44588bf8015638124d36e5676b04bdc56dc656a62b3a0e6c70f6e254ad9b,a827993fe0690e7900c0c21f5920a2fd66ff41e68880b933ebb1706816a40eb2,a9b45d51fddc3066d9fa7de9159395392ff1cc382c0ff751faa5924b5124c069,ab03ca8441d80167333fe7259734cf64e3e47f7d6f58f60619f3fa4489833991,acdb74f16c680fe77b2dbbec8dc332692396fb8338516318be82fe28a4ab5cfc,b02def0eaa9298b9c73c7ed408595590b9387f5dbc9a6c4ea298d064bd9aaf5e,b090162d68ee1fca6e16a0fd0cf5b5f7de3d498c5bcb3c6879a9d9b58660138a,b2f8176b65dd567c7b20cb751663ce06c1658cda75238576bcd8435dcd3dd5c2,cdddc1a58cebe9cf1fccca9fe89506fbab4c164da7c3c9b0e3a9ec47ff978982,db5340ec25f87725bfc02ae5a9f31d8224473151060fffb8ff85afebe51fa5f7,dc0590104d235c100ae3ed250d9b13b2ec678642e8a0a62d4cbee31ca379b674,dff6675d1c17b02653b4d30ac750a70d9500023d644c15404ffc7cf1198a8520,eafc83afb221d7dde2309431b5642c9bf80c8ba92567d5ec00c90adcd0b550ef,eb7764fa7b110499da747baad91b9f1b10d2ebb31adb2bc44c255b8b4afa89b4,f72af5dc9d5cc1662b0318c610680e762564cf374e4893ad13014d5e2f3db8d1,fbbb3fe48e392b5e603eeb398805720c335f1d3e8fa98a5e89bc6c26809139a1,fca0f2ef2e21917923e77eb3d6c8036a9b605bee03b930936594b0e2df51acf6 +R-pr750 1c288258c6724f6d343c02fb5288fcf62e76264e3c163b6dd459856f334abebf 30a2a053d32f1dabca7e3a3874c2624d03152c0b35540f639b0d10c561220249,51c1b916e48b81a142c1161321aa0abeddd785929a35ad777865f1e467fa8bcc,8bc358ca2f38908e3cd6dfd9186407cc0ce9d14c4321c07ee381a521704b2c35,94543183d9e81de2d51d6435ee41d29fbe597806df5c20ecff3036083c41f88e,9a9219ab43dc21467293f9a3de747ada7345112925fb135a4678ac200c24b80c,9e1673b44880da54007549c5f59a234d08eeabe59c0b086371d3d8a7e6ae974e,b64082dec2accfcac10a4de7522047a62d3267d2afcfb1bdf37075f3a0450265,b79bfb617308fdb6519fc5bbecd8f7020cf29992b31482224ae9562c5c3a8bbc,d9e8feb93cf88f718e4d55dfac5e15851b13c916dd65a5f72d5fb7b1b78bc253,e37f9ed5ccee88baad5f18c87a7d013bc2f75e78d7173bfce1ce5ea72a169ac3 diff --git a/docs/audits/review-corpus/test-sketch-v1.txt b/docs/audits/review-corpus/test-sketch-v1.txt new file mode 100644 index 0000000000..4c03872cfd --- /dev/null +++ b/docs/audits/review-corpus/test-sketch-v1.txt @@ -0,0 +1,106 @@ +review-corpus-v1 cluster-sketch shingle=12 k=64 +G-pr1026 00233819e5ca79b4,0040f41cf2cdc2ee,00554c64909e4866,006329be4d4c98c5,00be04ec02c1226f,00e2a93f397e2729,0190cac0351b9dff,01aecfb4210e9da2,01d0ab0d1b113b9e,0218ebb86cd00043,02df0e4d6fe70042,0342b01c90b5a17c,038b3f29309c3b22,03c45a12eb539e04,043fff7dd9b7d697,052b639c9a21ce14,05b6b7f9275ecaa0,05c6494194e06068,061654c568cdddb6,061fa71528790b12,0674db9fe020f342,0675a76d7a4b618d,06fe16a711e5b28f,0719c4fe36119ec8,078dbb4190fd934f,08326766b3e2cb2a,087e05b3d1c1f331,09110c6684c9802a,098a408bfb550327,098e1c123480c6e7,0998d2b6ac385d12,0a29b65bd2d1310b,0a78e8bca2c143b2,0af25da2217b9ee5,0afd466664cf3c1b,0b1e3a52c7f13a65,0b33cb29f4f15659,0b3cd171fd855279,0b6273c21ddbdd77,0b75a8b2070be85c,0ba3bc0e2cdc82c9,0bbfb35dcc7898b1,0bd8a95f457e0bc4,0c8ac93ff8777da3,0d1b544648b24077,0d60cebd27d6cad0,0d922bad253744ca,0d9fbbb8c3c542ae,0dbe83e70f32366e,0e69782af6cdaa0b,0ef3276f2925e500,0f2761d52527e0e3,0f281cb5ff3b5917,0f5f54ee223c57f8,0f8c149ffaa16b65,0fb1af42aa25e7b2,0fdda4e524e295a8,100af4fbd1906010,1045837adf1648dd,1067f998d31fd186,10850e8bfa0c87b5,11d46e91f215e6f8,11f0982f0b8afbdd,123326b137eb39cc +G-pr1052 00028a86a45efe1b,000e88b89fcb5fc5,0015a74b3c42f0d2,001d0c38afa5bc84,00314de3d279dac6,0033884767c35407,0036fa5d8449e767,0038d2cdf1f74474,004c6c6b7a2dea81,005fc33f4c1402a1,006e1b5f21edb932,0072fe6ee3073849,0086e4ebbf9aa654,009f002ac3a1bf7c,00b47a59627b9f14,00bf03f1cb2e8ee6,00c44768722bec38,00c45311217a8b44,00d54e7631bc3c29,00e4710d385839e1,00e591c16ec0536e,00e8646e82db8ba0,00e91632e5cfdd88,00edf5236882b733,00f96e6b069874a9,01047326b84eff29,010772ca66a4724f,0110ce9a255e0d1e,0111ba2c3ec29dc4,0113c03ab3790783,0126fb1dc4b7879b,012b9fa95e8ce942,0131a029074b0428,0133daccdc238472,013bb31783e39ed0,014745a66ac9f817,015e11dd0ca808bd,016541480b4258c2,016b6756e98d22a5,017a6969e4c7bdb7,01823132922c8d9e,018a2b1f97dd03b2,018a6d95db36ed5a,018a7d2b1f1f3913,018e44ae791f137b,01ad2f8b89106fe5,01bf48b827a3d1de,01d24e5c273c496d,01d6e508dedeadce,01e5f2279905bf37,01e9751d2e26080c,01eaa6010f805f6a,01eaf32b11b5ebee,01f06722d0822893,01f11c3a6c45eaec,01f19f5b32045544,0202305d76e5ce4c,0214eeee34f83fc0,0216ce6b688ce6ac,0216f5dadefdbfab,0217b3e1e145abe5,0230d812e887953c,023d9b081a98cff5,02485bd9f2f816c9 +G-pr1058 00b78d644411808a,00e7f34adebf7a62,00f1431b15f4c34a,01044badced52faf,014ae6d36cfe4e3b,02b65fca275f988e,0342626cda0e4313,049463b6133a233c,060ade7c8491f860,0628368f88a5a686,0633889301b941c3,077ab802988df174,090c87a79943daa7,09166ea1d22fa7e7,096cb40708cd6d5b,09d04dd37be03104,09d0c98ae57fd281,0a9253c9faccded1,0b5ec2756e480510,0b94aeca2b5bc0d4,0bad80b951ae8f52,0bcf3405e9e03056,0cbfab2e27645c43,0d91c001f4f58da3,0e18df1778f92cde,0e707a38d1d2362d,0ff57abb9941949e,101fc5645ff53b8c,11a1a3b434b83617,12264a807e333363,12d225d5faf4b968,135fcef17516acb3,14e90bc6e15d3903,1637097538976c42,163d46be4ac92449,16eed46804c93b17,172d0e05b7400a78,178625e8eb397e99,187750c97b69dd5b,1890138da69a8c98,1a8bb8ec7bad879e,1b22a49773a8ca48,1b6dcbf3bad4bb5f,1c5b5b597c64cd33,1ccb1c31ca0df9c6,1dff23ea5334d5e8,1e30a5004b7dd1c6,1ef70f871b5142b2,1f4f71123fab1168,2152b12ca1898054,219dbeff73f27503,2297bdeab053fefe,254d7732b914f266,2647933a558d1acb,26f764ee85bf6b2c,27b5e313ab733ef1,28636d420fbcea19,28e19848c277ca3d,296a2d253b6e897e,29bb3094cb559d34,2aa13fd2de07e7f1,2bb9dfabf49b696c,2bceb9834b712ff4,2e08510c0ca9001c +G-pr1081 000d182a44389507,001f208236e3475f,0025745cf1369a37,005c6a0e8ad99670,009a6ebd3b1f51f3,00c0a76404afc744,00d4852ba07efde7,00f14f6f079afdf2,01354ca1bdaeb150,013848ca9612168b,014749d3d96ced25,015fbcbddfa8dc57,01993073d52804ba,01d5d6182b830936,020ddf0412a70536,022e009974aa5d9b,027267e0e7d1f579,02761bda87188aa3,027a75826483131e,029fd8cd7dc99b0a,02c5b4aaa03e7b0f,02d634aacfe70021,02e1ccdceffe033e,03020dba55f4dddc,03459fe00877ae66,035d0fd3ace7a0e1,0361a2b6a0c3a27d,03cde114be3420dc,03ed08a7659cb1be,03fb6c9c648eb2c5,0403f23496784e1b,040ee92b9cb1f93b,0422a5f20897aa7b,0432299ed78aa1ef,046453cadd31a1b2,0472a2387164e197,047926a59a5ab70b,048df5b8c7d8a986,04ba2483ba3c0e28,04c23b22300babc4,04ede716265fb981,04f05422bca60543,04f75e6dda982789,04f892fa5fd17bfe,0508ed67da9e1858,05138b2c1196b9ad,051822e198de6253,053e7d4e0d3e2544,05439c6052e58e5d,05706b9bd39c1b26,05a7efcc852834cc,05ec84838354696a,05f25489d49b99c8,05f2cb65a7da3f06,0608370b41eacf1a,060f53bc11d24aee,061705989a6505a3,0620043fb58ff9b3,0638731bb4319ee0,0667deaf1ed878a7,067dee325bf4dfcb,06887c6b4328b438,0698c0c09303abf4,06be5558980751eb +G-pr1136 005689c497c435a1,0071795423fab65d,007c977a0a045202,008aa870a26db23d,00bd37ebc1d84957,00bd91174ecae264,00c814318b7f6752,00fd2d2c0ee8abe7,011124969f83e4d5,0124fa1e7334ea33,0145e083220bd50a,0153c1d1a79aedeb,015c906cab9d2fea,016ba0018a93e34a,0179cc7ec67ea97c,019794097bb36615,0199f2eed587fc3b,01a56168aa152733,01c71e365dbdc6f0,01d2640e6dd9877e,01deaa70ddb1ca03,01e095fdb8927acd,01e7f792a2e0319f,022b0241be39cca7,027eb4785351a927,028d784dd27ad105,028f038c26850436,02b011aceac523ac,02b16bcf24df62ae,02e22f814ee353c9,02e8012c248a986a,02e91ace44015d59,0318c785045c3902,035dec81b0896d39,03729abd0d8faee1,03788081e7719aba,03876b6cc5e13c30,038fe127d540984b,0392b5b4faeef0d8,03aa0a89cfabd656,03c1283cf609a782,03cf0f37722e2a6a,03e2d79943739998,03fcebbe278c88a6,040d81e7d8ebf164,042e57abf7d5f143,0437f5e18d7cd7a9,043af7e11a990df2,044a84f9b7fbbe07,045d9eb2ef7d9de4,047b2768a90adc87,047e62c6756a28d9,048cf834ae178858,049e5b5ada218fbc,04ba2483ba3c0e28,04d084b009bde1aa,04d700764f8b02e0,04e05c81b3d11a3f,04e99a653501ec94,04ed9b41b8610a2e,04ff0d2a780ab1c8,051ebac026970342,0540c0fd34fa8499,05834f0be2f32699 +G-pr1408 000aa4a005860d5c,0010ae590eb71dfe,007e724d0f39c9c2,007fe6fde9f37c3d,009433a0a3305e29,01009b70b8616c9b,014bc4e98820cde6,017edb43fea2405c,01c71e365dbdc6f0,01c9157c9b595338,01d3cc65f63e8fcb,01fe63c5cf38a819,021afab2d3fe3910,022bf34b6078b9de,026c5078e14e5ee0,027a4a09f5be90df,02866cdd7bed4d73,02b1e24294659055,02d3528a82469c3b,030a68128ad1dcf5,031f14317da411cc,03e58f5af10b7592,0420bbedfeb5720a,04383e4e199302bc,0488c38b173c2924,0494bb6145d6befa,04b8a2ebeced6e81,04bf91403725060d,04d39988554a678a,050b930a96c77226,053e1f24cc0c2195,053e7d4e0d3e2544,054153435320e585,05778ddde2927e03,05d7397bbeb171bd,0608f0aa1c710fba,0642ca6a78b305fa,06af808f3928207b,06ef1a9f6a449ba2,06f98cc3cf372817,07cf1486e9cd2c93,07d4cd1e0e8c56f7,07f39e44b0f980d4,07ff7eb09d8a4284,08b4dc6277218e8c,08b798f0c37a4a43,08c0a46ef42d195e,090206419579ccdc,0930c2e9f7454962,0980e3ebce771019,098cb3eab3a22ea2,09b86c6ee6daebd5,09c9c2ac16c9f42d,0a6e68462edd50a7,0a726fb5015e2179,0ac12ecfdf926374,0b122ef32ec1a77c,0b2d89e3369f1908,0b4f270d24a517ef,0b6a0da8e9540068,0bad80b951ae8f52,0bc9be0b2b6f2578,0be70060cf34c23e,0c3793b46a6cbc81 +G-pr1516 000b6d621ca4b044,0039a85ef9ef4065,00577fd5e1bb105e,006db43a7814a475,00bd89fd330b6bac,012f58afbc2033bd,014749d3d96ced25,018bdb46c2e96584,01c71e365dbdc6f0,028f038c26850436,0305bf1378ea9e02,032e0f280d491a8b,03459fe00877ae66,0377c7bab9538a0b,03ed224ad7c73525,040bb5cffd4dcafe,043d4b5c3aa5540d,044393dc4325f145,0489465256e55e94,04b8a2ebeced6e81,04e72554b735f1da,04f25ecb13e83fb8,04f5686ba46cdbe7,0502c4c595c3b59b,05143ce9a51bdae9,051c12f02efbc168,054983c2db90b8d3,058d9e205029aa2d,05c7fdbfb61100fd,05fa68663e6bdce1,064f0ef9f2ad8024,0651a44de2a62499,066c0926a2b09ca7,069a364112eacdc3,06ff49c95dda0202,0718238bad800dd3,071b114079f6e059,077327079d241d27,077aef6d9cb315f0,08260d44e5e99cd5,0846df09c205d70d,085d48957fc71291,08607c2ab718be57,0876474ec1eb0876,087a667ad33cd360,089ff42f4c4bb345,08a31a4851171c8c,08a4e44a71c621c9,08cc603196f93ea7,08d5eac97110075f,08e01dd674f3d094,09293e3e1c11914b,0930c2e9f7454962,09cf7494ed0beadf,09de1ac97674de6f,09de758819416615,09ec58a0e8f65ad5,0a10bdae6e5e8501,0a148db7ee5e3fe0,0a310818a8e00aaf,0a36599c7f0a3ac4,0a6e68462edd50a7,0a8458edf2a7ed0e,0ae5525446ecf897 +G-pr1536 0038efc275bea373,003cc64d201a42f8,005a6f5b1334b0ec,00e5e929c1667f4b,013f55e93981f78d,0167e3e57ea30353,01a5c8616db57b61,01c71e365dbdc6f0,01deaa70ddb1ca03,027b444eaf0e04de,028f038c26850436,02c0d73048319272,034e26fa6e201b10,0365c702e28d4e6f,036fd4ae021605c8,040f9e403a8af948,04779826fa4d5d84,0484896aba6dcc56,04918711b2c7d6ca,04dce9cef0f08fd6,04e05c81b3d11a3f,0523c58f3e931656,05882436a627aca8,05b2d3a521e65746,05bece7f0b41583b,05ce744c119c2d77,05e883143e18ee04,05e9f5a71dfccf92,05f0e96dcf720a66,060b240a414307e2,060f6d0f4fae78fa,062557b8a81993cf,062ef2f49e5a783c,0646026ea149fdfc,065faafe5e9013f9,069029288a243db8,0695003b84acbed7,06a215e1247f7175,06a2b02b8bfd9a3a,06b318ad2c61eeee,06e29c8a3b91db3e,06edb692a55214b7,07154d6e29aa526c,076591a9a7afa7c0,07826a615fea4a9d,07adb731a9157060,07b86164170e21af,07e9003c64520986,08555b130ad0f083,08604e8f65332b70,089a14adef6f9a2d,08ab0dcc1104a015,091944209d4424a8,091c7679ccf8e709,09c44ffae0da684c,09f8fa999e9d0cbb,0a2c7b021423f139,0b2006bd6e97f83a,0b46ab60bff9e006,0b4f7d28799e6838,0b7097c308b243a8,0be8f78f37d47a59,0c20ae8634d7b6e1,0c220863b890cda8 +G-pr1543 00120e7f77b55f99,001509c574b5d6a7,0038efc275bea373,0069a903194dcee9,0096a622d351afac,009b58b2b7f5ce84,009c24f6d7c272c9,00b0f93bbfbfb27c,00bd0b391a960767,00bf6e2bd6135426,00c3f6b82a724842,00ccdbd0bda8c393,00dbbf34d7274cf1,00e22c2718e37591,014981f5e1e2aad6,01a4becd760c6030,01c71e365dbdc6f0,01c9346e6f6cc351,01d029677ea9a848,01d773d9e2763ec4,01d836af8ca6d37c,01ffe1540dbf0135,020012e881cbd925,0238a5152e0de2e1,02429626dc3b7e9a,02473d62f0cb8ecf,02686641e6d41c84,027ccac95414143e,028c8e99df38614f,02d8962fc6bf3266,03043b3d26f3e726,03193ff41547b02d,03395cf27ce0e4a8,03459fe00877ae66,03469bf0f7cc9e07,0352ee2db84b986e,036233d6c5517094,0395690ea4f44dbf,03e50f7e4b36afa1,04017144cc8d3a3b,0414ac16fa510b14,041fb5313c2d6ac9,04207a794d5117f9,042144c3b75883fe,045a3c15ab46fe8c,047f80a8f3a9378d,048f0b92824f1e28,04a07af60c055074,04a56c594d37f4af,04f00b716fbdae96,05303357f83c7222,05322f31eee255fe,0548f574a159b7df,05875b1fc2bae231,0596cd480cf1204c,060f6d0f4fae78fa,0641a52371ea5509,0652cc40141f0f2f,067013368704f2af,0680cf46ddcd8c31,06924ed51218f214,06a7272c1dd71d85,06ab8efa1d270433,06b6fec4ce7e2817 +G-pr1555 00031a36c5322969,003038efb5fa5117,0039ce8a596e070e,004710719118884e,004dbdd25d799004,0052fa67e95ceeb3,00582eb2754cd938,0060acea5dfd2e45,007b193f2b7c2e10,008138da6b1979a0,008b3d9866ad92cf,0095bcc5a8069c8d,009929aed284a068,00a227e909c453a5,00a6e8193e166331,00b096adf3c85689,00bbc99ab262de19,00be6b09bc963cfb,00c612aec618b404,00c647fc48d23ed2,00d06e1fc4cf74c4,00d28091caba1460,00d66a50d2440376,00dbb361bd8fe7da,00dbd0d9e79af202,00e3c396f7f5a847,00ed7224598d900e,00ef52aced202243,00f95f2ca18f4d51,01082bac13239108,0126d38b005ec4a7,01270ae13b789900,0128545b8dbd009b,012d9da750fecdbd,01331fa128b482c0,013f8dce8c0f1f6e,01457dee8ab62215,014f968670479812,0156d4d34f0e0dae,0159e45cf0077c2e,0161ebade7b2561e,01699b59aaec6e21,018139f9fc77d4b2,018d574d863ea61b,01b6d42160eff137,01b6e4ad2e2930ea,01b9e029b32f0f73,01c80fc15aa5e599,01ef4fcaf078b0cd,01fa6846e93d27ab,0201561401f6b487,0209d23fd088ea19,020beadf8b14bf40,021232ea7a63328d,02229a8dbc63ad7a,0225921475f97c61,022d9f096de1e080,0232cf8e6e285c24,02330a6546e8182b,0240d453f1482e3d,02506a4ad0ddd01e,0264433c491b6230,026f3d3e458e9c6a,028f213586e13646 +G-pr1568 0012fc94caf1ba41,0017b905b6551794,001b1cad8722edcd,002b892427475a78,00340da0e8441a98,005421948510a8fa,007093e6fb67f5b7,007aad658d421f11,007c977a0a045202,009ce54f9d530624,009f4c02fd47dcd8,00abcd2fb3eca351,00d1fb844d35225c,00f2ce90d760f791,00f66511add4e2e6,00f697fbd8697a44,00f811236734f304,00fa6d2bfd6f6148,00fd2d2c0ee8abe7,010a8cefbdc34a9e,010c2665cf9d1d30,010def1185af35c1,010fa931fe4d1d90,011f271e52e8ef50,0121cf9865f7bda0,012a979b0a5defbd,016103c9e554bcbf,017042731c36b6e7,01790124e2d7cde5,017aa514b8eb4e1a,01c3250327de4240,01d2640e6dd9877e,02113408973947ea,021149e15e2d21ea,021584daac8364c8,022d06312d70c7eb,0244b54d40b9f8e3,0266f06702b88290,026735e4628dc7b9,02680c2a3661b06b,026ef85fbad60f4b,028016725620fb29,028d6baeec59069b,028f038c26850436,02a52a972754e64d,02b6b66f3af57cdf,02cec62cc9b4af4c,02e22f814ee353c9,02fec701318c00c7,030eacedcaf93ce1,031c75f1a1bc8198,03394c79a85bd236,033a1c0dee7b0fc3,034515b68419a22b,0346f4aa2084422c,034b98d441f8c87a,03678674dab35ce1,0382aed8803ac19a,0392b5b4faeef0d8,03a4d4569b972b70,03bab09f8e4e7c86,03e2e9fc4e3f452a,03f81e1df76aa3a2,040d81e7d8ebf164 +G-pr1602 0022c5ab9613d4ce,004e0ec2d02622ce,00576c6242680043,005e063913f1997e,00737712c96d00aa,007c977a0a045202,0081dcdec9bbc371,00db0c1874d16120,00fd2d2c0ee8abe7,016185a811786722,018c3979c481bc44,023027e82750764f,026b50bd5918be07,028a2f6437404b1b,0294d3b990819b0f,029905295dc5fd19,02b0fadf1a0883aa,02cbbaf816d9cae4,02dfc2bd2c983d61,02f48b1c4f2c2f65,03459fe00877ae66,036d831b23fc8ba2,038110a80e1c7c0f,0381baaea9b106cf,0384247c010f3227,03995ec8e309b70e,03b2a50e01a95c25,03e34cb7c396b7ea,03e5a4ee4b5a548b,03eeb732a28fbadb,03ff3dfb94dc52a5,0435b01dd9a6f87d,04ae1e6f40c47635,04b018225ec0b737,04baf742043c9814,04d87e64f1d6af45,0504a5ce51391a8d,050e8278fa58da72,051bff2f645b263b,0523fa23ca5613f4,052ae7bf182fbabb,053c3532b0910b7c,053f1c46162e3199,058e386f4011baee,05aed39f51b63b65,05d073434557932e,05f0d941c920f1b7,05fcf3a9f7afdee9,061ab31965bfb740,06366fc160ca9ee6,065c85ca8aa69d17,066a0f2664e22e3d,06931ed4e89daf99,069a5f776a737a6e,06e78f1cfff0ca0d,07154a758dd48759,072edb3eb6c3be6d,073c4474df3e50b0,0740908ea11c51b7,0744787108beec3e,07520cedc6c147b9,075464192425bc49,075e6783d3ba0cb8,07661bfd4b3b4f07 +G-pr1664 004d577ba8cad082,006a5a18a0218a69,007d64b346e798f8,008d48e93485236c,00c85d83f536e585,00e58e8aff8597a8,01073de34b41f5c9,017e570242ecaaa6,01ddc867553a059b,0229adca26e8e983,02b436c991fd1491,02f547429a5a4578,02fd04257b4c979e,03eca9e99b727fe9,03ee92a31170dc4f,042eb322117c615e,04476dea909a1ae5,04a4de436ec0fd64,052dc7a0e4b546fb,053dce5e86cabe4d,05ab6de47ac965d4,05c1ffa9d20536ea,060dae0a0ce56f06,0629b024a618f656,0664b185fe1fd016,072edb3eb6c3be6d,07adb731a9157060,07cf1486e9cd2c93,0868d8fc234ad26b,08a3a96a8b3c8dc7,08aa10a4982ab2f3,08abfce1ae9cc08c,08f298b9c37ca8b3,091cc40833926fe5,093ee019bc711c31,096c7a5844104c79,0977e19e62c988fb,0978767d960973b8,09f0b01d8b046828,0a4679e2e5506924,0a577c095022be7a,0aff280e2ebff339,0b60443870e860f7,0b7a0ee03da36a74,0b8d20b2c15a8dbd,0bc05a4cca7ea946,0cddf17d49a92fc7,0d0f9af18f71de3c,0f2d5d6196677e66,0f68e9397ee395d2,10614cd6363aeea0,10cf460b712c93a8,10dd00b90afa7955,114f23ca51053f57,11816387f0ce869d,11d3dfe5b4aa7df6,120d99500de509d0,1288b3c0263abc60,137ae2526d8cc20a,13f5ad637dc25d56,1414dfcf029f7a06,1419896d7e3375e3,145c49f64faedba3,145e307fec2e8225 +G-pr1694 002511c291509840,003c465251da2515,008b9e78aff09d22,00f7a9723fec351d,012f14b54750835a,01e2dc6edcf4e7ae,028f038c26850436,02ef0d07008e0248,035fc048871d8b92,0387250ad8a4a527,038a7dd4599b6cc8,038e60c24eeecbb0,0419dc75f4af857e,042c4c5da16acec4,0432ad52909d1f9b,046490e6d19db7fc,04aa3c75c85d3e80,04b891af86a96366,04ee34ca390df516,05d027d4334f798a,05e18d1a0f8eb572,065ae2ac7869e0a6,06edb692a55214b7,074e80a7b20a9b2f,076a7586d53b2c37,07a6db684e1e5b7c,07ecbbcf6ae54f14,08be8969d0b48abe,09e6bb3621010f91,09ee911ec5270582,0a0c77d4391520bb,0a14fa1c9ff6f53f,0a1b92cce8ccce0c,0a60df6782b541c2,0a8860d0e34e79ec,0a8cebb02a764e83,0acbf0b831b7423c,0afcd86399e11245,0b16a8e08b903da2,0bbcf27c925a83ee,0bf656f229edd7fd,0c150631747d9d37,0c9bd8622e9798d0,0cca789b04594f64,0db4bf140e1c7142,0e33103f1c73cb39,0e85adaf8580480a,0ee45cd2fab0ef16,0f5bc79453be6df5,0f7d2a3713f97165,0ff6ce816479b840,1030a842c9fc1eed,10478e6c8be1c072,10a41032d344b8b8,10cf960731ebc0b3,10d0bb328ba85338,10f14aeec5bccc57,11710f8fc0902151,11768d30c0efaab4,11a8a6db2cf59f7c,12293dbecf2b215a,123205af413eedc4,12941c5a59710c52,12bf919e5b7f7e78 +G-pr1779 00084a942091c317,000dc20867ee4e97,001cbc2dc29ff58f,002c0d18e19abd51,002c7be35b44a035,00307cb5486d4eeb,003676f4a989763e,00381a00a1f3c92c,003f3e420f19d3f7,00520247347ef185,00562aac85a4e3b8,005ef8a938d4727f,0060867fdb69bee6,006420b2bcbca8d0,0065efeed51a9d46,00754b0c26e8783c,0076cee69d0f6a5c,0078af6fa3e631c3,007c977a0a045202,008294a0d3c1513e,00853a75e60d2b61,0097ff15e6f35de4,00b0a162fd94b60f,00b2deb94444f4c7,00b950963b758ffe,00bf1f31bdd965e1,00c1d1b2135a6eb3,00d076d4b2fc343c,00d9db6f034b2cf6,00e9ae8675956c2b,00f0b17735c5f927,00fd2d2c0ee8abe7,01014943ce86aab1,01110e30adb05619,01169d16b78074ba,01169dc863e214ad,012980041ca43c42,0133a6a453379ec9,0138712a3a865af6,014981f5e1e2aad6,0155c8e1f74ce257,015b9b000a39dfbd,015f9cf07d3d4c13,016786873eeecf12,017ba5497fcd08ab,017bd77b04d66595,0180387e183a01b6,019b04da875d6a01,019b10bb59d85f49,01a62ae3cb5d2361,01af0ac96e73fdb1,01c30897667039d3,01c32c732085107a,01e012fb73d7bb20,01e22c083277a21d,01eaeeb820b59c22,01f137dfbbe230be,01f5070671774c66,020bcf8cc343f394,02215eab6c9cc470,022d06312d70c7eb,023b14b8be6f7c3c,025633a60e1af3b8,0256dab7fe3784b7 +G-pr1804 0203b65ecde2213c,02aa4b15166c3ddf,02e8b6b810a72089,03e44aa70dff0928,0480e50343561379,04dae1f3adc100b5,05707ded3387a447,058ff29c074ee314,0600bbe3b0c8efd9,0619e14e9b3df655,06b38dd945149aaa,07adb731a9157060,084ddcffd0aa8efc,096f03435b625862,099ab18c555bee66,0a08e2a4ddf94ba8,0a5797efc157c783,0a735e30a7d36319,0ad2c14c8866c3a6,0afd69a97ad2242b,0b4cae7fd62dca54,0d5d1d4703a59123,0d60e8f522247d23,0e0cee7ecec483be,0e345ec17abe0b51,0ecd330ce085e749,0ed9ef56629dd1ca,0f1a32f31e00b04e,0f308c8c5afd4b26,0f313d0942e7dbfd,0fac1f9544217298,0fae693170ebe3ed,10222f521c2594ce,1056d66c6f34362b,1142b4f4336a71cf,1150f30361dd3af1,11a030cb511c6768,11e58b5ccc10335d,1230f4335335c556,126b248f0616beee,128291fe30fdcf9e,1288275810223eac,12ada63318f03ae1,12e46e8aa0953e84,13a0fbc55b7a3059,144f522ce3dd3d48,17337c8010637d7a,174de3baf230dcf2,1953f25712bf6e59,1a7e44107f423640,1b7e57ad23659ced,1bad41bed888c450,1bcf34e4252f8f7c,1bd7f3483efe3663,1bf852a62e96af3a,1c39e5a63f1eea7f,1c78db9a60c7ae8b,1cc64be12d7b4e0f,1d6ef3b7af972d2e,1df5c8203e348cb5,1e37e2583288849d,1ec49286afa96639,1ef81b9d5ab61ad7,1f015eb855b18d38 +G-pr1919 0024f4c8a63b1e80,003cc64d201a42f8,00483fb9454721ee,006d674399b1172a,00835629964dc347,00dd22a77e0dea55,012a5051e19f36b3,015c1c4226ff6303,020b1ea785303898,0226d44616160370,023947e9c05a216e,028790f2b6d1b9b2,028f038c26850436,029ed319f011b300,02a86652bd02b3b3,02afee7ab6040672,02e8a7806d15dc50,02fa6485b55233d6,030cad1fbfb4246c,03729abd0d8faee1,0392b5b4faeef0d8,03aa0a89cfabd656,03d3c0eacfd9c3d9,03df6c9407c6e32b,03f8eac6fa3913c6,0422a5f20897aa7b,04ba2483ba3c0e28,04ffe61c5622e7ab,0541a8564782cfb2,0582a885ea125bbb,059cb5ae695cb8e2,060ade7c8491f860,060afaced9b7f16a,065bce901361cd34,066f533d34852a90,06a2db74d13a2db4,073576de13076883,07a81a3a36a1820c,07ac6922857464e6,07e6b48a557e55db,0828f5d428a9560a,08389b17a0bf7394,088211ee61b1e1a6,0894864934716847,08b84f628f72dd0c,08d7af4713e545dd,08f298b9c37ca8b3,098992b93526a700,09dab7ac4becd38e,09f9e19eeb0958d0,0a54ba29d0f82226,0a5ea1cb05e3eff8,0ac32f1b999beb74,0af24b7f4e8dee25,0b566d9b2b6b09dd,0b79ab7c16a84a31,0bfde798c8e36fcb,0c1165f35ab747c7,0c2e99d66857568d,0c71d74a2ad5e2fd,0c9327de24f365c4,0c948e35bcacffb8,0cb2749a09e5911e,0cb808f2fc4d9507 +G-pr1962 001b1cad8722edcd,007e1f4e73550908,01269eb559d74356,0141c35274918376,0186ba5acf713fce,01c88b62b29712e9,01fd7c30b7306570,023752eb37297cb6,0255a1eb706d6bc6,025a48e715ca47ea,025dabc7fd5b3450,02dc128d4f837dc1,02e22f814ee353c9,02e2d07818d844c3,031f8cf2ca65f5c1,0343d0685842a7ec,0370dfc1582da228,03bee2ae5d82c736,03c7f4a85f4f8d29,03e96b1f2e902156,03eb396dce16d552,03fb57d4384063f4,041a6e667005c4bd,043558b0805f70c2,04821f95485a9745,04b8a2ebeced6e81,0522a768acfc5501,05b6111db5db2275,06319b31b727be02,0648708cb4cc3ad7,0658493abcebb8fe,065bce901361cd34,0669e9de334876f7,06829ec471e18d0f,06858a6958d4013a,069e9344c0cfbe23,0745e61ccfdd8e32,077ddd942184c063,077f7830ae4c8b14,07bb9a9b0d1e1d82,07cc0f1c252c6ada,07cf1486e9cd2c93,07e8d50e199ab301,07f680e20d3cc396,0801385db35decd9,0818f24544f8e840,087276c0d98ad312,0881e8121d312ff9,0888d358ed272aa8,08c6dd12eb139d69,09250496abc1e1be,092de0c595c4c7c6,09317c3b59c885bc,097aac8e2eae4e5c,09898ef0701caeb4,09aedb454eb24d8e,09c915993912f525,09f1aec1c49eb0ae,0a1e42ef967f73af,0a8300fe1a550cf4,0a9ac2cbc17b8699,0ad468bf0485d43b,0ae301833c3cb50f,0b170c893aa8c68e +G-pr2064 00013b4e44910cd8,000667c2eedd9859,000ac3d339349b03,000b837e8f96a2a5,000cd8bce2138792,001da66a5fb8d2a1,00214f1d6699cbcf,002aa8ccf311f300,004c97fd172a07f9,005ce696c83e007c,005de808b7ba480c,00634b9c9159a097,007023d465f3b6a4,00753af7a051b236,00819ef6a545bffd,00888e31642953ce,00a473538d470f0e,00baf636064d678e,00bffca0dc2b41bd,00cdf5d1b720c0ad,00ee00d6f43c9099,01191d8358f75846,012980041ca43c42,012a57c531d7130d,013253322c9b97ed,0134bcf7eb52a703,01583fdf0b6ae383,016032862255c5c8,01689d5fc4e34d54,0178f3835185aeb1,017c0002bef6cf34,01b82229e971b32f,01c8234bc475bae8,01d0041fe6c53c1b,01d8fa95db4d5f6c,01f358bd0a465907,0207c91a451854c8,020bbed0824cf2b8,024ac765fd131c21,026cf2dbe815b030,02849960c45b3415,0285136e36de0dfc,028a6160f4214ea2,028f038c26850436,02c4a79a7100db06,02d08efdc64af1ec,02e5f15c09a80c59,030022d19f4fddc9,0304c3c8bc0d015b,030be60c666fbfd9,033e8e35c2daae74,033f147a02b1cc19,0350430c60179dce,0361e5abf41cb397,037057dc1ad04a47,03729abd0d8faee1,0379c6911e51b01c,038462950d4c13af,038983fca03eb623,03a25d9347950599,03be3526a7fa23bf,03d378dd3ae179a9,03d7d3d03dc0dc8c,03d80ae5f7bfc1fe +G-pr2169 000ba541defec402,000bde90c5116f3f,000d0dac6005a62d,0012e0732cc7c899,0014e1ec04527fe6,001b5651d32a01d7,002781c27e8d8921,0029abb24c3307b9,002d73d625eef08e,004ac58debccce17,00563b17798b588f,005fff7e66bd280b,0067a879de941523,006a036ea9c9e973,00712fb574084e55,0088092ddb4da342,0093f1c0e2bd5811,00a7455da8276266,00be6b09bc963cfb,00c850204ce27fa9,00db041210a59e41,00fd2d2c0ee8abe7,00fe25b144a7262a,0102d89bb89f7284,01079da2a8026edf,011a5c1eff5b35bc,011b85b785921199,011d3936a9b52595,0123f84845bbdb94,012f38b443b765d8,0146f45ae7a6d110,014a6f15accb85f8,014b270e9c339047,015920b6f83fac50,01622dd2ad1faf45,016552f37e1e5de0,0174fca5b67567f5,017781f06054ddc0,017cf6627b91e02d,018078fc86b92397,01884236b8215fd6,019be1298d9a2287,019dbe647a3bd234,01a2700721566110,01ab017b27569c7f,01af6c8586439f1b,01b9962fc8996481,01bc7c2bdce62935,01c08a7dbabf3f89,01e08b50b71b8039,01ea8e8d68a0aa87,01f861483590de7f,021343a6ea87d6e1,02309dc273e40d54,023900e6ba88842b,0243206518b093f8,026423f9f578b995,026866e35415999a,026c7ed468b914e7,0286cac466189999,0287e62c9afd951c,02967f54581e5c9f,0297ab4734835086,0298b544762aa3af +G-pr2184 000ba541defec402,0012e0732cc7c899,001710344c0ad3e9,001eff24456bc60a,001f208236e3475f,0034d7b104f1a7ef,004dda6cdf1dfed4,00563b17798b588f,005e124cf4a85cb6,00645bb63cfc5511,006598090d2a406e,0067a879de941523,00712fb574084e55,0088aabdabf3fd5c,008c20f13553855b,0093f1c0e2bd5811,009f83cfa9f77dd1,00a7455da8276266,00a771a73e6dbe2b,00bbc3436313767f,00cd3eef318b787f,00d14874e49fa85d,00d9db6f034b2cf6,00fd2d2c0ee8abe7,00fe25b144a7262a,011cc1c5e9ce3027,011d3936a9b52595,01311b598d6d1ced,0146f45ae7a6d110,014b270e9c339047,015007dd0efe3630,0160de9142854c11,01730d682c0e7254,019446d5d098db32,01ab017b27569c7f,01cdabe27d2b5625,01dbb9699da50c89,0201197bb41da981,02151f7c5a17e4d6,021c1de2db340c65,0238c5e62c185413,02593a6150f41ac1,027ffc4bb0263c0f,0283d81f7de6f4ef,02967f54581e5c9f,0297ab4734835086,02b7becfe5ffaee2,02b998b1022cca6d,02c4120edbdf48af,02ca433e8dd36f4d,02db450ec489aa76,02dfdf482cb9ed76,02ef7b34b81b39a8,02fac50789eb00a2,03020ae1af3e1995,030fab506d3d364c,03190ff404720f7a,0334cd5db5470521,033528993300105f,033c2c30c5041273,0346f4aa2084422c,034b8fc97456d53b,03598670df4b9e1f,035d0fd3ace7a0e1 +G-pr2216 00137a0cb34ca2fc,002ac1df835ea040,002eb6a9d78c99c5,0036e87ea62217ac,0044c6138f70cb5e,0048a4458d784ac7,004be879c75215d5,0058282761144a02,006c01be90c0a0b4,008760c133e12b51,0097e49a2a9d92ca,009c5cd229521f2d,00a0b128d9c971a4,00a35cc451a2033d,00cc7fdb12b5fada,00d61ef159b8dd96,00e9d0aa786ac533,00ecdd8a6860e958,00f0c897dbf38a36,00f18945a2ace430,010def1185af35c1,010df9c6ac449ee3,010f6d0a3f70b0c3,0119596fcba04b60,012448a941121aa9,0132f52fd2d11d5a,013c0d73a090bcb5,014fb4691f288bfc,015b9b000a39dfbd,016a63753dc963b5,01771148f3c58db6,017876013cae3157,01955a2cbbaf3bf8,01982bed00359a39,01aa1eadfc85f55b,01b6d42160eff137,01c71e365dbdc6f0,01d6290b2372a609,01d8e54f03203584,01e653e7445efae5,01e8fa38fe4bf38f,01ebaa14c11dd664,020119978c5a0735,021ee7ff27ab7086,022255869814c7c8,0249aa32842310a2,0251d48dd86b71b6,0252548392aa90ed,025d5cbb284baae3,0278516366937f42,0287ead6d430991b,028f038c26850436,029480eb12b5aada,02a6dc444a899b9e,02a77862f5e0408e,02ac0ebb78fe890f,02b3218270f65ec4,02b7a2f3c550d6db,02c186c029dd9c1c,02c2bfc2bdc56340,02c6858b41d79a98,02cd923af4488a2f,02e22f814ee353c9,02e68d69e8fd1565 +G-pr2229 00002540eb9026ce,000c297cf92e71d6,00106bd3236a2fe5,00120e7f77b55f99,001a1fbc9a138a4c,001b1cad8722edcd,001e78b2df26067d,0020d421157cfe54,00280c342c986e7f,003cc64d201a42f8,003cd8d8c3803a36,0043009facef2ee8,00469cdc71fe911a,0055e6e16c5eb884,007e0a1b208aab3a,008513cbae00863b,009068a8c62e3c68,00990f87f3885ad0,00a6995db76ab253,00a9b1cbe56acf43,00bf42c56dcfb872,00c850204ce27fa9,00ccaa8fa146dbc1,00d076d4b2fc343c,00d16e819a4041c9,00d2d7d320278965,00d8f644f121e849,00dc88a61f153335,00dd37b26ac9d5f3,00fd2d2c0ee8abe7,0103446f4bb540a1,0103ce90b666fa04,0106996fc591283e,010d93da867eca23,0111a9b734cfa632,0117482d7f71ea3a,011c070896a81563,01230c2329ab3f6c,012c5d9c797b11e6,0141c35274918376,0141d0a0e5905591,014749d3d96ced25,0148d3301eccabb8,014e782aaf4c9b4e,0152daf17bcb26f0,01592eba81c7a218,015f2e210de4114c,016084b58ef7e7d0,016f9dba1a14b617,017feee7078701ff,019aa16285987722,01a026c230d6550e,01bc5d843c619982,01c9ce885c6bdfbc,01dcf425f86f8756,01e1d1cc3f905541,01e34a9d9da32cc6,01e50b1b1c791448,01eab326388a8e4f,01fcffab1e6b47bc,0204562a2f91c614,0206c4dd45356bd7,02113408973947ea,021504bc5fb38ce2 +G-pr2230 00222ba12ab827bc,0036a0180295e93c,004be879c75215d5,005cc3115ec180b1,005d465379d0410f,00942b452733813d,00af97e85b9d2c83,00b4b82d423bdba6,00e9d0aa786ac533,00ebbff2268b748c,0121cf9865f7bda0,0126344a284ce20f,012d0dcb4abebff9,014340b2a68401b7,0151a2f1f623dd3a,0153c1d1a79aedeb,01615d50709c1518,016a823a5b0a1fba,01760e64a910cbc7,018c1112f41a0be5,0193ae115fc71e19,0199343db7e94a3c,01bc0479b140e48d,01c5ac76b7c45dd1,01ceb9f2c86b3e41,01d2022cfbaa68fa,01d2640e6dd9877e,01d32e8786dabb73,01e0aeda4445a156,020e1011da480b76,020f7b5585b1a50c,022289edb698978c,0232eb80cf3aa7ed,025c5611cc468def,027b5909f377f463,02aa68649f187912,02ac0ebb78fe890f,02b011aceac523ac,02b02584c245e68d,02b690662283ed01,02cb053f0c0b820d,02d90c8075429a60,02e22f814ee353c9,02f3d6664c7aa287,0312794e4396d1ad,0347e2deb6e19a68,0348f177ce00e18c,034f8d35cbe1c1db,03544b36fb96ede2,035a1df2490dad97,0360d13f104a1d0a,03729abd0d8faee1,0382aed8803ac19a,038be5cf294d279a,0392be233461382a,0393505ce249d4c5,039e7c52ee17dfa2,03c1283cf609a782,03c4e0712215665b,03c9c0855ff7622e,03e3b9540529a90d,040af0caac37f52d,040dc91427ae4f15,044403d1e1afb172 +G-pr2233 0001baed340e07cb,0009e4d225b13f65,0016e9e4e3696466,0019584c84a2a0f6,001f5178beb8e9eb,0041f5d717ff3815,0052121c4829b9fa,005b60524bb16f89,009a2b3eaea3ad48,009b8f55b25b5a88,009ea1f47028bd94,00e66043e2b77ec0,00f1f13f9ca78868,00f2541239e0f9b2,00f8a37068318db1,010be78c3b459285,0123d0e53996545b,012c5db9a0be9bf4,013d9cf3dc6a13a5,01445c8e915ea2e1,015986870bf6154f,016bf30bee17259a,016cfe3b95b68dd5,017aa514b8eb4e1a,01c2da8f829a37d1,01c9851008b4cd8c,01d2640e6dd9877e,01e94e2fd8da745a,01e9d29c79604209,01ff1153ee854037,021ae074f64d5f81,02336e90dd1d1e31,023fb57ec6bb2136,0254bd88d3647945,027363fdf12300aa,02a341a784f8274b,02ce05ad90ac1834,02dfff0c5eaf7890,02e22f814ee353c9,02e32b00cfacafd0,02f53bb3ea2b42a2,03179417a0b94226,032f73f21896d145,0333192eacf61970,039646af23e3d262,03a428fecd401e5c,03bd8f50be48cccc,03ea6aee04c5de78,03f699ebc6a66462,03fa5ffcf8bf4503,0409a6429fc10920,040b21f59e562eca,042cae861a281c56,04559eab9d24270a,047209239ff45986,0472662c2b6d7836,048f0dfd8a89b49c,0492fb1c6f20b53d,04c508a1a98042d8,04ccc3c234fe1083,04d0feedb4f06cae,04dea125341f1423,04fd43bccc021e03,0505908bd90accc9 +G-pr2266 000bde90c5116f3f,0032633d2121687b,00a5b05e4cc8995d,00c56df8c736f46c,00c850204ce27fa9,00cd4e1d7a4afe39,00cee2218a9238ed,00d2d7d320278965,011b5ee38e2a88a4,014690739d396c3b,01e085b79a600cd8,0249e9e93fc1c184,0279e3d3f5b36235,0284b3abfa88373e,028f038c26850436,02b4116d9c88d527,0301a94c49833305,0392b5b4faeef0d8,03ba61c02a255ad5,045ea274266a502b,047d061807442836,047ddfcc14224df1,04ba53cae59527c1,04d2d4650cdafb59,04e11ec79afcf66d,0544de53b7540a91,062e30f39614efbe,06361e145ce10cc3,06403f0e9d2bd232,06b6f1aafa55a5a9,06dbd074a202dfbe,06f947e4bcb5ccf4,070054dd7c218acd,071fcf0d3bc681f9,07378ae964789983,074410e2ea8e0aa6,075ce8d0a3cb2b50,0794e54f04751775,07d75b8a295aff4d,082348667cb37708,082625093298255f,0866207f2df62a84,0869307d8863ab4b,0899cd553316b94b,08af42cea7c5f1d1,08f2ca387d7babb5,0983b5532aea39cb,09ea041dfe75ef5a,0a5119c57b6a6962,0ae8d59d3d1f57dc,0aedfa23c0fa4ba9,0afe1e55eb03c575,0b74352f943477e1,0b8c6f9e8c97e37c,0baa788d8504ecaf,0bb0bd04c02f3877,0bb672bfc623a98f,0c806d5087be0dd9,0c8851cd1a8a8a9f,0c8856872fe5be6c,0c936914bbc8ee99,0caf9057a7c84aee,0cb97649f03fd55e,0cd7dbcea237ec54 +G-pr2337 001cbc2dc29ff58f,0025bf09107153d8,002ef88da2545c9e,003cc64d201a42f8,004443b123690909,004d9973da3ea075,00518e938c424fe1,0070108479e4ae8f,00be6b09bc963cfb,00d2d0392c797041,012980041ca43c42,013e53c77dbf2701,013eb888c4ad9c95,0153df77a85698ae,01595b5e47f10ab7,017b994545cc2e60,01895e98aaea35cd,01a9a2f08b826f73,01bdce457b6b0bd2,01c039dc63fed42e,01c226e901a1652c,01c30897667039d3,01cbfc983f569a15,01e1d1cc3f905541,020a7603bc1ea4fd,020cd658bfc72255,02100a774a720114,024715fbeedfad2c,026bc82521673df1,027037f5c6f8c9b1,028016725620fb29,0283254cc5482a0f,028608519b5b25fd,028e0dd1fbf80c3f,028f038c26850436,029fc68d5ee51e54,02c1218310e897fe,03215f316508048f,032b90b24991ccbc,0337a2af55915d6d,0350945a42c12c54,035367303d24a148,035b18793c273d20,0365c702e28d4e6f,03bd7bda77e608d9,03cec8ec4431279c,03d49fa6fe1234c7,03df7c96cdb78dd5,03e4b9b892d3739a,03e9ec9eeedc1513,040422aaf68e8241,04145be31315c10e,0416f2bbc663f7f3,0417929bf539263b,04292874c57287e0,043618b3e94d27bc,0441d354f02365bc,04604b245080022e,04ba2483ba3c0e28,04c04b58e3ecde53,04c2a665ab4dbf91,04e7435d87155799,04eae727516db25a,04f68d9e26e2d822 +G-pr2710 00041988c0b4b24b,0007148a30030a92,000ac0aee6be755d,000c6649b962c1f2,001564df1a35f2bf,001696acee569b2e,0017f5e37ed77e06,001a6c66b59c3b5e,001e6c6b597176e5,0020782948785814,00218454f9f6c323,0025165887734175,0029d154339b9107,0031b98ed22c7f6f,0043f324ae09ffa5,00465b78e7dad1b8,004b17e1abca2a9b,004e4ecaba95c53e,00505c2ba1117165,0051deb76f1bfd91,0052920435c3cf44,00543ce8cf33f9db,00547b07916e7c4b,00585b341d6817f5,005c6d3fa4d3a830,0065f8a6c350448a,0068cd0dd1cbd2e0,007301d2a701efaa,00746d8bdc013cf6,0078b3629ff26755,007c01cded3a705e,00801fab5d2b8871,0081eea6bfb2fd20,00822ad14378ec83,00828c7775aa4294,0088034c5aa7e56a,008bd4acf4f5a54c,0093b0ca0c8283b1,009b68aa2d6ce75f,009fa1c0b46b1955,00a14d7b98c805c5,00a1c9aa2028768c,00ac64f8bf0b92a4,00b38cf7ee351d16,00b3d61f695c9f7a,00bd57dd6707ce36,00be6b09bc963cfb,00bf55de30c8593e,00c27e988639d0ea,00c70fe766e60959,00d2d7d320278965,00dcc0d325f76577,00ded932ff15323c,00f367eeffc0d2c4,010122230de41085,01079c8197ca5e33,010b6cba87341ad3,011100e616887b9b,011753ec6d99124b,0118cc07f06594b8,0122bb50e2fbd521,01264b36afe056f6,01280cdf523129f9,0129fc970156d97c +G-pr2746 00023b4501bcf42b,000541eed2b4398b,00075deb13f920a1,000761f47ef724ce,00253941832cbd70,0025cd4c4ee16ace,0030587a8c599acb,003a826d3cd45172,003bad9cd1d594ba,003d281b65dc423a,004c58687d828161,00521971f6976945,0055312928108b89,0056851f62f6bb8f,005743edea4ca1b3,00576c6242680043,00579cab1d0e0d81,0059005fc9b23a88,005e7f7252720ac7,005fb3cd47c70471,0060f1d18612cf28,0066e3e5f7a5f7d0,0069946d2e3ec521,006e7b195df30210,0072478bbad813c8,0074de9f5f9d1330,0074f1af57f8b225,0077b0aea0456785,007b1b3cb86f9336,007c12e2b0bb7acb,007c977a0a045202,007cac525760b5e4,0081a1fe30a2cba6,0081ee3b80265e19,00916b03b841a78e,009206d9b7e054ed,0093baf8ef5cffe2,009554612ab203f8,009c5cd229521f2d,00a31c2904470bf7,00a568114fc79a26,00a7011f69d4a6a4,00af76cf89ef46b4,00b1ad1c4b2293ba,00b2a40644fd6e62,00b38cf7ee351d16,00b4d7cc2b27b2e1,00bb83a1fcff9de8,00bd75f3e885163f,00be6b09bc963cfb,00c35e884ab7e7d0,00c87da7fc2bbad6,00ce3fde08e7e18b,00ce410e679150c2,00cf23d9aa23abc6,00cf60ab4d49e929,00cfa3312b1f1619,00d28091caba1460,00d46d2b69ce2e2c,00db121df8906d20,00e18607bb68d019,00ef116ad7e3e8c6,00efe469022cf718,00f46f22ae42df3b +G-pr741 00d1cb0d38ae0946,0126afdc2edeefbf,02998b50fcc4d9ba,02a1418ef3983f3d,02c8f61259001af5,0553fab83b909ce0,05adcc76dbf7859a,0605af6237d67466,0649ba8f3de22d0b,07577f2f82762d70,075ead371f26531c,0915908871d4a9c3,092ad6fbbb0e1888,094e4de4de240646,0a283691f5f61fcc,0b52f2ea6413e9a5,0b6a83a0f8b54669,0b6ab74bc544dd6b,0b8719d7ab3a834d,0c77802d11f12e5f,0e020d160d0e4a79,0e0bfb1d597a0e00,0fb5d596b96b1f6a,0fec78707059b593,1255784be7eeb3aa,12e18bf84db3be1d,130ab5f5f8c309ed,131ab39b3ee058c1,132ede06eeb05ba3,140fa89d70416009,15d01591eb6a3c16,16ffdccc67e07fb1,17a27a3bde2360ef,18018fa9e8083e70,1808dc496b212cbc,18aac18579cdb03c,1ac8af11e42bd01a,1d81b451d5e7c114,1dae06a87cee41ae,1e808b1a430616d8,1ffdefda3a5b2fab,208f6fd430a20c64,209102eaf05cab7b,2133661df4ef408f,2158d5bed3628352,21b405a15cd9a9bf,224c759397865b33,22a702fbb47dd705,22f01aef734b0e63,236c9ca185932844,23d8dd868541c3e2,25876048fbc31b05,264a6b3a5f6b65e0,270c2f66a63432bb,272b3cfdd3ef096c,2785423398cde2b4,2789bfa206053df4,27d7d93ecded5486,2915eac63c02dff0,29df133656efab6c,2a3237866b6e911c,2a5283c4eb32d91b,2a59f5603264cb63,2b6cc2b4dfd7e51b +G-pr753 000c199894f2bb7a,002642836c691b80,002ccdb4ee1778c8,004719f268d443b6,00737712c96d00aa,00a81d57ce9b63a6,00af6c049db4b78f,00bc1f43b06d325a,01416acb47d066db,0147371cb5d82828,014751bd5ddb3bbe,0147d03ec048d48f,023023138970751f,023b82e4eef40757,0244b728716b3d21,0256fcfa28d5f54c,028cf8f92d4a8068,02a9d95e243223f9,02c898b7f682fe9e,02e783c47157d025,02f85eb09273dd50,033314e36c19d66a,0343e5bdcb55d760,0377d464514d4053,03910300a486be29,03d0ea5e74d728af,03e62f695db84d61,03eb49689c8983ed,0422a5f20897aa7b,043ffc2e60852e49,044ab519cb16f1b1,044d1fa3bb2a465e,045a162a4e227e3c,045d0f65a9d5f94f,048babc1cb67a1bf,04b8a2ebeced6e81,04b90735343eb84d,04ba2483ba3c0e28,04e4843a8b0863d3,04e6d464d5e30df3,04e7435d87155799,04f68d9e26e2d822,0554e9fede8f2da5,0564967bc112ab84,05677a292633ae65,06065cb036f13c7d,060ade7c8491f860,0619a5e88e5f7202,06391152f51d3b7a,0646105bfd5d7b38,06972c297c42a2db,06feaed4375082cf,0710f0863c847a19,074d1847336b7105,075cf52aa5e64161,079674fc372a6c8b,07972bf9fcaec847,07c36a1d578bc2cf,07e3d4ff49507eb0,07fdfbfaa4c9b661,083ae7ab13403e0c,0891d17d3bafad72,08f1b41ba719601a,091d254637b66e5c +G-pr888 0003aa867f5710dd,0012411dabcfbff5,0015e4101fd88ffa,0016fed4ea0da30a,001cbc2dc29ff58f,001e0cf4f12b6d27,00261e77b36f1ed3,00261ec95457925a,0033e13503eca165,00367609e43d03be,0038b25a36f3c1ef,003948b4d5404640,003b1f9422a92467,003c465251da2515,0041d32e9a6e0cde,00422d623d43119f,00462cb3241138db,004942d59f2880b9,004be879c75215d5,00508f2e29a35612,00530f29012b1b66,0053afb4bc8304b0,00598b9b208b2ab2,005d1c65a5532f06,0063e375a3f37880,00653dfdb5f2723b,0076730a753b06c3,0076c206644b288c,007ae69852c04d36,007dc40f21ac57f5,0084789340ef6784,008604b34f547983,008c1e138c7391b7,008d8e26d43cad23,0092a4d63375407f,0095a65cebe8f7a3,0099f60f595fc1c2,00a10e2ea466a7de,00a82b6e69075c00,00b5b00c9155e509,00bfa328a824ca0e,00c5f9d0c21a53f5,00c7469658cfc327,00c814318b7f6752,00ca1dda497c6b5a,00cb5c4cf8291048,00cb60562390fdaf,00cfa3c068954758,00d20d94369ee46d,00d5aabbced06d1d,00d839d5ae71b7e9,00d8d62cebbc31b0,00d9459692c16c61,00dc025b929b1bef,00e2ed709bd764bf,00e438ae92ec178c,00e49dcd1e383cf4,00e6aba249c8ef7e,00e749795d1c3d59,00e83cdb7888bec9,00ea421764c9b82c,00eb743a47cea0d2,00eebab7ef8039c8,00ef116ad7e3e8c6 +G-pr940 000dd25e8edfc41c,003adaec4ca9b6c7,005529b2c454e164,005640bb5a7ee93d,0061ab4b59607fd9,006b9d8b9ad6e438,0076c206644b288c,008c460f80d3c629,009dc2203d1ab24d,009ea1f47028bd94,00a0808c813e2189,00a97115cec7a6da,00cd24e9ebfc7060,0121c81cc8286a65,012673ff104a4cbd,014db5f21c696ab2,0155a98f6164d6cc,0183592621babd69,0190b8bcd3684808,019a3ffdac5699d4,01d29a5f04436797,020db6d2f1d9afa4,0248350d9fcbd989,024dea284afac223,0250a1b5fc853224,026cd936c7d7b610,027058cf542af58f,028cf8f92d4a8068,02ac0ebb78fe890f,02e22f814ee353c9,031a6d7ff8f6b1b7,0397f731b495ad7f,03dbbb82df412825,0428838de537a67b,043d6d9611f92e61,04821d5a3ad6cc34,04dd7c0c836318d6,04ddba57eab257ad,04e175a288a2cc33,04e7435d87155799,04e938981f04a7a3,051fb18a1e7c2028,053551963e90e92e,053a45f6f81127d3,053e5b0ff13c2cb5,055e819657f3b804,058f0a14aef6170d,0630cc760b6cf6d3,06376eaae0d0c5d9,06522a7348af72b5,065eb444624dd4c3,0661259c93bf2262,067f07ecf063d261,06af808f3928207b,06ca837bf1fb9709,06e2529c71938804,070de833fe2ac895,0748ba87a3d912b8,075a9b3fb9dd68f5,076556b12da04ea6,07848cdf3720b4b7,07c7639dc7752811,08297316c9ee2146,0871675e72c173e4 +G-pr941 0018e7e8b46f516d,002291121c0254b5,003adaec4ca9b6c7,006b9d8b9ad6e438,007da1c25b11a274,0084e9ba72f99698,008ed1356d99c790,008f0bf20e57e4b3,009dc2203d1ab24d,00a97115cec7a6da,012673ff104a4cbd,0126c2f22f38fea6,016f05a1b67ad3c0,019fceddd4d93f84,01a71b06415382b8,01ab67b1b6efac3e,01b9b8c03954852c,01ea010f57512e5c,021f739cbea02f4f,023ad2f369c84f93,024e7a772f02dba8,024f026c490bd924,02564b8cee44008c,026cd936c7d7b610,027058cf542af58f,028cf8f92d4a8068,028f038c26850436,02ac0ebb78fe890f,02e22f814ee353c9,0310e279a66c88ba,03162f118ea5e0b2,034b8fc97456d53b,036e4b87e4bc4b84,0387287728fe812d,03c1283cf609a782,03c36e5a47031abd,03d880ff9376d033,03ec2a28e07ee84b,0407e4ccf8f129c0,0413e9eb7910e1cc,0439453ad335734d,043d6d9611f92e61,0456e1961ee030e8,04821d5a3ad6cc34,0484357646190e64,04b34275d653eccb,04ce1456a095a07e,04d9358dc067c536,04dbf534aba494d4,04dd7c0c836318d6,051fb18a1e7c2028,052b6c71ceb41eb9,05440d4e8a78408e,055c17f433e5cab3,05785c6dedd898eb,05f01203e7adc331,05f796e031f0b8fb,0630cc760b6cf6d3,06376eaae0d0c5d9,0641a56dd48435be,06676011a595a43f,06e2529c71938804,0738b61c6114bfd4,075a9b3fb9dd68f5 +G-pr953 0003ac4e2092ae65,002b1ab86cdbd954,003284d45b7be55a,0033e13503eca165,0056859d6e8f315b,005a019bb881f943,0076c206644b288c,00772a1a7334c28b,0085af65153836e6,008f6e514c37947b,0091198df1d9466a,00adc124bfc080ed,00c7a6f5c43e6f5c,00d1fb34e56dd0e2,00df91b80fa8466a,00f2bce2d5c2da5b,00fd2d2c0ee8abe7,010eecf92eebf90f,011480f318a05ba8,0132ad9ff8a4dc15,014981f5e1e2aad6,01501762400f99cb,0176db30afa3b866,01860b84e713aa12,019d1cd84e8bb774,01aebd3e1255909a,01c40cef383efba8,01d22f8ca3cf76ec,01d2640e6dd9877e,020b8a1215db48d3,022380920f651a25,022bf1afa98b180a,02448bf3c95ea095,024bf04e8a3e6074,025047920c1f5751,025633a60e1af3b8,025b6052b9f3229f,029f8dae82b74443,02a3d1a2c0400e97,02ac0ebb78fe890f,02b3218270f65ec4,02b3b46766f0acef,02cef6d2e75b04a3,02db7f3682248eb6,02e22f814ee353c9,030607e01536a247,031158dc91f8d3eb,03269ba0bc5f1ed5,033941c08ccc5d8f,034b8fc97456d53b,034d6541c51de8ed,03573c38d9939ef7,0371540852a6c0ac,037705c75d886a2e,03b40d17950c39df,03c33da6a4fe4f0e,03d8ad0fc2c1cfaa,03e6ac5e9531696e,041326b6a98f578d,0421bafb247d45c3,0422a5f20897aa7b,0431e532fbf494e7,04524f10e017664a,045293822d12fcb0 +P-033caf9c30ca 0031360a9a9d04f6,0144c9bf49d75bbf,01c14e1b6b2ca7f9,02497922355453cb,0324bbb6a45e9b2b,043661bbd931bed6,04e678afeba2b625,065ef25de1187f68,08fee363b4f95e91,09a42091df735cc1,09d9e872684b6e71,09ec91c7cf5d52b4,0baaec80cb1c2388,0be80348821f2a0a,1034cc58df3848b6,126549f9d83af951,153a12c4953f24ed,184c01f31e7bb76a,1853967d1168436c,1970bd75e8ad06ab,1a5a31d97d45d6e4,1a82f2181027ab2d,1b5d31865a070afb,1b6b250b0bc15300,1cdb12f7b00b59ae,1e9640a8c1a49f5e,1f2aabe8118f3c37,239f1c8c27febaa9,23fe0c6f0f430246,26c43089efb46e7f,27cbad30b145967d,29975e6fbc407ae3,29ad2c618ef2a6cd,29fb953d7beca8b1,2a3b58565d7e43ea,2af827d6ed05e133,2b194f34e96561fb,2f6c3cd8ef6dce4f,30ac3f5b76fcc717,30c9746c66046836,31c6519255130336,330a16d52f793f56,33431db26a57a68e,334b25ac7f1738c8,34003d36ab81da93,34acf8ddc6c8a8de,34fc62d6b79ec247,3590235ff40b3abe,35df1d441929111c,361f3c9181628f57,3714b3485056532e,372e544b03438e93,373fbd8f1ff47a9b,37a114ca03d8fb5f,37d4742f19e72de5,38898bb10cb6bc2a,38ab0be78545b1ed,3a7574b67b5121ab,3ab9dc1cbc565d3c,3b206b8d7dfb3070,3b7a504fb9d41baf,3d7181d14a8ccf7e,3e1daed23f39f0ff,3e548669dcaa6fb9 +P-06a15fbc27d1 0219a4b8744bc9ec,02d44b0ebc1b16de,05197755806b3c03,09c80db4f33a8a81,0a8300fe1a550cf4,0efd00ddf0a37ec4,0f16320d7849fc7a,0fcb2cc4036cc1e0,119ec154a0f674cb,124f405783787e3e,142d9bf0e52a3734,17fb8a73afa6cf51,18a94efacc4762a4,1dbbe4c2e7ec40d9,1ebfc26a6b6d10a7,218b4029a4a1292f,2240719ecf195028,256cdce2494185c2,2678cd85fd08ffe1,28b2357c6d2bcec8,2ac2c02b62eaadd7,2c4a915d70f933dd,2d67e1cf1e1161eb,2e7a5c9414c6cf06,30121b3a3be97a18,318d84985251d47e,31c183568e70f8f4,33c6e0efbc21520d,375c8cb343fcabe1,393ec2f3a2adf146,3aeea34d73235e0d,3bcb22cce6d97435,3ca10af6d2a4fa22,3cd0628ee1f2efe2,3eb3399bc4ebe96b,48936fa135bf1acc,49672849d06e2915,4e9033b73abab65e,5211658a4ab4fad6,54f905e8795e74ff,5698ce18749d56f1,5804e5f55815c3d4,5cadd6ba65dc89f3,63406dc67cb9c311,6567c386acfc4a45,657594f31be42aa1,6b9bb7855e7aa4e4,6c2f07b806495e8a,6d10708bc1ec15dd,6d436cb439019b74,6e6b27052e65bf05,6f3fb02f02333d64,6f7ca05ab8107092,734e44a0dc1d444a,744b0b684ac268bd,7a32ae52cb1c18b4,7abb9a493c8c753e,7e323aea4f00c428,8139b5e8e473e851,836f95f2282f615d,85fafaa7235439b2,8d44cfc5bbe8e1ba,8e598ed3577b6045,8e5d87cdbe1ef11e +P-0afb0412cc4c 002a3e446a06c6a0,00a639ebe59c8534,00b366f2f06ad5b4,0260830850ce2bad,0272c42ddc3f279d,061de0caea99ed6c,0742c588b37f8e1e,07cf1486e9cd2c93,0c614cec5fb20d06,0c8a94cebddb3faa,0cf12a583410311a,0f00dc50d5526ba7,161a904c672262d6,166fd9bad49f7bb6,173dce8f726d9735,19b58c88fc0fb627,1a38e4b44d433a39,1a75432312137ea6,1bdde652a2cd8bba,1e63a1e0586edb7b,1edda565dfefbd64,1edea0db3b60822a,1ff97ae5a27ec16b,21b38f1ae2fb1be2,21bad5ef69cb04b8,22eac16b4423f8eb,23567820cd06ac2e,249d65af2d59747e,271add331bcb5d22,27699c6105e0a5df,2c4b410aac6031a5,2d4e67b569f5741e,2dc49b02028aa9e9,2f3f120f3dae3ef4,30986557857fca9d,3128a1bc086a4594,33d49a69e07163b7,3478f7a2db6bd187,35fdabfd43b988e9,37928884236e4719,37a8b4e7b91090cd,38676a07bf9015e5,3983a9a056912c51,39dc3a57d7fd7b47,3a2d98a5c8769334,40b5c9bb11b05b04,4169823207c479df,431703a855858e32,45bbe4f84bc44f82,476861b6f9ad6115,48c9fee98149edb9,4b3eafcc9b885158,4b3f1e094d38c7b8,4b469c481529ec8e,4d10196b97a1205e,4e20a533ee867fa3,4f61a3d8802f4c34,503d6201ed19f681,52bf692f007af397,52d57210e7239018,53146453c100986a,54e72e24a136354c,59cbd451c7e3be22,5dafab4889d8db13 +P-0cfe3b858c03 01714d45f84edae7,028c8d3f3f7df3d5,0726d577357449f7,0846c7c2751b1876,086ac1154cbb5f7c,08be6eb51dda90b5,08d8f00b788ce0ef,0a6d9ffb1d047315,1192ff8a895f94c8,17e667a5bed482c7,1861b4c893ca24d8,18665c98aac9c04f,18f732d41d7d9aa9,1990780d7f3210e6,1a23811aa495eb0b,1b5597720d45ef0a,1c3b433289d9028b,1c7545ab6bc682fd,1d9d5b4a2ac0f04c,21374ad14b9cd922,219b6945621b8dc2,227b0dd7030c4592,23cd1540336678b5,23de1194e959c2d6,26688c2f8f6f13ed,26ae1467e2bed124,26bef196f28359dc,2890ea48d0faade7,2954dc8ca2753b09,2d298d19ae812ebd,2e50d302ede61715,2fa6300993acee87,3073dc8dcc19a5e1,3245b1aac6eba79a,3359a9420142f0ff,3593c13275673cee,38085f93de45b59b,3a0ab5bb55f20aa7,3d03f27bc9261d45,3efe7b822febe969,3f71709065c3ba42,3fa966904100c21d,42c1a1a221724422,4324e31e3e3f900e,43807f23e0b9eb08,445fa6a8608c29bd,459c1b83c95245be,49084c87c1dbcb93,49b2db4ce2ea085c,4b151a2b9995e03f,4c37532e8e362396,4cf13db73871dbb1,4d2f36b548f9fa48,4df68b08a93a4e50,4f11022c15fd9371,4f3b9e0f91440d61,5248a9b9d46b9ce9,54710ee8b620b006,580ae607064300dc,5850981f94312847,59201dba352961c1,5a1673378a860795,5f5c34653df91f29,5f7b68e8c74d1353 +P-101aa47005a0 03e5caf585baf7e5,0497bf133efe84ee,0628895c43d59c2e,07e20dedef07b0dd,08ea7555e5a85e02,0d12facfe54afd79,0d6d6770891d894e,0dbe522f4006efad,0f034b7d5489a6ab,0fe7f7b15e44de2c,11b863068e7770bc,14863487db206c6e,15cb808fea8e967d,166cb7bbf1dc3ced,1728bb0dedae5efc,1770187554eeb95d,17b719760ad123a5,1bd6486e3dc3357f,1e2508a706332521,20061711ba219367,203f643db1ea374a,2047de7349b972ef,2064355e9d4ea205,22c2d214007aa2b7,23cbaa5ae23982a2,24c169b8a3096ac8,25d43d72f0336953,25f2afec3f9fb186,261669f5651a9315,27b32f8d5d71c393,27d8b5cf448e1017,27f73f18949a8ed1,2802ddfd14b49319,2905c9bd05baa134,2bc099ec6c573fca,2ec3f234568018c5,2f2412df19e08387,3001ab4db497865e,30b5681595dce44c,3152ffde1c8232b6,3233b72b4c8aabd7,33a63fa786a5449e,380ba02bfd8b565b,397f8b94c0a6d6de,3b46babc1e3da07a,3c02e2ddba29e3a2,445317f50bbfa594,4656c5ced2ea5693,4a048c58b36d1e52,4bbc9fc7973603c9,4c9894ca2e4799e3,4dfbb29cee537120,4f5e9078f23f7e1a,52673d72c66bf53e,527490824704ca96,52cd2f7c8e10672d,53f205b6a1c8584a,54808939abaf54bf,563f05c98ab78826,5b58cb0d7e6b982f,60e05b34b9e3f2cc,612b01d3aeb11665,6507d05eb1d4f3ef,653362afa38f7a56 +P-1486e2af4c6e 00d766503d58137a,037cd5dabe7d5470,07cd45b638fa3481,0c0105ca8e529d7f,0e2be6d4ba650484,10a78eade99fdfc8,19b27632d2def6a9,1bcda4a40d4f29f3,1d6fcc2bb869cd7b,1e2b532ac0b897ad,22eb1739bcdfe4c1,24c927fd30ddee54,25fa771d8ed8e802,26c11c4eeaa4bc28,29cf4908b74a8904,2b091acdce7e12e4,2f702bef6b4c1914,30ab0ae196fe7663,313f205d5a334e7a,321d78aa4cc54b52,3724bfc8dbee6374,39725db619ba581b,3c76429526cf16a0,3df706b21b75479b,42e675a4d64dc798,438fe407cafda3e6,43e60d64e2d37e3f,4770998e21c72620,498dbd8f7b236e04,4bc6a0a476d57d17,4c4eef32bd59c85f,5018a765a98300c6,5348b5c21cc0120e,5b8440eee0885a1f,61535ec1312aa670,62fee4e0aac45071,6c63122547efde01,6cff3e6389e1c5f9,6d992994ed16ca33,6dd1950385ae4928,6e1d85ff5c486ab0,6f871e8c39f6b465,75feff0874f4598f,7691296cd8709507,780ef27e29745000,7c74a94f9328e566,7cbc7a0db6ea02f4,7e4d3e2614122cd2,7eed5d395969ed16,837bce00884209c4,84d264380b7b9815,8ae8c6213d935f95,8b6f9b5584b42e84,8e114e1350aff427,958e038d5d73ae6e,98f430dd786b524f,9a0edf86dcbdaf21,9cafb343c3f942bb,a10ea1492cb3592d,a4c837bfafe20ac7,a5e89ee19d3d0a98,aabbccc85cad32b1,ab38e95f0e66acc3,ade63539141f7003 +P-16a3b1d9bab2 00cc2b8626be8094,01d5b77a2e9f8890,01fc03fcc3f56703,0260830850ce2bad,05a1826a62e87fe2,0742c588b37f8e1e,075269c7b4f0b592,07dd0ee9e1d02e5a,0c497fd7b0c83fc8,0e40f7fb519917f7,1133d2035f4c4564,134922756e9c6a23,13bbc369520bd4f5,167fdd471b37b4a4,1d0079a8b8a1148f,1e138da228a4522f,1ee03840eb41681f,206e65ff95404dba,249d65af2d59747e,257cda98d46b9c69,25df20678093632b,26c7d5e609c1e5f6,27a24b2ab2981bcb,28ae323c71fa7541,2a172c61ae9ddf0a,2a59f5603264cb63,2e942542c3f26424,2f8dec400daa1b21,32d81ff1f104eafb,33f6129fb00e025f,35c942b0e5f3ff1e,370729f1cff5782e,3752232e4ecc216a,37fdc29d51e0a1ee,38c9652dbd4964d1,3a1d8a4d52d9b1f0,3c9135b9233632e2,3ce3083753cf0cd3,3e3fdb912f8b9c63,40d275fb3449cbae,46d4582054848e0f,4ad25fbdf974aa99,4b3f1e094d38c7b8,4ec32b29519fdff0,520b1571278f5c32,52b464104e269b65,5336a0102628e08c,5372624cf0b645f8,5474d9c739bc8bf8,54e72e24a136354c,55095af65ad67f97,5536ce1c8cdfeb3b,59f92ba2f3e0961e,5acf3cce7b9830ba,5e94b0acb7b93911,60e4669578cc05a3,618475896e255495,6234ccbea1a7a596,627fbc03abd0d5be,65098b49ec8fefa0,67ac2e1b970b44b0,6c417b7020cb71ee,6d386708a7b8ff3b,6f85de94ff13a5fa +P-19fde4fa1f1f 002cbb634d382921,0046623124216c9c,00bd39e7d09a74c5,0186b2dfd5a1840b,0297bc4c3afa6a65,02dad0ad7d082b92,036827d07b58a3b2,04c05dc8f0939cfb,05443ce1a0c403db,0625c4eae186ffc9,078dfe16313bb74f,07c4269ee8bf4ed7,0892abd706f00e3b,08c448b835f14c7a,09b6f2d6113925ec,0a0ba0bfc1ccfe27,0a90704628f6c859,0aa649006e2bb6e8,0af9c9950fccdd12,0ee3be0e2e51164c,0f5369d02eddb1ca,0f968cabc224a8dc,106f8222132051b7,10835ef11fcd5a13,119670699e53f82c,1224e792cdda99f7,12858998987d18ef,128be9753bce3e7e,12afd6b7c5bd07c6,12c5b5d0ff9d6dda,13c404a77226d935,142a7e3d60a02716,144596fa27fb655b,14761fedc82129cb,1599b7fa198a5d09,1735c06d6be5fb36,17a442a91ad822ed,17ec64e079e1012b,186cf1fbe0a0025b,187750c97b69dd5b,1890d155c1fd0e96,19b50f6b0b4a4934,19b9e54be3b944cb,1a0b2bc0a43c8b58,1a6dee2001184d7d,1b29c9accbcd1e30,1bcc15b63fe01560,1cb95981413c868a,1cf5e035d93d11a1,1cf6d4f00899f2cb,1da5910c1be592c1,1e420d10338a5c63,1ec434f081377fe0,1fad4fea668ee2fd,20a2c603b1c31ab1,20e2ded169c35640,227539828427a87f,2745fb7a83e0f897,2799581c044389f3,2acdb9c66ee90f94,2af41c6501c19c36,2b79f8e7f87c4d3e,2bceb9834b712ff4,2cf735a4696a5d16 +P-1e8aa7eff08c 001bb9230f4b66ab,002a24192d9d1337,0064b3d5e0022b00,016878fa6d13a479,0181ea8fab19e87f,02895c00ab8471d5,0314bcb603970e05,032396e944b5a371,039a28f1fc465570,03ff3365a44b29ad,04746009a150208f,04b5e7af148ccd00,05030521a81c1bd4,05647f802fd8b0fb,05b1575267bdb28c,060ade7c8491f860,062733b2edba1e91,06413444c8da5691,06479e96b48491b0,065697412c69c5a9,0666bdff2b4199a5,066b0dfce8f4921d,0673538af7cf6dfa,06755e5685a5f4e1,072bb7315a41c4b9,076e65da7e47d8ff,07d01f880a60df72,08eae0ed37645b31,090ca0a9ba7dc265,09650c81711a06f3,099d1855ff23ae19,09ae8d5d472d6708,0a10a8f226acaf12,0ab7283f28d89ba6,0ab86fee0e86f198,0b959d1173c2bad0,0ba06deb2ed5778b,0c58c71690cb1073,0c6178152a731ea0,0c894cd622f76317,0cd93897a93d432d,0d91c001f4f58da3,0e4d67c0ba51a592,0e708530a2e5e936,0e79eae83fda2c80,0fbb55db0498bc49,0fc49d9baa7df049,0fcc99139d3bd9be,106f28706b0dbac5,10f23c2840f9c3c9,119d82db87824429,11de5e4849733d05,122df9eb5b804a29,14e649910862ab5c,155d7d114524aa81,160d4fad6054af19,162747fbc4ad8420,16309b282e32b815,167d0aeee3366078,1684e9c513c1c3b8,172e30b6c43037d8,1896fe89c7a1f13c,18d57c7a7fbb8492,191e85f11f960e35 +P-2f194b384bba 01323d4713ad31f0,0160b5079678e06b,03aceb90e3e74e88,051702c4fafa7f7d,0b94aeca2b5bc0d4,0bf39218346c3e82,0bfa54024e077b1c,0c079c92d741f075,0d91c001f4f58da3,0ee2e07e78323597,0f018b2d079a6a4a,112240fcc4aba239,117a485bc3858c8b,11e85243eb6c01fd,1292e28dd3432e19,131c1c1ff008a070,145e7f375d1590d2,149b6192a1e38cfc,14b9d9639665498a,1821a9eb20f4383d,19b1701e580154a1,1a5f92c6f0d35349,1c33d0339cc0904c,1c844c520dd7609f,1cdc7778fb21c62e,1eef643945d30e7b,23a23ab123bcad9f,26a85819c19d0abb,26e70e12486b2b08,2853c8783304bdca,2cbe998658e4e095,2cc9aa8f35a0bd53,2ce06473e7302dba,2f70edddeb2a6582,2f7e0f9c831f5aa7,30f9a1bcb0b3a18c,32238e3b12656414,34048c07b6f291c9,34181b450cb7a0e3,3937efc9dec95101,3957319237b3e3eb,3aa7fee6159c43d4,3bd6766dae4a92e3,3c847b319938885a,40bcc3082232c2f1,4104cbee8ccff71f,4199a5230c303e18,41cf645d8eeffc2c,424b5e1ef2cdee5f,42ad5e915284dfe6,42cb26eed468e6bb,4300a2225091b398,4313829df35c602d,4572fbc37a21975f,46303ee45192d1e9,4660468547983cf7,47e0151ab1b4ab49,4a00f6716de7f469,4b4745d8b656fb58,4cf3f2d8ad4ea2ed,4d890d943ea6aa40,4eb810e68e9c2c62,500ac9ada0f961dd,50999a0876a2852b +P-32457b5e3477 0044eb77685bdfeb,0068d1094c4bbe06,009c13b9fb220f21,0131f98cb3f2973b,016d1d5f1f85e4bb,043619a433ee4e4b,04db2c0565797bdc,04e38bbbb529599e,06b756356f509dce,07a5d18d75c9c2ab,08cc36975a1919fe,09304503038f384f,0bcc56263b4d3112,0c3bfc742e2001ed,0cc42ce1495481ed,0e5d4f83ede70455,0ece6dd69cf980b5,11639fda322f796d,12a3272ec4506ed3,1a23b1c5a3bd9bd5,1a2fdeb6db53bd23,1aef3a083549ca2d,1cbc5c71f9c04e3e,1dfce4e0040e6493,201b91b53e06c758,2313bef11dc4e224,25ca104dd3bd4f6d,262ac400790ee040,26bab762488aaa9a,2774a77dd72c19c2,27a9629ea3615f98,281a7bc6e619f3a6,29f69fc25d48f91b,2af63e44159a2f4c,2b1b01ef05465a83,2d47eb071d39c125,31a1329696c58316,3367c88a2c15326c,37a8b86c100e954f,386d2ad885004d91,3929aff31dc47e8b,39e912dacb4fbdb6,3ae9162735e2f3b1,3bbd057a84f849b2,3cd790a8e15ae847,40c406a656df8e93,42f8c61fbb8b3b0a,466b1c2c4c611fce,472dd492426b88be,478de2af90b07985,4a3c7861d7b26eb4,4b158bb9d221d4a9,4ec4f8564bf653c3,50858b72ebacb84d,51e155b0b3f8e2b4,53d1f235c232ebc8,54e72e24a136354c,59045be7ded00b63,5f57360bb99198e0,609e271475cd61a1,60dbbef73227241f,6107850ddfead303,616340277f3559fb,6818f733330f4b9f +P-344775d57610 02a221e54083e80f,0c974fcb247849da,0da89386c153688c,0f828021e6cc6b17,1094e49712511c73,15179a10bcd181d3,15d7e3e9292bed4b,17542c6c357de893,197bcb150953fcf9,1bb64b1eb1f56a14,1c1560a569a9a594,1c60916c2d49a2d6,1d137946367dc770,1e20d9dd2a388e6e,1f16fbd451c57ad9,2489af326514eff2,252df6726a5f83d9,253315362f2d2c80,26ac98ac053f2e1f,27e5004f7af8dfb3,2963fd7960abad12,2eec78ed24f24c90,3046aec4db9516e5,3061de40de31e272,3321bc714cc69303,34e5a83cfff242b0,3599050905afbf17,3725d898c3bb5d28,397f448b15efd7d1,3a2269b784b2f8e3,3a7ed6f2f31969e6,3f9c7a5468c67798,405c142f5747338f,409e583aaeeca829,42bc5ce51eacdc30,4e90ad0eea73df87,4efa1979243f8531,4f64c4dfe8d0f6d3,4fe923af2ac6ae08,5eb695412a7fb7d3,60179c72fb08d269,60b3c8a33394973d,627dcd359f587bd2,663b5cde17d8a6b2,6729c2127b07a0d8,6883eee514d121af,6a9d8a02ba7e7f41,6c9d307df6d1aaf2,6e30a063b6a6e30e,6e89bd72e3dd41bc,6eb8844fafd71750,7023558f450f96cc,702fc351aa456024,7090cd741d019ba2,739e933615d87334,747ba8045e569717,7605c21d09429ecc,767ecefd2d996983,771fbb806b335449,840bfadfddfb0447,866edd7a85a4bc78,90a7c38cff015690,911a80fbb341898f,915cbb040a4751eb +P-3b35f5a6934a 01015dc44f66ddd9,04790bc8de27db3e,054ade0a2341d529,06a43da00e557574,078dfe16313bb74f,0bb842bec0687d47,0bd41e3573ef56a6,0d340feda9119e7b,10e550809a0e141e,150975814aef6768,1772d1f9ffc8aadf,2094c8cbe1860526,213644c32d9f476a,2176ea3034f8f533,255f2324040afe9b,25d0a9247159fb29,2857f61a24ec017f,2a27ff7d653e4bfd,2b15e481204ac82c,2b9dc7b643cd4829,2bd5add2988c8520,2c78de0f9310b2ab,2cdc8f8fb8652c93,2d136fcd70103af8,2f24facb14dff51b,309b7c46b4267301,30e6b111d31ad9d1,30ea55652567d87a,31adf1eaf2374f35,33636748c55421e6,33ad21d4ce075bca,34bc85207ba4ae2f,34ef04a33ad5085e,3ada3ee3b6a46303,43dbd0b2c5bd48eb,48c062dcb7fc4474,4a5c23fc890fa969,4d1c33d8aee9b595,501e3a2abf88d4b8,516b785e7f0b7c64,539459d9a57bdb10,54364ee07ac3eecb,56439fead947d634,57534617133eeb3e,5b9c6b38b053f194,5cb2a57d35730c54,5d80cf2ca09f8f43,605d17f0eab1ac64,64653f85cf4faf33,66f991fe1f90d00b,6835dda703c1ff2d,6858a5520c24abee,6a8cc28c202dad5d,7047daa478d15588,70482ee1df7ea96a,70aac3dfc48339bd,767e26e0c7d46658,776d6fc6df07679b,7a04afa641367766,7aea878c2cd37a2a,7b36874a308d7578,821608760e54fdcb,85671fc0004fdb66,86cd22aa9080a4d4 +P-46e545d0621f 00118cac18ee991c,0b4b16713ce97d37,0e2164342fdcc1d5,10374532d61c8b13,1065ec3b043d9b70,1097072987be7950,112240fcc4aba239,11b77016a1272a00,12104448d3045c3a,1416cab495a8becf,149ecc82dffcb68e,1565fab5663c5de0,158cfc15ba372385,16da077b95cd3495,176c3bf75d07d5e6,1773b1f7d29f793f,17a2086ac76a2c1e,1c8ccc8892369562,1e63a1e0586edb7b,2021b6d2b99609db,21054130de4654f1,2290de3468ece873,27affb564f4e8d1e,2ce208b3ea459ce9,2e34b5039d77ab43,2f9d4731e700e685,303d93cebde360e9,31e11d7fa8ee8071,32bb4b7ab857414c,374bf8ea8a259cee,37a8b4e7b91090cd,38d2d46f4890f7cf,3a19c30deef84e15,3b0aaafff52c0e0f,3baa1dd2799a6a75,3e590f855c89ae72,3f96de1b18fdbe33,4084d954e0d5c5c0,40a1b23f9121321c,41cf645d8eeffc2c,41e650e26fb950d0,42250611e0d3e497,44ee6e415f6214da,4621dae25129038e,4690505c545a6472,472852748c73c40a,4863a9b45d8961bd,4ab6fae20b03f335,4db6ec771ae740e2,515f7c7dcc386c48,5400b54c0f958c76,546f8bbf44138e23,5536ce1c8cdfeb3b,5578d66d4b8a353b,5645aed7dc76b7c7,572b18cfb3acede5,589e3687e4f1b0d7,5a888650680aa032,5c48cab11c236dfa,5ce41f88b8eebf93,5d0b13a6c5ed4e13,5d95c79903a27910,612e0eb65c1e8bf9,61d42d60068c5756 +P-4cd71741f944 01980fac4b78be01,02fc555ff0ad31dc,031bb90f7158ed9c,0434d78219626721,043c17833444edf5,04a0d4e4660bcd60,04ac2e3f062a0071,058e099058d99433,060382af472256b3,06535bd65f4dd4f2,08fee363b4f95e91,0b17d7ea7bf62e86,0c2ed5d7c408d602,0c89174c1f13d87e,0cd395258832c34a,0cdac836bbe7b775,0d0df1d9b1bbfc54,0d0ed0c3d2594d72,0da7cc7734fc0a86,0e710aad3bd146ff,0f3c3aeab1415868,0f3ea8a6dbb318a6,0f5236c6a61df896,0fabd72598cbfbbd,129fb77bab30c232,12bdb354aab96ee6,139534ec8210626f,13daccfce055fb28,13f1655517b2eaf7,14ed44dc136b4f15,1672d51685f0e713,168c8bb95acf0a9a,1837e8fdb40265e1,18a422fafd02e5d8,18a8a59654eebc76,18cdfbaa1158556d,19b813961bc5b673,1b9e5d8ec0617887,1bd758397d56919a,1bda86eced303c02,1cdcf985b5463c10,1fcda79125c8cfe5,203fe8f4e6fe5a68,225a1c5e826b50cf,227e3830b75ca214,22a00e5f9d6e8f11,2304500cc8e20bbf,23b3754ec39fee74,241bedd4dbb7a033,24aea8ad322234e9,24b12cfe09dd517c,24bac76f1d183a92,2509cf68be266003,25757fbd848e7584,29854a0a4cc56319,29a2ed6ce9533ebc,29f55f29efdd405e,2a6f3d92919f0c45,2b17d1d83152ffe1,2cd2ba0f73c3149b,2d058376c124d6b9,2fa01b37b0c89309,3042fd9382e5ac0d,30d3d99650a58094 +P-50edd5a3123c 00c37d6836e782f9,01beaee7e4ded128,01ff0ed31b82bfa3,027be78db6c16c5a,02d2840cfbf74e13,0330e3796708ea93,04a8c77a71be1ef0,0bc84d4b31ad35cd,17084ec180f03a38,1b3863cffb374457,1fbefc05300bc4fd,228fff8b3d3de3a0,22a0829cbbee64c0,24bda3fa48a0e685,2555308784f255c5,2592aeb84b524aa3,2675b4d64d8a67ac,27dffac371a752f8,28272265f88df7aa,2c3a3ca78805df18,313121cf7c8c7979,3328a5d9d8b4bfd1,3669fc2b6d502428,36e80d3488648b06,36fd18569973e3c5,37922e18eb0d074e,3823159b39915189,3a904b71f445ac70,3b74b28b9fb3a5ea,3ecb64393f6d7bae,43476472ccb6f34f,453c496ba1f9d194,45f0de18b18a9235,4c330145def618fc,4dd3bb60e1824eab,51e1044d551fc2c8,5bc4b28e107ae66b,5cb91007725955b0,612295721cb8082a,62676c81154a3bb1,635b44b503beeb1b,63ba8742baf47929,65f0fcdebd1ced33,695b6d570d40e483,6a38f6d66b3b758a,6ffc7d2da91ef9a6,70d3e571d1ccf327,71656c90ded69eca,74a9adaa7257f37d,75e5acd3f0a85002,77858ddede6b73f9,798a0e4cd8bbee73,7efebd1eeba0bc00,80df0b3dd48cd14f,81e04775c4dfe077,8bfd1bf34b189666,8d0150e7f307ca44,8de320d33b1e64b7,8ec7a7f9178ef347,9170159ccd22f0bd,9268eb1df76d2456,9501d923adc391e7,98192e1f1762dc68,9a2bef9bbae90aa9 +P-553007c9399d 000225fdf97e68e3,0018eb871d51f099,0024bccf81086579,00996b9c9c81a36c,00ab8291b834a5c5,00ba9ffbbad550a9,00cbf097305b9d58,012a5051e19f36b3,0178e0b0f97809e3,01ba1c6b4ea5eaf4,01f456d85e38b7e6,02734c671ca0355d,02cdc379c829a168,0352929471566be4,0376f875828c48a7,0381a66b69019933,03c141af921d041a,03f2ad8835490d8f,047c6bf687798a59,04d6e0810b3d0b3e,04e012cd3c326b43,04ef190afefcbe4c,04fa7c1d2be22335,05670abc9251624e,058cdae4c3d2205c,059e5cc490506a78,059ef2497dca19f6,0607ebc0a2d7bc46,0653e223335510ac,0695199d2e2b3ea6,06dce231cbae26e5,071fcaf8b2e0fa1a,0722a4fc746f7058,07869a94e1801299,07f71cd5816edf7d,08834d0745bad713,088c95552bfef48f,0957bfaca8dad3a9,099a251faa2554ea,09aca2da6b83cc92,09d7b369b1c3851a,0a074d2ac6852d39,0a0d1aa8bb0f9684,0a3a9621918f2af7,0a87d3872625849b,0a9253c9faccded1,0b70bf16312c106c,0ba5a5abdbf1ec7d,0c6c18f22f061f00,0c94d9772d6e7c29,0c97b153ecf2bbe0,0ca3c9d0d39a9d4b,0ccbc86a180d988a,0d03f3c0640f10ee,0d9e8a779fe1f01c,0dd250b0aee6748c,0de09765da244367,0e3467332517be29,0e70c533be0ad4fb,0eea761c375a43be,0ef406bb3cc9ca8b,0ef8bef1e334c81b,0f5dc4169b52f061,0f82c555ecbcd58f +P-5a4122fffeae 00393025e408f812,029e51141e87c3d9,03c0b8c208ddfdee,04502870f231df21,05820c2c344b51d7,06039a70658d112c,0698165d6f4b0d22,074fd233a412a9b9,07d47ae16e6dfcc8,07e0617c2070303a,09fdf1c7b9170ae7,0b40a2ea5c06164e,0c50b8e3770f5e46,0d7f89c9a742b1e9,0e59472550434f69,0f4dccd9b07893ab,115430a278c336fb,1161119c00b0ece4,11a658ed24657b0b,136b59d6f61091c3,13c44873c7b6a354,13f198629e2b77de,15b5170220e24f5d,174f9b46af06ea6d,17c769a313f51820,18343bb6f0f936c7,18be858663df538f,1901c092c03027d2,193c2c1faf4742b9,196f669d5d4a3c00,1c67b1eca2472d44,1d8b2c6b66e1fc70,1e0fa8c9b4b037a7,1e6885a4c8e775db,1f92e9090932eb00,1fa1b0df37883d0c,211bb02872aaa5c5,2227e596f702913b,2480dd8c7c4b0e2e,2517cf862a37be2d,25b34a1701b06184,2767c6a73f91afa2,2be59df5475fb2bc,2c771753a2344826,2da31ab3551ce7dd,2e2a293f42c3767e,2e599fc515898fae,2f230e6fce20b7ae,310ac799b410b266,31c3d1cbd6beb5e8,31f40183302c3694,3247a1b1f7692e41,3261b2d0c2834cbe,341dede16ab28ae8,3551c1c0ff54c2d8,375123ac26e7e5f0,377d32675adf5df4,3b1718679bd5c5b5,3c15b059fca1bdd4,3c2b794eb10f2162,3d521d39008e17b5,3e32d6545fff6dbd,3edcdedf9b84762e,4084e3f75e19d88f +P-62b51b8e169b 0260830850ce2bad,055bfde2af378fe8,0742c588b37f8e1e,0dcc24bdb7b4fa51,10bdae29ddd11597,12f8b1c3a98e0d58,15e2d55cd0f5402e,1a7e28cbec3f5436,1e1c34548abc3b01,249d65af2d59747e,2555c241d198f1e4,261db2591c27e583,2c1cabad5ffeaf6e,2f5d1912a1443d0b,3165f61b492149ed,334dfa5f621c9838,3c1cfc36b265ace0,3d03a3da8adcf1f8,3dc14debccf79ecd,418a4dfa4b77bc45,42c53bf0e9cc220a,44debf5f0981b9bb,46734b2823f0b2a9,4b3f1e094d38c7b8,4ec0b8a2072d034d,4efb51c93de6551d,503a126ca1f52451,50e0ab3c3272a15a,54e72e24a136354c,55bc849085cd3f69,5611ef296d656af3,57b388b51e466f03,5d375f00570fa35f,5dac48231d19a76c,5f055eb90c46070f,61c79c0a33f72641,652a28ee4b7c5054,6556f678eb22a19a,65818f4c702aaa85,672acbd4ce376b93,67f3f4d5ace6a5fc,69b106a78d12565b,6a955ecb7ec061d3,6b1a3573f390eb58,6d6d88605c6c1540,6f30d9bed7764f64,6f84bcd4484c95f7,7072fc7100ee706e,70adf27ffaa5d358,778d28d50b674971,77fc3a2444cfcdb8,7bcb234124688f29,7c775fb6ad576170,7d443e133e438f07,7dfe426d77fc9e77,7f0c36d15d7ca65c,7fec2af503dc5f51,80c400269adcd380,84c81e30fb69156e,86612b8b10bb9f53,8f0be3237ef9ba27,90834c55360c9224,94ac24392b30ac94,955bbd91a54791f0 +P-69fb0db38747 00e7f2419e84d48a,010c47a4c5b38098,012bd1ff8154bf2e,021c75e85cb107bf,0350339ef0a764e3,03962064a814df81,04e26149ea49e61e,04ef61418c18068b,062c66af3bcb1277,063b3eb93ac79db0,079f373b917ac1a7,08b7c96cf861ca59,0e1812a710815d22,0e794b547a62523a,0f74c312326ed163,1022cf51e2560344,1151ed4c096e0e94,174ebe8c481871a0,1798ad779ce7a916,17be5feff72f73f0,19782bbbbcb897bd,1a060d005946aaa2,1a4d0e14dfb27e47,1b2a029788ccf60e,1d65959a334824b1,1f0860d34cd3bcb2,1fb5b1af6cb8022d,211bbd754d004d6a,222ec293ff255cd0,2246f01fd6cc556c,226753f21ae023b6,231908936b1992ad,23fa33015f3b1603,28d50e330e4b889c,2977cf81023570bc,2a6c92d1270689f6,2f6a742198056259,30b94b4fa3251aae,3263884d97472c9d,3452f5a9817b13ef,36c9011afd264f37,36d9ec7b13bd2ede,3cb5e7616df06828,3de9e17cc4e5ad2b,3e0482eb277f7b9f,41082459cf109aa5,423b0d288e57287d,427780aa9eaf686e,46b7c95aa245b9c1,480ccec5ddbcbce6,4a8d45a0d1874e5f,4c31ab9f013fe994,4f0e331038b28fa5,4f15d72292bba58a,500a7fec91a8857a,50904dfd0d4ecffd,50bd776a5da4accc,515300f7244b26b9,55d3e9b378063875,592ad79aa4942f53,59e096d2b2a6bd11,5c2ce1d22efaf44d,618aa4715f3eef9b,6217a3d5eb9d2ded +P-6ec735d37bbd 00cd29898af73e46,0272823c4e198a5e,031e8c88eb2c2666,04b1dff575c530f3,051d7941a18243c1,05c93fee5773b165,05e15311507b2ae1,06501619948b33b5,06f67e508dd9b9c1,0ce176c8ec1c64a6,0d16d2c952cc28e5,0e4d86e074be0d56,0f9ac9a4ee8249dd,118009fcf1a5682c,1357214f6a91690a,141e0bafdeef8d77,17d1216e04d03dec,1dd0b73b32d363b6,1f42b8cd7eda6091,1f98ca85638ae9f8,1fa19963da20b3c3,211487e056236f24,216e327c4436f446,22fc73e29fcbec4a,23562cf8594f9511,248ac732da8198b8,2b175fa4ed5452ae,2cd706975a77eacb,2cedab52df59c393,2da7ae7c61f40ca9,314c5594b4397658,31d8be545eabb234,32fd7be15c5e8e3b,358a7c3acd0b6195,38055fb674ac9bea,3a0e6d7a275f2dbc,3a9e00c5a3a0578b,3bed1d595f14f001,3e67a0dcee55c19b,41ae5c2943a5fc7b,43e4838ce3469e91,471767e0d8684b91,477d4df609306c05,48f88dccb5446b66,492245b9abd3b004,49b580ce49627f3e,4c611ff07925a58e,4c91f52a4a5b4b2c,4d7a5c61598358b2,51aa3d9595573571,528306d183766181,536357c647258cec,53cfe8dd5fbefc92,543b84f5e83ba86f,55e747f2c1383e75,56054ee46c0e18ee,57e85ee004032cd6,5843a5e06a7ef2b9,586035db692e62ba,5a39bdeb38f952a5,5ebeb9d3378e0f6c,618e10de9d4d0521,6294319a65d07657,636a9ced4e804272 +P-7a1b8a7554c4 021e62cb5fe0c47d,050369444fbbd65d,07711c34af64e362,08f062136d739d6d,1546658fa9607641,18fe7a4914ea6ada,1ca79f4ffa9c23ae,1fc5c6f2fbb774a8,21f3c1208912aa58,225aaf2d57396bc1,2351eb3a10ec48ed,23c71dbed37e5471,27e76509e93e5dbe,28c37f49cc67b92a,29728341542725b9,29776e8c3350b2aa,2ad9748755212c62,2c5455da7c6e2d33,2ed5a98ec39c02da,318dbd4a6980d210,321fdd4b2f8d2ec0,322d11a03e85d7a8,3286fcf3f5ec8db3,3338bae653a58143,34ad3b4f9a0e3ba1,35075309e4340b0e,3acdd26c3ac89217,44180a1cbf03dcb3,4a156cbadac3144e,4b6aaa9176fd8361,52d0c37c5916c785,52f7d830ff56d751,535ea8148d8f7ae4,537e3d3f14071a0e,54ab7892474f16ac,574860bfba04857a,585bae9be79cfdb2,5c4e07f837db8954,5ebdaf19919035c3,5f112fb49c5c6cdd,5f2d163dffb5c656,5f7bf6e0150b3586,6096d0acd6fb23f8,624773ad9dd6121a,63c3963534839cf3,6c75d5fb6977ba40,6e723590de5507d2,6e7b535b239083df,6ff850267f24921e,7231d7a81df18fea,73cbb73480fc3c8e,75703bd5bd27c59a,7cdd625a022a2a1c,7f98d4af47e63069,815ba06f2b6bc770,83c10a1e219dc9d3,845bbf843e641d70,8488bf4901a33b5b,8b2895b166703d5e,8ccbedcb7a88fed2,8d723b0f05c60455,8d767f19a98fc6aa,93c553ddeb4f3e19,96c177e11bc2eff9 +P-7c3e4fac1fd7 012e8bdc6d083717,02261312a27c7c10,03bfae335f2fe8cd,07cf1486e9cd2c93,07d1bca53d2f392a,0802f30132e93296,088da736fffcb163,0e6e15ecc2620390,1219e3ee0841af88,136c1c0d0529b944,13cf87def8cef580,156f7f043f99e97c,18d814afeb221454,1a79c366963ec6d5,1b07797fd9f6e4b3,1f0eb053deaee2e2,1ff9dba39dbfe46f,21858d963d057aca,24d29591a43291e3,27cdf3a3ed58a628,2d44e9dcaa203a60,34a4c1f65b603dd2,35453b27f28f61f9,37f1208c82ce8a7e,3a6c957d461db22e,3fa79dad1e94feb4,43460d583770e027,44da6b7b0ab8f6be,45bd894375c08223,461b425b5d638f09,462c55043c5afe32,47a99ff8bb259f2c,48da748fb6fa17d2,4ad6e1226dd0866a,4d452e1e113bacc8,532db5476200cada,55099abec2019d28,584f3f1d0131bc25,59d2c32020605657,5b1bb96384ab5188,5b8c3ed43e7868aa,5c6b36dd9d3809e0,5e4e1cd1f1834430,6711dd48a73660f5,67bdb5513f606fb4,681f079826df8625,6844f877ad7b2010,6891031fd9c77cae,6b4b64b88c6aa6df,6c404dfb1def3a77,6d7047ff072d6dab,6dce8cd78744b681,6ecc7e9a626fc38e,6f1839ba5317b312,6f74e25224246866,71a436045315f389,7524bf97f27459c3,75b2e8290142e55d,78808f897fc870a1,7b089d1f3c231eb9,7bd4d2234d591582,8097e1bc30882e8c,8178920f1bcd547b,8b737180bb2cd198 +P-7fffab1fddb8 005eb817826d29ca,07585b0ec0f0bdee,090f6d9f4cd5f894,0d9846991fc14892,13a7ea0ab281cb5f,155b1e7d2297e8c5,186343deb5a9c01f,2255ed97d7bed20c,290ed2b3cba840cd,2c2acb90eed8bd44,2d831e6d4cdd4466,2d9a88c6e8076aee,2fd4855602c42e08,32ac31c32d333ee9,382b7fcc03b61a7b,3b3066c9d84660a3,3c995c5f10523d4e,446820ba77879fe0,45efe01169afb637,529b0d15dfeffe2e,5338c2ee86617f1e,541e8c439d78ab41,54e72e24a136354c,5bec460e807c83dc,5e51f72a59676f04,5e706ad50175be78,62d0d88816e99303,678b2fe35afded14,70fbf3c3faa9c68d,76e501440ce24cf1,78c4f46dd89a2eea,798be064af9d6820,7f15db17acc06b97,801a9643b76953a9,83eef87cc7afa852,90aa60271a40740b,9563b92059931bad,960adc0ed6b05f3f,988bd2dcb06077ca,99b78e0f02e323b3,a16285cd17f10ef9,a5f0add267fb66c6,a91cfd9ebbd20149,aa3b82b1e9c7882b,aba3dad8e1dbf020,ae2d55f9bcdce5f8,aef9b03d8526c405,b198c5078a10797f,b83b6e54231bd9f5,bbd8ba40e5dbdb83,bc7e169b02412f5e,bcc484158bedd03a,bceb978cd43ea9f7,c0c0fdb3219a6f90,c14c7b9d7265bab6,c1dedae55a4e8c73,c477fc5a343240e2,c765007dbcc6593e,c8672d3e22c9b6a5,cc6d4dcb40bf0aca,cf1761db15100476,d10c1cd00e9b07e9,d244807cbb65b623,d98afb95919ecb6b +P-8453b248d79b 007616f6b3704e92,0131527b1197c74c,02b3f3b64805d96c,0478ace2d852516b,06a56450add853c5,06dbcc83b9796138,0741407d7e06d6d1,07cf1486e9cd2c93,08bb91df3dbd23b8,09a1bd87d122858b,0a8630530f3b8bc7,0b7028e3e65551bc,0d6cd8fd9402fd25,0e854b19b2b9688f,0f0a025c226002b5,0fd22b9d41f9fb34,10a624bb992db08f,10e523486a979dae,117a485bc3858c8b,11fe2fb73c86013b,12b73169969adc23,16cd91d680b768e9,191fefc13a3330e7,1976266ba36057f8,19bd2ab2228dbbeb,1a2cc165b3182185,1a76911af1154bae,1aac1e76de58f7de,1ab479812e2e14b3,1b14f6d5fd0d9372,1d19ce83dd50c7e0,1d6dbe3e989e030f,1d6fa14fac60ec5f,1dbce66e9ccbdf32,204dae9c10502d06,20d62d69c5f8fbbd,20f4a99b920c52aa,21f05d874f7ab71d,22e05a5e0914a47b,255f90b0dece6d94,2652c86e40ba7bd2,2660e1e458556edc,269d6023b7f9a5c2,289576d7507277f0,28e299132581b00a,2972945a695160cf,2adb8aeaeebbc3cd,2bc938a484a2342e,2da32abedc15e2d4,2dc617199dea0da0,2e45ba6074522fae,2e6af31f15712cc9,2f9f3500ce067112,2fb043f6ccd9693c,2fc82715084f600c,300429e602dd1e0e,30698ed934a034cd,340151f46bbaa5b7,346d574f947f64a0,34b34286c434c833,3563ac7b44aba752,35e769990d1cfc00,364f14d54fbe34a5,37de37157c48d9ea +P-9e598f519091 0220b2a5b8c00819,030ed48f3aab0160,04c00264e686353a,0684e4446190f88a,06f84b6c871224dc,0718becfb34a16e9,096939e9991f4a3d,0ec929985a997515,0face6f9b33228ad,12ee6b360620c048,17c2defa33c97a71,1bc7315a9bf0b299,228415b57683255c,2570628956e90b78,2c1910ac7a65eda8,309e96cf19ed9769,334e8782226a1bbc,352d4a1b3ad159fd,3566cc8111f7fa3b,366c00913c8c5212,3738a85ef932d483,373a084b2ef38ac1,3a4b691dea9d30c2,3d905fe34db3366e,411490e8b9eade9d,4564133c7d07d2e2,4566ed58732c766c,4aaad163588c5a7f,4ca5bd19485d3f65,507fb54275d1bfdc,5161a3eed7673f40,5447b5c30af2587c,548b72655fa79495,54c86850bbad2195,5d6c3ad63f53682e,5f46456e8424b004,603da60a69a1d78d,63bdbcbc1f7f74b6,65071f811169b715,68e4a872c59fb80e,6fd80286ee6de431,709011b565f38442,7205b0e9f02f578c,7276ce6402948ae5,73c91087ad8d760d,77245a6a4cd95b88,77314fee7eb4e4ff,7e5aa433e65d5a7a,87596242f079900b,879c3eb7e0491bb8,88a3765de6edd75b,8bcf211b56e70425,8ca7253157bf8419,922d797e2e53f2a7,955500c0eb1f470f,9e0a077851637ef3,a273e81f7837ebfe,a5ef17f4f93e5c34,a73d8e59982c6bd5,af7b6f564290a614,b04e410e59d58a80,b09fef37f9714d3a,b1958599d71250c6,ba06fd3be90cdb49 +P-ab41daaacc1b 054900cd523ac69d,06071ced49aab0d6,08297316c9ee2146,0cc83112a0b6b1c4,0dbaee2f0fb0dc57,12651305d439a3f4,1b0257e8ec119a62,1bd2fe3374b25bbe,1cbd9473521c0a6f,1cd3eeb42dfd18fa,1ecfd702065184d9,2128ee25d48f6980,21d4dbaeae1f83d9,302344cd51dc771f,34ae9fc55e98a907,368ea79275de11e2,38849359e5bab1e2,3d55324c337c6fbf,4623c72937dbc563,464e3db95bd1f183,4676f29f5dbc8700,4cdcd10cee887845,4d802703a9e9c83a,4db0e32d937eaa6a,53326150c8837171,594c9438472b878c,5af96fb7e6f82b83,5cb0800554b44b87,633631b95db3c09b,64c798362f145b69,6e3cce89341b47c3,71cdaa7063129d68,7554a3327b16e293,78994064840e6756,7b242c44bd385257,7cc7466a3b7525ce,7ddcaac7d3aac910,7e94caa03dfe2ab9,7f124ca8fc8de5ad,83567d4c2b78e701,899a74e71bc1f333,8ba4ef2e11b7b88a,904563cb2f3a1888,93bd7e54255e631e,943b20673d2567db,9cd68df52246a49d,9d1ed5d9707a835e,9ef4a29809ade0c4,a45baa84b691d8c2,a4a1aef98b9befd7,a4bfb2ab959c3b55,a8c9bf50f3bcc4ed,abeb16067a641a6a,afc540279a986f0d,b02f3a5c1790f540,b5edcb37c7c0a962,b89ea93c59281455,b928e36de995ae2b,bc258ea2090c222c,bc8e0b3de508880b,bd570c8b28521fa2,c1537d26947366c0,c69bd64f890fb23a,c7df31a335f5bfd7 +P-b18dc9a4b9d9 02d6f4877987af7f,05c3f128a0e3d047,0977c5093e4a5313,0aba8390829d4289,0c6807aac37a9bbb,0c8787ba3dd06c7d,0fc814047808f45e,11cf0221ada5328c,12a68410d97f57de,1383ec41bbcc5dd9,13e49166472aebf0,19c127fadc34678a,1a9ecc43a69a4948,1e4679d2496ea0df,1f109dff7911d8d6,1fe261630125d953,2207fd195b9996ce,248ac732da8198b8,260523a56e17acb3,2864bc34a2021cfe,2aac40bc9dec2aff,2c6789a5b4582bbe,2ff0e1bb8d9f1c72,303dee51c7f3b15d,31b36d46af795fe9,320c168f32d7a7b4,325861c396992337,337d5180cccb8b72,351ca057b522dc09,3555a80cc349298b,3c2b926fa8010b74,3c455df163c76fd0,3e051b0696f13dcc,3e5d22e8322cfda3,41a0bbb302ce30b4,421228205433a46a,44abcbb40c033a89,4773673aa203c6eb,49056aec94e63b3d,492df5c855151787,4a8cb737a5e74d5b,5203680689d9a131,526a0f17b97b9b0a,535f7db3b9535fec,537c52019c9ab393,56c2e47e27e5a117,5903a230f4704140,598b131a01e0f6b2,59a491b2dfe6119d,5ba46f457b048ac7,5c0085d9a34d76e5,5c9cc193a8160509,5da682de5a089db5,5de537695b4c62de,5fa8dd4f85ae04a6,61bcfcafc4df369d,653183914b58ec06,65ca02904aceec1f,66b2d30a2be55139,67661e7e3ca1d854,6acc42c12704806b,6b7b993ac64cd692,6f54f3b8f4920301,70f73a9b3530a7f8 +P-c3a0a45804a7 039955caa5380bdf,049c0dbc41752ed4,0b63901d6dbb4c16,1178d0778f4db08d,11ee5e1c89b3f09b,14a262311f0137b2,14bf8a3aa0438a68,14f67dbdea8e229b,1acf97c44f3e0b8e,22d09842528f3330,28954bd82531d2c9,2b452d6c1f2a4d82,2c4cd7e85e6ea4b1,2ff0e1bb8d9f1c72,32a4b6c0eb1b85e5,36510dbdde5cbf2f,36665aacc353f309,376921cd670ac2cd,398fb44e24ca6e2d,3a52e4059770fd80,3c7effa775264437,47c32ce51e9fbd07,48cf390d991c0d9d,4b23e6c46ee04544,4c5a169bd08835bd,503e812817b061a6,565b19fe76a00032,598b131a01e0f6b2,5be0a8a51ea9d76a,5db55555a2137a65,5f9b56e308909ef5,601c1f097def76d6,6120d11c446400ab,6a8f16a3d66a1dbc,6ba9471b67d5014c,6ce5c9503ca056e3,70b5d5ea96b7b18f,7124a1efdc9bb4bc,72641e670242acdf,73aaf8d492e6da6c,76cf1400545ef7c3,784e2401798a29c3,8627a0ff742ec38e,883d4fa556af8e92,8b688466154f5010,8be08c27e5179278,8e43b2eec73bd852,8fea59ad83008e7f,930574e7a93c7879,982adc1138fde00b,9a803bbd6cfcebe8,9d6626fe7de69db0,a6893a3f663a5790,ab6ddd316d668351,ad41d8f6b51bca3c,b9ad2f9355a0e7dc,bb5c952da957ec54,bdb7808c87752231,bdcb204c3634647c,c296a4b771a54aad,c5b97b850b1a9133,c8267c1b355a65ba,c9ecc44d2d4b803c,cb075c433c917178 +P-cc6f48e56473 00d44257951cdf23,03a15fc0635f6c13,0626a98bf0acc080,0863010cb501a874,09f159356396255a,0ec2bb35298bdb31,0f93bcb0a18d1c72,128b989871faffa8,149c0382f9daf24e,16e74244432b52bc,1882ec5acf8c2885,19ace5b3a43b3a5e,19eefa49f8af80d2,1d0c95c99853f501,1df7c61eb9d9fd49,223a920f9bba6ec6,22e37c108372624e,251f9d21956cca66,272ea2c61c99d394,27deff58d553e87b,2851a6c900bf15c1,29c67371f046e8e4,2c826961509d11d8,2c8c38e738e3fe1e,2e05b77e046e0409,34860521b11fb6bb,34998a8a0c8a0745,34ce25cfc73db1a7,350f699cd96f9f84,380eb61e669fd7f5,38109dc0f8cbc65f,39024779a2b16d43,3a9a9dd15e7d2cca,3c3c396fed2d97d6,3ca0f2c5d4e8bf0c,3ea6bb77c91d0fa0,3f000d29b1090427,3f696849593ad4f9,401784b0ad0fe43b,41ce33fdac763d8e,4225326ec0424685,423aa82a841eb72e,443c0a5366c9c7e3,44fbb44b81665f2c,454d4c57ef909c3b,459a52665d47f0a7,46582a08782d2c19,483d5e541f8a2be7,48a03ed74e3265c6,4e4c178e1ff68651,4f654d962a7362f0,500db2f8f4365bbd,5561ae369b1b6722,560b9cc186d16db2,56204a6b4a0611c4,56d16925db171fc9,5a600e993c6ff091,5a84954d5597c13a,5b0136a6a8e9c50c,5babab775ceacd2d,5d866d3484c08c54,5f502680ae929ddc,645ca91e7d42cd61,68132c015a8052b0 +P-d624d8619161 03ce2d428a12b89e,0455d60a82d905c1,06718120676fda8e,0742c588b37f8e1e,0970c6ae257fc078,0be6886c19a8279e,117307ff6fae0f0f,11f5fc85825fee40,12eb7b695f47cd5c,14311862e3cb53e8,155b1e7d2297e8c5,162bc4c6a3d502f7,2255ed97d7bed20c,255e1c6f443ff7a2,2bf83c3ccb0d49aa,2c2acb90eed8bd44,2d9a88c6e8076aee,32ac31c32d333ee9,3c0aa68d471fc6f9,3c995c5f10523d4e,3f35c4c25777307b,4982961b32a72205,4b23e6c46ee04544,4b3f1e094d38c7b8,5022c4bdda2f1429,5039f09ac8439c5c,50775c572b666f26,536ee112b669c3fe,54a5c06c2ccfdd87,54e72e24a136354c,562af5fd48ec8fe0,5bec460e807c83dc,5f4ca8811dc26c9c,5fadf188776ac994,5ff1a4d68dc9666d,63a35a8d0dec2d93,69f2bc10ec0f7044,73e75104abb8b8e1,74db35a4bff36ce9,76e501440ce24cf1,9563b92059931bad,987efb9d6b230dc5,98a5db8adeba337e,a5f0add267fb66c6,a91cfd9ebbd20149,aa3b82b1e9c7882b,aa69ffb5047baeff,ae0b120582503473,afde55338660f6a1,b5f2c5b7d52aea26,b6533b0569149c03,b83b6e54231bd9f5,bdcb204c3634647c,c1fb903350b105c3,c4cea9f41cf663c2,c56215c4a1b14f75,c7c41fd80c0a1c1f,c7e6e9f4cb3d98f6,c8672d3e22c9b6a5,d244807cbb65b623,d5184353bc99ed10,d71ce7c9d6ed2f49,d7c467d394ea6ddb,e6d228c7183c16cd +P-da4f5964dfb1 02db23b385874f56,04b1d15492795c6c,0742c588b37f8e1e,0b7fc581780e0ce5,14f43c01c982085c,14f67dbdea8e229b,18e702419051fa1b,19a593bdd056e5d0,19ea74894888ed14,1ac7a5fb5bea4da2,1acf97c44f3e0b8e,1bc7f400eedc128a,1db3fef5fbacd503,20df08009f5fd00c,227544c5c82eb083,2640c9e1fef70bed,26d92e10af1c9c3e,26fca831aaf3fffc,2dcde878be3a7e76,2ff0e1bb8d9f1c72,3234f16d6204318d,33a3ebf797a1ed04,390ac2fb88e37010,3922d53696bbc2c0,3ebd858d870ee71f,404be06fa06140fa,405fdc70f6ef9ad6,464f8c6d14f33e34,47c32ce51e9fbd07,4818c87512464fb2,489e50358a15952c,4943918331898aa8,4a8e3698e7fa3d26,4b23e6c46ee04544,4b3f1e094d38c7b8,50a163eed6399137,52325727ef96f573,54e72e24a136354c,56d377dda6a36eba,598b131a01e0f6b2,59d16a8b75fe93d9,5d1884ef3579372e,5f309e3013fc80be,612776c08d02136a,6557871ae0870635,6a416904a2f2afcf,6a52e6e19120b37b,6b834713b029e5db,71b3c102aef4d8f0,728ec789e0e49832,75ae1c5aa54c95d6,791102e88a5a6f69,7cd382e6f3c16eee,7cef93b29e8ae66e,7e5c437079a4b44d,7f81e103177865b3,81ed41d5e2ffc15e,82ac985868407080,83c5235bf23c997b,864d7ce626db3bd3,88ff89b85e91d752,89270651eae6d966,8cf0d6a5ca1a56c5,8d0671ab0d6f0876 +P-edb10eb7e5ed 029f8dae82b74443,03d8ad0fc2c1cfaa,0537b51e8091b2be,05c4f83789a9aa31,0bf9437b8fd6842e,0cd130c2fb1a1328,10b403685f5b764b,11ee7f0909695238,13336ed2fee6c589,17548cee6b3dc16b,1dc6f99b5a2c7e38,25a0cab4afa0e8ad,294e521a01adacad,2b7fad93b856bcd5,32e2cc9a6e66f0a6,331f68062027b4ef,33451e964c057820,35633cb23301252b,37a228b895386eff,37bd78c9fa521968,3afde2c21213c6f6,42f54e7db97f8cec,461849435ec6a6c9,47c19662070a9b97,54522b97005655b0,5511a920225ef3b3,557c95824dee5bee,579e14ba79226a85,5940a601c5f0d9b2,5c0275b5f82001ec,66ad03228b076dc7,6720526bf357160f,70008ca83e20590b,78385ce828d0430b,7bd3e4f384716895,7ec9fe55ff01b393,81cfcfceef78f771,8456768e6a5ae5ce,869f7dbf79e10574,8a5ef1222816f45a,8f219cce5b6f3a17,904a3e324c61dfe8,96bb44ccc95fadc2,96f556369c5726c8,9730253cd44b1c1e,994e686bdf666182,9b0e1d51ff4b5ccf,9bb38ad566c57d89,a3db01cb87f8a100,a4e748f1ade9568d,a5be354a78d9a433,a60bd1714db047d8,a9c53d06d227be02,a9d68b74234a0225,aa597127887bf8d0,ad7d3caf665fc6c8,ae9e4f5e0c3decfa,b0d3e4d9c1e4ab12,b2a985047158fc1e,b35cf00434367b8e,b654252e4ac2b60a,b8122adcf97c241a,c022ff086b8adaed,c3ba4c19deb61ddb +P-f3abd4438999 00bc3af7e3befc03,0148b55e73e147a3,06eb1f10e97579ad,094a163db2f24f3b,0b9120082568f958,0f24c2039f5be5c4,1115029aeb41643a,137064f55cd96443,1729888b6fd9f3e8,1738efb348605a70,18a236f3c30206d1,19fe50a0cbd4b75a,1cb8feca9511f44e,1ec2083290b5061b,232e19f1d38396fe,23629bbdec41d336,25ebeff2140684df,26ebee13176fc763,28ddbd94248f3a39,2916fa7b1ff79350,2ec9aece762ca819,322340a14a254124,3233a7a891e900b4,3481f09c44bf5fd2,3685ff163bc792b5,3883e592c088798e,392cb2b8ee16e14e,3a6936fd7a93b635,3e507add95ccd280,3f45b6ff6c3694dc,3ff85dacbe0f0d34,441bffaf742d1744,461f51802232470c,4a385ba5a9ea7305,4f0e146decc95e8d,5016d89bab4f666b,50272d02008c848c,572a27dbed854861,5a82cf7dc0170779,5deea4d9765ea2bc,5f0e3392b33df616,6116f72fa228fa10,61c4eaea7c16fade,6327dbc3887f6c35,646a683e776a543c,646d6ed0033d3234,67be53952ccac0ce,67c96baaa023fcc3,68764ce133a791ea,6d1e533be2f8e770,6ecd46006ee31ec4,74051a322b45355e,747f1d3c77fb883c,76acfc9d71f58bf7,7b0882adde4528e2,7cebbfe0b87f82d6,7efbece9608fe20a,7f35fc38819780ac,811cb5a3ee528dd5,86841aa12ddc96e2,88b4971598097fb5,916f699eb2fbde3c,953b0611d5ec67ab,97e65ad9e384c50c +P-f72984d91c0d 0001860d18401e2f,0038135aaacdb473,003cc64d201a42f8,00ee00d6f43c9099,025bc0a697fed8f1,029673c0e107df9b,02e2b5f201626f03,03081130ad9f82ae,030a7f3b6dc638b7,03843f034d5cfebe,03ad4b4466258820,03c92aead826751d,03f51637c9bf6a74,0423d77f58d7979a,0427e45347b86fe5,042996df428feea4,043b1d4033eb55e3,04a032d38c89f183,04c350faf1fa5a6f,04d08e0a929f5cb7,04e6ff7c88cfb8dc,0506effe8ebd6000,05145c10b0fd120f,054384296e88df24,0554ba984646ddbb,05603b0d4b0c9fda,05d0f3c8b814f57b,05dd45ffa999abf4,05e2fff15c34fde6,0607ccf2fcac27f9,060e19d5cd199f9e,06234af1685e73bf,0628368f88a5a686,06622a5383e3ed01,06c29a0d8db5337e,070f609b6548d822,076bc9c3699791f5,07a1edf25b93850b,07ad7b5e7087a170,07f89df5b31f676a,085573ca85a654b3,086cc5a718ee1888,08f298b9c37ca8b3,08f4e09fb5805b93,09048c84b60f8380,09166203ee465859,094b8b340961468e,096bdc0d09e33687,09e047fdf38f25d2,0a01aed3d9a0d72f,0a34978a829556c7,0a40d1665076ea43,0a5125c99217c470,0a53848930d4969f,0ad3902252134a6c,0b1000f365909033,0b1248e7491e7f71,0b19a1388a6818a2,0b1a65695c83d213,0b230bdfa3130191,0b95c3f927d6f7f1,0bc8074ea7e5303e,0ca11b208b4bc9d3,0d3271e0cfb340ef +R-pr1411 000db23b567240e6,00142ef715837e85,00324a7d39646939,005d9d5d7bbbc1ff,006efaf480c4794a,0078be7b4243b3ee,00827318c54b2cda,0098e83167b7777c,00997948eddeb9e1,00a3f6d2c32056eb,00bcb241e28778ae,011c01284b1aa3c9,01204d000bdb6bf2,0122bb50e2fbd521,0124321de6e8a1c2,015b9b000a39dfbd,017042731c36b6e7,018f14d6b5a40339,01ae3b1e7b4f8ef5,01b40f052534b878,01d0db2db40a6513,01e3b36afa02b2d2,01f3c2be20ed5fa1,02231a295c6d30be,0259b2dd5d4d2092,0265adf398bde1f1,026cd936c7d7b610,028c47a996d960fa,029b1dc13bd4185f,02a41ad582163726,02afb460b04c5593,02b0a501441bd0af,02c8acde50d9dbc8,02e80db3fa08dafe,030337bc74b41101,031a422a1a982c3d,0329e28224a561ba,0347b1d6de40c96f,0355139eccc615f2,0368b806d579fb41,03812c685895d133,0393505ce249d4c5,03e50c95eea7db38,03ead4fb976df9fa,03ebf6ce817944db,0412e6491b3b1a30,044bbd631c5e959f,0455e0c607a0de45,046620cef6f49f0f,046a0e39d53d8767,04936cc4b1bc7a90,0499f2967a32bac5,049ae8357367e84b,04d48d529153957d,04d4a28b8fdf16c7,04d8dde6e26bdc1f,04ee5620f85e9a90,05066df2fb9a83e7,0531cf53aa2bd067,0536fde2230e65ac,05452bfc1f7146fb,0549506e20313d98,0556414215952d21,056560b5a236f6c2 +R-pr1714 00f0d7cd3d36d00c,012cfed80b986c33,017240e6f81cb379,01c30897667039d3,020e1a5c88a4ed11,020ed2c347b0c6ba,023a6048612754a0,0388ac4daa3422d8,0392660dcc4a0444,039f3fdf4e5a6c2d,03b68d5b0b8accb3,03d7f60093ac16f4,04372671b0acff54,04a78b4b58db9956,04ba2483ba3c0e28,05bfc4dc0099e601,062737f52f697b98,06366fc160ca9ee6,06b2580b807055d9,07053c2321c06cf6,0711701b3a475229,07c1b8a54d655a2b,08297316c9ee2146,0864f7f37c44ff94,0881e4bfb1e034e7,08a0d26eb26540b5,0905f82758fd691d,091b34a194f975da,09877940146d2cf2,0993d021a56bbf40,0afcc7bc2efddb07,0b30fa84951b8775,0bedd5afae124417,0c4324fd936d0eae,0c462c8213672da9,0d1b62d5e60b941a,0d26cabffac5aa47,0d39a97d9cff6418,0d6fde0b49f5953c,0d8eb9141909a65e,0db591ccdff752be,0dd79670dfae45df,0e398d36960aa6d7,0e6b33a591d38076,0e7f05acd94b6016,0f9e4173bc9a108c,0ff37614110f7ea8,108df137d6e6c993,109512aeab8a86c9,10cc0eb52cca9adb,11235191655137a6,114bc0063a775576,116cd25fe6bedc87,120c2b83fa170d8f,12ede0b77f918b8c,132e1a073450cc93,134ffe6a6430882a,1367ed21fc6cdd6f,13e17e43733905ab,13fde03ca372a5f5,144230bcba30ca77,1460f6d1177d0ec1,1468efeb7ef9faef,14ab523c7b950ba4 +R-pr1782 000eab1208d8e4bd,0012e1e6e3208cef,001df239bebcfdad,00d44c9b3d6c75f6,00f4de554eddebc7,017dd38fe8db2367,0198ead498201acc,0220798665ee8785,023a6048612754a0,0266ff0b8f127c1e,027cdde38cc70ff3,028d9b500d0d7271,02bbf02546acbbd3,02c6a5da4839f2d0,02db1ea88a36f8e8,0360b58bebd078e7,03ecafc032eac3fc,040f1ccb7a2bc54f,047506c2b5baa934,047708599c58c8fe,04b712eb4138a31b,04ccbaca6617078f,0549c532c736059e,054cafaf8cd09298,0579d1482d293646,058a8fd9b3e9828f,05bac68259779996,05debca7c7775ddb,061bf66f90201862,06726f744510884a,06d363c654d9956a,06fbbfc5a4df755a,070119fc14639fa3,07aaefc6c5c83329,07cc694dc0bcd771,086ae5ca408d83c9,08d872dfc3be57e6,08e5d90088c3a90f,09877c2eff946267,09c44ffae0da684c,09ddd1c344754a0d,0a0a7f4d57d443fc,0a58d9d53b7aaea4,0a88f31f49a40ceb,0b39829d2edee103,0b4f270d24a517ef,0b5f09932e697bb6,0b76987d27a83627,0b7d5275fbb91c8d,0bb540c7e6ceae6a,0bedd5afae124417,0c2f7d2208287334,0cad15dcbfece303,0d53971bb7956b31,0d80d01b884b02b3,0d837b8c3201c969,0da022103e8747d3,0dc50daf522ab161,0dc5c641940fd49a,0dea1e4e18a2a8fe,0dedaa6bd0fa25be,0e12c76970be6311,0e6433578240323a,0e8c62be93639d58 +R-pr1806 0023eb1fad9de2eb,003d322577cb1fda,0051fbe2adef5e0c,005d342c3bf0e181,006bfbec16f4e759,00d80acce7efbe30,01015e7c2655a32b,017d97712e4fc07d,01c71e365dbdc6f0,01c7576e3982d74e,01d3c3422815d36f,022946db5ba9c799,0262572a80006cf0,028cf8f92d4a8068,029d94887afb72dd,02a3a6c3214de0aa,02e34e2dac6d13b4,034f7d1d01cb15ed,035c564f1fe29861,036dd8d6e3435128,0395b8302ba7413f,03a702e996f9e8ff,03a8a0cb2aee24d0,03b291bb138799a4,03cd88566e485ccb,03d237c1fc7eb61a,042468c308e966a5,04689273820a6668,0487e5a9081e4391,0488c03209605733,048e33cb7f227d0a,0492e742824bde46,049ebf4e7f3397b5,04b6ef3ede6e02fb,04e21f5ba20c3845,04f60ea8e494f3e8,05091bdd8488c164,054e33e7311c0714,0587bfbf0a51c9ae,05972843454e80fd,0597fc7d749a5710,05a5bb8816e51017,05a635aa85c57383,05d12b7d5d78a9ab,06298e2a043d3ca0,063030448d370a83,06366fc160ca9ee6,063fdbd189115926,0650f7203aa62dd8,0656ec6b7497c222,06765b6b91b9d623,067e3b3de8b5c9b0,068a51a61388cd4e,06b0d9ad0a72440d,06c3270a15270cb4,070f36dfb7b5a01c,073b6ae9a972a0ff,074bd9ca32a7ac5e,077857fd36e0f653,078dfe16313bb74f,0795f2c568b28b5b,0803abd4e6d69620,0815e67ad52660c8,0846ecec80eca79c +R-pr1808 014981f5e1e2aad6,016b27eafb638cdb,019ce3f6d1cb3721,028db1b0d5a77418,02cdc379c829a168,037843b7b5b90a01,03e56c2181a792f3,0426567bf964146a,0470b58f357177b6,04a8e946d0f635ad,057843716310898d,05c26f0799208233,05d877d166cb84ca,063a0807039e8cb4,0673538af7cf6dfa,068532b563ed505a,06a8aa4de39ec91b,074d3e79765a1129,07b03aee13b47cd0,07d0730e2ff2d335,09c686c8a571ffa4,0a2ea9bae4b3db63,0a82f2677c7e7133,0ad578c9dad988b7,0bf53e152647b40f,0d063bbf20526047,0dca1e8b2fe8add7,0e2ee967813c090f,0e530876c8667d1c,0e6b33a591d38076,0e844695b7b45a40,0ea92d829b9f4c1c,0eed2ed7736c00c3,1018e31c434b87b9,1027ca626ce020bc,1063539e46884712,107afed395504a35,1097f6404f44d348,11278420357c255f,112e46587b64fb8d,11e800408ee3dc33,130ab5f5f8c309ed,133853bd211db7ca,13819fd31de3ed56,15290aa969ba8ba2,15f48c93389749e0,17522f2e4dd22057,176f4f6ec3762cf7,17f737e4beb98402,18aac18579cdb03c,18c447287c091134,18c7d9fbd2f94bd2,19c310ea398ead4a,1a63852ffb3b693e,1a7e2d4096b83601,1b09cd39319ee06b,1b6abbba6c6e29cd,1b84c8fe3aaaed0a,1be5761698aa3f65,1bed9733528b0a20,1ca848ee0075481c,1cf4d8f7146d551c,1d5060335f2fb72e,1d763599f27d47be +R-pr1867 003c30bb613a26e9,012834525b9d2c75,016905f56832318e,01b80a8288cfd215,0202c893149915eb,02eea3580474ee41,034c7fac88b41a98,0397d7e6f85d2b56,03ac06fc2e037c14,03f8522bc241eb51,0438d82c52ce40d5,053ff12281c8cd80,05a6542cdffcf72b,05f9ba13e181a87c,06012114ba7c99d6,068afef7a141c2a6,070620ab25841cd4,070f0361e1ff396c,072ced5f791c6270,07b1f194c67b0d09,07ddace8d1023fbc,07fa5b2419484adf,082652a7cdcb7791,0883b6a69b1be7c2,08e662327e2db032,08ea0ef4243291a4,096728a15ee9daba,09b4508af0280d8e,09e3f31f790db022,0a007c109f848c21,0a2322c0119169b7,0a7407eca6782bf3,0a8eab89070af523,0ab7283f28d89ba6,0abeb747b6415ab3,0af464f501313709,0af50b8c906d1a3e,0b1e8f323c37373a,0b4f270d24a517ef,0c0ea893dba10591,0c4caafd17923c18,0c6b46f8ee728632,0cfefd376fd51e9c,0d13cd0e0dd8e615,0d43e6d7b2f287bd,0d93ad7b06df7c9a,0da15bcd48b9d6a7,0e423c76d407fd57,0e7477249049c997,0ecc5a964de19846,0ef980b265ddc576,0f3d818f72a09c5b,0f7ea7a146e891fe,1005f02b7be39687,1016d4b54d9c9a9e,10a10f0478da0eaa,10a5a1454ca04962,10b54e2e38500d73,10d276ee88fbc055,1121fbc3314aedd7,113239aa38ab70ad,119ac679d0776562,11a0554430182516,11b85c1f87bf9054 +R-pr1870 0052b37fe93613b3,0069fa76f86efa11,00cf881bc0462bb7,00f26cad022f71db,011ee4529c56fb4e,0135ac9fd194d255,014adc7266eed685,017f538d1da84067,018f8ae4d66a1c32,01b17b4e9967542d,021fcd78a1b96b71,0252c69d62d2f000,0262315d687adcad,02c7cd93c5f18baf,02e71b895c68bcdc,03149c91f18c1c7d,034330fb8682dad9,0364e9f341934c04,03a58016b99006f0,03c8809f4ebc0e68,03edefaee6f506a1,03ff3dfb94dc52a5,04144f3dd2814192,047f9758a8cc2044,04a08d6e7aaf3848,04bd604278d73575,05452bfc1f7146fb,054c3a1db89ce716,05c545e9a848459a,05d6aedabaa66e38,05d8995b84c201ed,060fbb08ac58b6e0,07088c6e343e6151,0769962f148ba915,076f44d20bfb8b4c,0789b52639d48217,07d65a4b5b33a144,07f846d32062e6ff,081395f05c751028,0827f3fa525574da,084308e18a0cfdcd,08554f400a34068d,08a3bab54ce977d8,08cd65c6351fc940,0935feb169158718,099b1671e085227e,09b98a759bceb011,09df477282146bd8,0a26d2dccc5617a9,0a28d0c6ef0a2f47,0a9a85e786a50d52,0ab6d2f85d123ce3,0b085eb462543de6,0b0f1b23632a8d57,0b4d6affe71404fc,0b5417a338206901,0b6b02324f707b72,0b711a41363060d2,0b74869930bf2f63,0b8719d7ab3a834d,0ba18c194551fff2,0bf53e152647b40f,0bfb3fe0e74155d4,0c24a52dcc198f37 +R-pr2092 00082d54883afbb3,002230bf46e47704,002353a84839d6c7,00386464bed2db16,00737712c96d00aa,00a5fa644eddc22a,00ca2534e8364be2,00f90dabdd4b822e,00fcc623f7b6957b,01a27805f0f5d74f,01be7178e1a24345,01c1befae503380c,01ee430f2bb4d452,02013ba50d4d540b,0203a0e15fe7b848,02061519d3b9d1a5,021529c039a7e345,024e286e08829c60,02b1b1cbdcea7bc9,02bacb1881b3af43,031a652155261149,031f2165de8b61ff,03594ac30ec64d4f,035adbc356c270b3,03ca42efa038e4ce,044f854b2dc8dd3f,0482efc8e4cd2a2c,049c00df74225709,04b07765f4924c1c,04c0793a5c67eede,04c437e8577b446d,051bff2f645b263b,053cb0b01a64c328,0542e69f7b17b85b,0553458824380438,0556ca89aa44cf8d,0559ade66825fa52,056c245f14b3021b,057a49ab2b21272b,05a50db36e4b0a51,05b57dd687e36961,05b9649b4549adee,05ba1511f274d37a,0630c56cb95b3931,06403f0e9d2bd232,0670f26c93f6091f,06cfbfc800bce37b,0739233c1dbed73a,074217780df69dee,07957f6935c5e4eb,07befd3cf2d131a9,07c496800997ad88,07de7ca535f1a8b5,081d1d2605c01ce1,085e10b2fa9b9f85,08618691e72b90fe,0866207f2df62a84,08c0a5513e9ea1cf,08cd73c57fff2244,08e9a0d0165f1869,08f55d04bb086867,0906659fd5d3da08,093202833cfafca4,093ddb5f7c5088e7 +R-pr2321 000a72385555714a,0058909e1c3e847d,0062227aeee88345,00bca3904dee4194,00ebf127c8aa2c6f,01115ab50070b542,0159e9867cede317,017b3d0704b2d823,01ab5c2bc89b13b9,02b4a06a4475f40b,02d3ae952dfe3cef,02fc877fc0ff788c,033c64e0462b6e12,03538268cfa8f78c,041e6ff068fa9e33,0465d6bacbd75537,046b4871d6c11fe1,0470b58f357177b6,04e73ad16086f489,05f35e5a71b3b69a,065e1f15095e9445,068d8a817767ea92,06ceed1f8f4a1efa,06f85ef4ba056441,08afad969a5775bd,08e006b2bca0d6ed,094cfcea801b8924,0975f78c3eb2f53e,09b536f4a8b2f292,09f1a134bb54349f,09f9fd794d901dbf,09fbdeabfa0538e9,0a4b57c03bb72c68,0a8979d5d387b404,0a9190a1f520fb51,0ad1c7271697dbdc,0afedb6d3abe1693,0b6eb55995b35d5d,0bb49643d17b7599,0beec10d87e62aa7,0bfa53d6fa6a0308,0c2f22530f4e3b74,0d6b68bfc3241f99,0d987323605156bf,0e60057963e52091,0eee9d83426b8c8f,0fc989f487b3c82c,0fc9fb8ef7b20b8b,106e8b7edb10a73d,1114e93ed4d3c6c8,113f01c4c72bd295,114d709eedc81145,1259ee8ca1f76d08,12f7f92604322098,13a9f1321b6f195b,13b0530b131c49d2,14566529df6b983b,14c5b2b511f8e592,1529f9a6fdceac9b,1539029655fd1a2d,1546edebc0b3cec8,15891fbb4159e5a8,1593fb4db0cb42e9,15a67d2d98491b87 +R-pr2365 003a2f2b20bc68fd,009990c4b465d908,00b3e1228fd9d5a8,0257e3e6b1385a18,02de1af095e023cc,02fb0cca11caf2b5,03323978c8de74b8,03549fbb4624786b,0359c968e9bea367,035bd7187ec03a74,03a4501cbebef24e,03cc25bd1ca51cf9,03cd6121efe2c0ae,041ab42b3b9a3bc7,042cd0418882693a,0432b44c73a5709a,043f2874bbbdb677,044586f47ad5d881,04521b6abd575cda,045d656ecd6eb92d,04c1ef17b2c84b91,05184d12da21b71c,0534b6fbec622ce1,055d96360d6baac8,05b3f0051b656162,063fb6ae59f2d6c4,064ee3aaf756c130,069861526f167389,06a15535ac5c9b9a,0710254cc1c07a63,079fff870cfa6497,07b5c6584eb66268,082d2b1390f173f2,08be2f73b4d607ca,08fbb9df22811837,091a3d68eb34cb8a,09490b0ee5069a40,096db8d6286a7553,097c545b5902c57c,099a8615a0d9f360,09bc6028e9cd9f3b,0a09a7da012af4fb,0a0f83f299856f39,0aa0d4bbb77d7c15,0af7868b578c304f,0b296b2ebe549354,0b5ac43ed415f384,0b660eaff63baa68,0b8bbf9700fc070f,0beed5b2ceda3cfb,0c5265bc329f63b7,0cad2320e681e2d4,0cc90dc43d62d83d,0d0c0c7174963e76,0d866edfc3e7af3f,0dcaa71447cadacc,0e071cc6b3e025f2,0eb48a055a4b59b6,0f1211857a7e4b74,0f22fda670a6af6a,0f3de889e1c34d9f,0fd259608c5c5cad,0fdde5639c43a115,102f05e9d181c2ac +R-pr2368 0026cf9694399318,00485ffc60aa622d,006a6aacae910f0c,008cee9d9004ba42,00a568114fc79a26,00b69102db69abc0,0121cf9865f7bda0,01682489dff6e7fc,0173ff7ce5b5eafa,019c299a24872a63,01dd7f1a8e3f9c62,02341e150f671695,028cf8f92d4a8068,02e0e7b6059e9b87,02f709be017dee35,031c20bde258c9f2,034313d50271d4cf,034de4bfea2ebcfd,0387c3f9fb6e04bd,03f30ac81b70cefa,0472662c2b6d7836,0483846f0b6c6796,053e7d4e0d3e2544,054e2754ba0be9fa,058e1bc33d181fd6,05a20624ef32614d,06476c3941b446de,06506a33393c6cd3,06580555b9b3b4f3,06894bf7d6b3eac6,06931ed4e89daf99,06a2f3cfee6b2eba,06c8a32db7eeb35a,06dce231cbae26e5,07037a33cf175710,0736e5de60e6d8e5,074834006e7cef10,07520cedc6c147b9,077a02528f96fae6,077de05d6ec6c92e,078cc6846a2ae242,078fa382cbb124c3,07bc6c6a9703afed,07d371c28af36446,0801692dec88ed5b,080184008f373e0d,08257b63c0ef4aea,085661564bd5e7de,086aed6e0d21dc9c,08a4bc5505149105,08c6a4101b128e35,08defcd8df138123,08e9a0d0165f1869,0928f40b9644dc50,09504d6ca4de146d,096a4adb215d341f,0980da7628ff8ffc,09ffea155daebef8,0a40ad4cd86e6da7,0a520cc619759e71,0a9b9e134c255855,0ae4b4c586f5f63e,0ae70e4e1ca532d3,0aeb7216ee4a2cad +R-pr2415 0011b17d2f3e1475,001e2dd4bba4643c,002a18e8cfc19a2e,002d379a96fc8d61,0031f041b11d641c,0042e14020488c84,0043f8b2c508abe7,0077ccb9de6b5b5b,00a81d57ce9b63a6,00bc9b5ffe486a95,00be53d69e3177b1,00cdc9fb596b01b6,00d788f38e1234e5,01002177601d7afc,010bf6cddc4b1c6f,011a123eabc00b73,0121cf9865f7bda0,01354f49c2b4ff9b,013e7693ba15fddd,013ec33cda811a81,01415be737b33365,014bf25828552899,01b5ceb50af83bed,01f2fe06065d7fb8,01f8aafb6a6322e3,0203ed2b56ef984a,0207df85c81f0303,020a8eceb83a9310,021529c039a7e345,022e820df73e3869,024007b51de227a9,0246eace4e4cd098,02477421db56692d,0263af243f6b113e,0290f807240e13d8,02948f4521859bc9,0298cb33133309bb,02a3299adadd9d7a,02a84a7c40e02c8e,02ac9ce648fdef9a,02be02ab3535ff34,02c92d7bb8f41290,02cb5c5ae71a6c0a,02d6990fd694723c,02d8a2c693e2b4ae,02ec4ca0ca9d6401,02f5e8a11bfc844d,030a1ea27c0c3fd4,034005fb40e356fc,0343795e53af105b,03518b2c362ea3fe,0360d13f104a1d0a,0384dcf9b8807913,0393b5993d720367,039845cd561fd850,03a26f631109082e,03aec6b5687f7d2a,03bdca125bb6bb99,03dbfa2295b02501,03e50c95eea7db38,03e7c166d857f81e,03eeb52f14d37917,03f2ad8835490d8f,03f3a9c566ef17d9 +R-pr2416 000225fdf97e68e3,000a72385555714a,0018e7e8b46f516d,00247dc312188f23,002666fd3da61777,00276524afca7f5d,0027661dd7e657cc,0028813ef5c2ff17,002e94808c0c14d7,003657da38ea4d2a,00634f7dd2390c39,006486f33ad6bf62,006a6aacae910f0c,006afbe41611c1d9,008ba62859e3d507,008e14a797224c7e,009218be48141ff4,009a708c98372d2c,00a568114fc79a26,00a6979711f604d4,00d13bb6a5ea037d,00dd2c758255b487,00f0e6ab440dec5e,0118d598eeed4a08,011a123eabc00b73,0131f316b6032326,01426c921d06e922,014d044fea050691,015038568c8dc380,0157d971a83db2fd,0158a16bce01b20f,017c2e009d3ca919,018c2eb012a33e04,018e8959fb9a276c,0192e0c6b08ef707,01957e7759289c15,01bfcc318991ba13,01f4594c76f21afd,01f84d46d8af33e6,021958691835f9c3,021bbb14ee3b4046,021ca01b7aa302da,022b63f850987b93,023c30b0da2579d6,024007b51de227a9,0248676c2251f182,0248c41f443c3ee9,025381dcacc76422,0258d4a3cf3e17cf,025d282a04d78c58,02857d44b3d00357,028cf8f92d4a8068,028d1971cd52ed3d,02998b50fcc4d9ba,02a2b95c422d098a,02a748efdf5c1c58,02b0ff3aa8f2db05,02b3133dd4a26c27,02b7eb923ff7e661,02b8fb77809eae0d,02cdc379c829a168,02d35621ef6c8799,02e89053fd5351c5,02f4315143b40774 +R-pr2420 00010d430eb15502,0003829b0cdb2bd3,002d8428655b8aa6,003b3fc011f2c640,003c25d849ddd55c,004e99c832a3207b,00555e82c752ec1a,006b55259e9b2b3a,0070c6b2a3ad39f7,00736570e017174d,008f1a004c9ebd76,009730c4912fb837,00bf29128020cd20,00c5ad6bf9ae66ef,00d0fd104ac6152d,00d9db6f034b2cf6,00e29a3938208baa,00e9c954a8536074,00f6cdf2e667ab71,00f73fb7a0181e99,00f7f84a84eeaab7,0100e007463b8116,0110ed9bbd825b43,0120a8321db31d4e,0125fac73b0963d9,013930a19cba1bd0,013dce974f4ea2e5,0147f89897f49d4e,014dfbad5a0cb553,016caeecd0bb5e74,01741fa0e9875897,01759f7782bb87bc,017b1589ee119e53,018c3d7a691b11c6,019a7bc02b796ea5,01afd20bbed64947,01b5c51bc9426598,01e440713c8609d1,01e88de998c80350,01eaf8250fa930c7,01fa229283896381,01fd8302fb9f8063,020482526f7f6cc6,02070f9931023b67,02083f59e366ba2f,02171380f2545e9b,021b5f8d9db7e846,022a398eacb6a831,022df2024e6833e7,0230360cd9c341a8,023e27952e442241,023ff4822a4cf13f,024759767b3708fb,02620f2efec1eb73,0263d7a8397b3a5e,0290a3a884a92c81,02a25d45f3aa6906,02acc7e177fbabd0,02b12a118b1bc223,02b41a248b69d848,02c2c262a14b6f85,02d2840cfbf74e13,02db1ea88a36f8e8,02db3954d8ef6f61 +R-pr2423 000376060e53e37f,000428eeab98df09,000b8ca2d59d9090,000fadfcb8cd2cc1,00196bd7f7d96cc4,001e9c4b59501c17,001f0306114f17a2,002326560fdcc3c6,0025731c94fb1277,00262b2ad5af0951,00315bad2d55cdad,0049a3fd386a8d2b,00669e4e16f8f712,006e65c93a21caa2,006f5800ab5f80f7,0074b5858cfa61b4,008cee9d9004ba42,009c53fcd8689f36,009fd48eaa9a83cc,00a10e43752fdda5,00a4d7a7e958365f,00aed8e5e067fda6,00b5cf6363c85270,00bcb241e28778ae,00c7c69912e9e346,00cb7a09b3abe8a5,00d2be101b08c4e4,00df9ec02ba323fd,00dfcb73ca766a54,00e0098e3f2fe0d3,00e04027adba74f4,00e27e519d968dee,00ea70af15a1ae94,00fc8f85d642b31b,0104b7bab6fbc513,010b571384a150b2,0110727f696382d6,011849209124deac,011a123eabc00b73,011a136578cc4f3d,011e47f4aa116566,0121cf9865f7bda0,01251dc3f0f9d315,0127eeff18006e7d,012fe11b89c1af5c,0131240d476d9e3b,014884802a5f187b,0150abe6069897de,015d2b3027131799,016185a811786722,016d27b055116382,01706da425bbfb1a,0176adcdc44128ee,017e5be76f54675d,018ea6e7d3662447,019c1eba3667ddb8,01b3af2648ce8aab,01b3d77e55513fd4,01cb56dfb7ea609b,01d1fb367b472cc9,01d897513c1d67d5,01db39d87ab0440c,01df0041bd3db1b0,01df230aef7496cf +R-pr2424 000aa1761bcafe07,000da4d68896f219,00175093eec276e2,0018fe5b79049aa4,001d57b19e9c750a,001f49da07fd2286,0026726d270f1c92,002e707652b7ef9a,0031af6b5f291e00,004429af72687d77,005c073c46919741,0076d027443431a0,007bed9fddce790a,009471d2b4945d56,00990b5e9c1dd5ee,00a2aa4a08d5e4bb,00a81d57ce9b63a6,00b5b00c9155e509,00c3df6d8f91cf01,00cc8af616584e30,00cdc9fb596b01b6,00e99b48a887b9a5,00eaa25bfe83cd05,00ecd62897596dba,00ef6bb8b608c4ee,00ff51f1082fb14f,011a123eabc00b73,0121cf9865f7bda0,01303be754da34d5,01338fd39d354f3a,013c148dcf45a569,013ec33cda811a81,014d2c115e6ad163,01522b5aa8f5834d,015728fe3cf2885d,0174fa4ac6022e67,01770d3e8465d585,017c1464121046f9,017d56b0d2680052,01bb5feaef9f4f74,01cb5ec6bb46d722,01e440713c8609d1,01e61fb27b0dccb3,01eb76edf2679701,01ec1c395f4b964f,01eeba2553e5539a,01f3f907de046dc6,02118de52c994dbc,0213f5db93e0edaa,02193dbd5d8ad39b,021d35e7d5c9d214,021dfc5c34ad68e6,02251a89997fdc44,0236d45084b1afb5,0246eace4e4cd098,02503f237231d16d,02647c3785964652,026861225139a34f,027be78db6c16c5a,02a01eca17280163,02acfd707eca0271,02b0ad6640e04b39,02b21d3f4df7ac38,02b5a5bfd728477e +R-pr2617 00050980e9ae0993,0021ddc13d9de0f4,0023a4f07741fb1d,0030d7707c81c191,00402307ef22557a,005fdd97f7bff0fe,006eca08cacb855a,007d9247d273bbbb,0080151320ae63e9,00806ca19ee16e21,0082fd0c0949ea19,008cee9d9004ba42,0094a81ff1ce25de,00983e8986c8c242,009ad01bc282c858,009cc7cde1fa9ee7,009fc8741b3d28c5,00cc8af616584e30,00f89e17b54c8539,00fc7a10f65dc48e,00fcc0f0d8ebe408,00fd519ae79b62da,0106d8d56a91bbe0,0108a061e59af50d,010fbbea125d5e81,0122fb74a920a862,0124d4d490ea66cc,012702cc28bd0121,0133dcff25bd8d6d,0134a2ddc9b8885e,0139974bbb180109,013b0dd2d27f4679,013f398390de3061,01432e07f8caba5c,014e2e00cf6af437,015359fed064dfa1,015b28cf9ddaaff6,015cdd2d2f976b92,0172cc181f610221,0173989ed52325d7,0180a9d484310e90,018a47b1926ffb3a,018b7e5933899760,01a8540f380fb8e4,01b19eae8abeb0c6,01b77a224df8cc64,01bc5ac160ae7610,01c71e365dbdc6f0,01caf0e85636a5b0,01ceb1fa35fb2876,01d6af920d5fc4fc,01e1e33945d76949,01eb51f66f1f19fb,01f9492599161069,0209cb59ce275b98,021e8912fad16c2c,022d861caaa2d2ac,02329f11a45318c5,02350d9b222b602a,0244ae3db5cd6224,0246eace4e4cd098,025174a70626f816,027d2ce6afd4b13f,0281a61ca6ac8cea +R-pr2707 001d608fa96299da,001df5c2d9cbf476,0044af679c0e9adc,0046623124216c9c,00762798caef55e5,008322d6e0da4580,0087c68cff298807,00ac2df35955a0c2,00b2f8fec0e05e0e,0107be2726deb662,011a123eabc00b73,012a96e2a3258f40,012b7bcbf7b86276,013862e0a434bc88,014df8062336e9ed,015021fda9f66fe5,0157021fd769e3f1,015825aa1fa2f07e,0159e9867cede317,016185a811786722,01657b2a6a24f167,0166c2c1d3d947d6,0169ccf5c5f1b188,018c175084d82554,018d0ec9a768ab48,018f8ae4d66a1c32,0193c9e81c47dc81,01d03a4cf5bc9947,01d1a26e5fa5e77b,01eaf32b11b5ebee,02011563de07f524,02020f5ef1ecac8d,020a69bca52cd368,020c519a02074ab5,021509ad964ddf99,0217d67a6b20062a,021cdc1fa8a5b6b0,022540674671d483,024715fbeedfad2c,0247e55f430e1c8a,0254566db50cc1ee,02618ffa31d157a8,027970e78c23fbfb,0290a3a884a92c81,029f8dae82b74443,02aa58e2ad5df0e0,02cdc379c829a168,02d2d860f8e5dc47,02d74e050df69d7d,02e017a8870594e0,02e622c8daf90d1c,02e7ff4290a628cd,02e9b53d6ce9a3d2,02fd83cac0b7f57b,0315a4d5ec9d2c26,03187bb6585bd63a,03293f93c91bc58e,032a1239a1bf6c33,032df00de5f686f8,03374d4c440f7466,0344445f5c1bc6b9,035a3ba1918aa688,0380d6b4cef9bd67,0386094f314db25a +R-pr2809 000c3cff3019f92b,001deea66d81828a,00312a6e63841306,0042e14020488c84,0046623124216c9c,006f80510c67d863,00947101448d439b,009e2ab89b70525f,00c389cd4c797a16,00c8a76df06f2387,00cc8af616584e30,00d1a945d39b7962,00e543e3f8906169,00eb3707d727c487,010cc11fcf72cb9e,0111e5800393b860,0113992a176d369c,01182fed5269ca5b,011941de18debf22,011a123eabc00b73,0134128dbbee17a1,014925197461abed,015977f67291e711,015d16d289c4230b,016185a811786722,01676c24940c55f9,017241ce7860dcb4,01774e4b1541bfe8,019ddafced4e60d9,01a4d80c2edb65c3,01abbdcb1344bca2,01b594d1632b1721,01c3176d5ed47f05,01c805da3cd343fa,01cebbc8a3843af1,01cf0b660e95095f,01d94534daf8846e,01e1fbeeb557ba96,01e34fe514e1678b,01e38a17c25153a5,01e968b582f12cf7,01ea653fe7657a11,01f38306a268900f,01fa0f3685360053,0203917f6129639b,0208b38aab006b30,02099324349cf0e6,02194f74b0d1ba9e,02294d7d9b2190ac,02373ff42e1372f3,0240b2852c655981,024715fbeedfad2c,025ab0bdb83b8187,026889ad8d590b28,026d7004670b0de8,0276c005afd7a7f9,028119a4db0c9bbf,0288810bd310f552,0290d1623f37d4c1,02944ce89e95aed0,0295ae5258a7cf60,02afe07cfc0bdf9c,02b794423bb64466,02bc91ee9aeb9972 +R-pr3008 000ea424553a1029,00464b09d1e76b93,0081b3d496b517ee,008aae4fc7e9c415,008d385edfa82aef,008ef6598ca58ce6,00ae5135474dc7a1,00d8871bfadb1376,00e8984de73a4a7c,00f8d7792139dac5,011b047470d13fb4,01461a529723e13b,0172e89ab0968e0d,01c2a489b9682555,01ecf7d903630b1b,01f8c4baa509ce0e,01fffc04faf1440c,0264cac0953b8ffa,02680c2a3661b06b,0295103ec6bf494a,029fe5a72c4a2be7,02d6b9db9f1cdf8f,02db1ea88a36f8e8,02f32e05afb98fb4,02f7d548100f3051,03187bb6585bd63a,034f7d1d01cb15ed,037345cb4f21ce06,03a7bb99afef476f,0464dac639b4c4c8,0487e5a9081e4391,05066979137f7034,05202e9ae43528a4,0530f751165909db,054b68a15b654713,05ad8b52a9d40116,05d16d95e0365085,05de81b31616869d,05f0945a98c932e0,060e3588ec68cc6d,060e886dd0d357a3,061d5daefbde49c7,06396a76392e1e4d,06b44c7e5033e476,06bb382ce508216a,06c0c8dcb6cc472c,071fcaf8b2e0fa1a,074adb7bb82598a9,074e80a7b20a9b2f,0766dc625f3034d5,0786db7d54eaa4f5,0794b2bae776ef70,07c1b8a54d655a2b,081f4596bdc9c826,0826d62bc62d4c51,082b22f1774d0991,085fc19587bd8048,08624567741407fc,08a28185e33dd2f9,08d6c361ec4c6438,08e6136fe9ad2e7c,08f82c2b6a837c3b,09156f280aff0c45,093eb396e741dce3 +R-pr3030 003a2f2b20bc68fd,00762cfd5bf5b9b9,008e1b0b6895c3c4,0093efca36ac8dca,009b404381ae896b,00a72fc4b3a011b9,00bfffdec6bce9f6,00c6de07cf8b6173,00ca22013263cc0e,00d0bc3d28a48819,00d0db831c302fc0,00ebf03fc5fc2238,00ed0fef1078d34c,00ef762217915c23,010f265b4fd00993,011a264c09b3ab80,011d3e660dcf8626,011da93886f76fbe,01276a15be6a93f8,0146edc4b77d9972,017aa4601be3a9fd,017b4e29bbc3f8cd,0202c620605c5fc0,020bf87ee4eb735e,021a40cfc1484ad7,0242a7c3c74872d4,025fb49338a19392,0261b80103d834c2,02780d2d47a0410d,0284004274c46cca,02a175a15a1c7249,02be7864cdc91663,02c6ef6248a97cc8,030c26afe4683f13,038c6627b332eee9,039410ba20bde9d3,03b537a6eb25476f,03cc90671f98e060,03d9f85c78c38067,03f59a9f7e20a6a3,03ff6e165df6a4a6,0408289dc0e07ef9,040f6b627b3af082,04301fe8099e030e,04928fca16465ec0,04a907a116aed9fd,04feedb38acebfa3,052650fc2f06b1cf,057fbc7d4dc70747,0583dbdd289c96ab,058b9842a8b0e3cb,0599ff0578c17f99,05df3f5162702aa6,061cd9731cec1e33,061f6c5a9e3270c4,062979ca7e3271c2,062fbdffc302ee2d,0652c062e1856a8c,0699ab2a27a46842,06a459133d3cee3d,0708f03d29e1a3b1,0743276f82a0b126,0745738cf7256ed7,07852ec9db157c21 +R-pr3048 003a2f2b20bc68fd,0056dcd0ce2a2aa6,0089da9fdf49982a,00fa56df2074256c,0111b374f180d4d8,012428ed8a0111d4,0145f1dda22398bc,016d5d3a0f23e1c1,017f8b784497985c,019514d82f9f7779,01cdac2080f77fd3,01f04c6f7218bc92,021385a4a5f84669,022109d345046f9e,024007b51de227a9,02aebb2da415a0e1,02b8e4d6110bee39,02eab0e4445306aa,02efe15b8a76d9be,0317311310f3947a,032ad0766196e5c3,0369fb0a8de212a8,0377c80473b3f02f,039d7b1bd49116b5,03b32691cfa47451,03bd015fc25b7065,03d0973688a13359,03d5fe966baa1c6f,03e407de2c93520a,03e76d5e72d32c71,03f504f698fb7987,04077a85e6cbbfc6,04083c61e2fc82a2,04093c16ffbe4fd9,042282c0d20cd25a,046e2ac6adcb7001,0474781a6af7c647,04761d3e881d17ac,049e02d8b32a120a,04c0d6a7475396fe,0500ed3da5e99ad5,0502e397a34c583d,0520371c248cf9b8,05325b0364e4e378,053406ef474ede3d,053e9846514b7c83,053fb3b106f69c53,0552942598f41133,0570259681a2439f,05720243d25b262a,057c010c6eaa5a54,0587795e82ce472f,058d23117371e5d4,058d83528b44503a,05a244cb54880108,05a7383f7db99220,05bb0b0a02596b86,05bcbbf4a9f2c297,05d5dd01d5b349b4,062067d69ff4de3c,0623025ed009b810,0626d4124a7100da,06351d3a3973cbb5,06366fc160ca9ee6 +R-pr3106 0020ae53a0bebcd6,0058e04219858f44,00b5b00c9155e509,00cc8af616584e30,00f5fe018c814dd7,01123d891bec72d9,017e31d5959fb5f0,019b9f1bb359111a,01ebda77f678f127,020385ec21389ef1,026185c92af394fa,02ea384b3e981e02,030f2fe2b0048a98,031e8c88eb2c2666,0337dbae45e55d03,033a7cce80522fc2,03b812d5cac5da37,03cf88fcad938802,04042cbf6382a26b,0435df33580eed2e,04486dac5fd7d9bd,047bef5f4ad99026,04ba490273d2302f,04c7db12c352ac37,057a089a571402e9,05851cdc593e8460,05c798a512deedba,0627dc9e7f581942,06481fe3a3152248,0650c71fe6c732bd,066bd10f862d85d0,066f69e5d3fee6b2,06a6b450526b8784,06abecc2eaf17b39,06c3270a15270cb4,0743890c0a0e9b26,0760c2029ba60667,078025e6d67bf650,0786a76b9952ab22,079aae2f3958aa3d,07a3dbc022ee2f4e,07b17c7a62a0f930,085d90408a90c01d,0877fdca68b598f9,08815bbf1928bc87,08950cf538ecbe41,089fb416bf93a9a3,08b3aea6e8e6f63e,08d7338f93e4bf5f,08defcd8df138123,090af63abee1cb3a,090c40deb918540f,092b2409c9ba4576,0974e2620d3d9521,097a7f707f082c92,097cf08f2faa4447,097d7ae6f9b075ad,0993d021a56bbf40,09b782d40c89adbc,09d02a5f722712d9,09d0af57e7f18226,0a0f2322bf93f3aa,0a1f221d1651b3f1,0a439b6fc9742f92 +R-pr3216 00019eb5fba780e1,000fd00c1e0d17fe,006284906cb24496,00868c0863d20de5,0087d4096dbaaca6,009b99476b1a8ff2,00d118e892efbe3c,00d1a3ba4e865b38,010403319667523b,011200e9e34cceec,01554f8780b7a007,0160a99ca3ec5267,018fdfd171f72e95,019d17ebbe86e697,01be60b081c6b9b7,01cdbac2efbbb213,01e6bdd6ac73d608,01e8cf4ddc583a99,021f5f74e70ffe47,022ded5b39c32bd9,023c6a4715791814,024007b51de227a9,02612d431b7feccd,028a7bd454efe447,02a2616489db1669,02a7db2858d3627d,02ae6a8608f08ae5,02d67d29dba6cb34,02e98ec6c4d954a7,02eabf23bdf7a699,02fc67427adb97b3,03126a0c0b268c37,03180d8defa9996f,033a03523e38e3bb,0380167f06605046,03a82d3dddfaabac,03c1ad8d577ba951,03f65990c62995c8,0401ab468d896469,0408289dc0e07ef9,0432b44c73a5709a,046dbd16ea1e9b1c,0474e00669d03cbf,0477cd0345d7f449,048f9495e27e9d51,04a094cfa04b1cef,04b96831275a8bf1,04ce29c60ae1aa83,04d31a41ae3942d4,04d674ae243bb7be,04f0d4779c71cbd8,04f8c5f603877131,04fcd11d925053e9,0506a8aa172b08af,0515b3cfa142caf1,053e4e3f170bb5cb,054d2ccb03b662f5,055f473ade88f462,0565546c1c523144,0587cec42859afb7,058826932c209eb8,059ea0925f0d449f,05a119ac0d16f626,05a244cb54880108 +R-pr3307 001e21626b2c27c8,0025be343fb07f53,0038f5626735499d,00397d65bd634edc,003ea3a9666fd412,0040ba39b06168f7,006bd1a5290d5b77,0075625b0dc1200e,0088034c5aa7e56a,009ba2ba7ec08051,0106c6e0c08f5884,0121015d9d31a9b2,0121fa743d401401,0148c05380f90d9b,0169e86aeb5673e1,016a4b2af498848c,0197065fde403ed1,01a54189fea8a104,01b08be593c4d17e,01d0f83f52e81ba2,01f9ee55cd78b93c,0224237eac883c0c,022552fa43608792,0271eb8924e04314,028efb7a87b096eb,02e66170870fbd1c,02e6ee6f4a07390a,02fad7c4fe2b0235,030b3d56666d5a02,033cc8215ff1fc52,03543ababcdb5a47,0357fb205d6abba5,036518e4d0932832,036880ad8bc4319b,03851e7c88496a10,0396825a967a3e17,039c5247f0b70af8,03b9974ce6c3b154,04521b6abd575cda,0454aa78c7238167,046a5efb7b099137,046adc2c985dbe64,046cb895c3b8ca77,0498906d18a1b2e9,04b336f01446650f,04c728cddb439823,04eff842a73e6879,04f8499d18534a86,050095a61b3f14e9,0502e397a34c583d,05106bdd2e2bcb3b,0528d0bad51fd902,053628071ba6f49b,0543ddb1634bce76,0545cbe713ac42d1,054b620659b668f9,058d600e9de334d7,05a37f9866527025,05e273b083a22dd2,060c24a648fce438,060e1b77c1b0c000,061cca727592349b,063517a108983cf3,06366fc160ca9ee6 +R-pr3309 0020faf88152da38,0059caebe73bb9d6,00948317cbf37b11,011998e0cdd5ab11,01975124ca6e0413,0218ecb8eff7bdde,0232d5968f14ca15,027c7c1abeb7afa5,029befc60486eb78,02b418e101d7742a,02c9825881595445,02e26a508e574bd7,0338d70485da77fd,035bb87eaf458d4b,039779c9ab4d1005,03b43f5f8f038325,03e82e23954575b6,042ea0f5ac69a87c,045846c60268160f,04bcf6618d48ba4d,04be19ff742a0058,04c02b78ff345852,050c82742f598c60,054323103c24b0a3,0563c8cb285d116a,056741f642a05d13,05b875c54118c49b,0628d7e8ba509fa8,063cfdb56badd7a7,0647827c2c6c99a7,06816a2d7ed5d0ff,0691863a21596b0f,06932aea9b372ed3,06c3270a15270cb4,06ce60a639f07447,06dcc2bf2e46d449,06f3e03d53b18b83,079fcdeea20ac8c0,07a4e1621391692d,07b155f9b7c3ce12,07b77b6b92386f0f,07c144a11ba000d0,07d18886512fd1bb,0836a18904d7d910,0849123916cd5c05,08be45dc535f7c01,08cb073b8c644880,08df1e3f1d6aa979,091212638aa979eb,0913c8942893654b,0924b2b38909f1bb,093bffc20fac98ca,09d13b0a517f3d47,09f5634abda3e102,0a0178a36900f919,0af9c9950fccdd12,0b2a289b21c20b47,0b4cc927a341144e,0b5b969af2bfc705,0b9202be0df8da45,0bc1d7ec6e449a5b,0bced0b5617cef52,0bf40590117c0ff8,0c21f0a231b92117 +R-pr3319 0042aca04c13249f,0055e2ef0a830326,005a425e495c11ee,00fb49a9a92f3c5c,00fe3e932d2a30b2,012274011e96eadb,012c6548b3e3c628,014889db31acbc19,015af4212dcb3924,0160b851c5dccd6f,01a4ae8feec4e1fc,01e43e3b448e67e7,02275c0ab0baf75a,02f22a65b9f163db,02f76b86d8469ea9,02f9ebb0ac1974e6,03537cafe0a1950a,03b11014d5a82afc,03cfcde06a1f8020,03d8a1c9b890c8a4,03f65990c62995c8,041f523c1b7f60d9,04539ae9a6f25e91,048c0c7939390c98,04fc9bc01402522b,050d76ab4612d329,05461a5f021c3c58,05461a5ffdaee1f0,05ae3d0b3c7d7664,05dc1186b048d6b6,067aa9e7dc840606,0692fbf11ba6ca2e,06a516f8c8d10330,06c3270a15270cb4,0710e47e1e37a73c,075270522d1f47d5,07b77b6b92386f0f,084add288c91e521,0856c760cf32c04e,0859be71351ec919,085a8e0635d2b1d6,086b57e53ad31254,08784e5b671d2ff9,08b5b4793e61f61d,08ba2cf70a23c916,0956d5019eb1cc74,09dce12d468e504d,09e79b28dfcf217e,0a4f1e3e964c0608,0a931f35b368513c,0ac963bd77c25c63,0af9c9950fccdd12,0b2fbc8892c63674,0b33cf682428b0b0,0b7384b75b09ebfe,0bc30789797f2330,0c764a2362443479,0cccb209fdec457f,0cddc2c15ec7b9cd,0d25963ed02a8cdc,0d4587a040c7ef83,0d97a61662998d99,0d9e468ff2a8a60b,0dce2e395da5878b +R-pr3339 0008d97feca332d7,0011ad774d6e004a,002289f9dbd9bf65,00258ec4e1efde6f,003137f1af43e04e,00381630ee16590e,00397cdcd3b4b641,00422c2462571cdb,004a9aaa623ef781,005c0395e94d5252,005fb4599a5f0451,005fcc2a54ba8334,00628926ed33982d,00637ca5ec3a7dde,0064a9e7ebc2c8de,006f2ae379d86b5c,00759c0ca79fd2cf,007795cadc3a3c5e,007941692a53749e,007caabf0259943a,0085dd09433a66e1,008a9dd84193c42b,008e5b8c49fbb0e6,009679b75a8ed4fd,009ac80d6e3b3684,00a4204995352ec5,00b412217a8f939c,00b7aeafa08923f8,00d89dfa3d93bab0,00ddc31ab4e73482,00ddd4ce8119ec9a,00dee0f13587c6e9,00dfbff2a536d1b3,00e7b918330a53cd,00f0dc14b6620818,00fe31e53059b35e,010b7f064a171875,011421e156bde1a1,012274011e96eadb,012eb05c1cc2216c,013ffc9fff9e0d91,01419157d14d2490,01568d632a616b3b,0156f02d7c7f4788,015c42db7e9a4766,0164728d52916479,0169cd6f926977af,016e78637d45d253,016fc3fb285b6ed9,017f3a2a62b30a0e,018488e339c7ee2f,0191322a645b8642,0194b65a5cc107eb,01a0f6e9b968b904,01a3c1c2f94c75a3,01a6d905c9dea7cb,01a71a3af8a4ca85,01a7a117bc71f75f,01acd023cdea2674,01bb5cbdf4f85d69,01be01c93288972c,01be95ce2eba060d,01beab3daada21fb,01c120eb4cd35ca0 +R-pr3405 001094c976a02ea3,0029dadb7fd2757e,0079101d2cda375a,00bd334a416cccc6,012702cc28bd0121,0170c8a3f7ad6935,018706a1d235d154,019ceded31948b17,01a6cbe6fd8c2abe,01b4e8154031155a,01bdcb70058e5551,02183ee4eb1e1e3e,023270b750efe0f7,0247a63bd62eaae6,025dd00086395b55,0294f05077de30d9,029baf9861c7289b,02c1cbc332772424,0301223f81bc3eac,0312948258da49de,0347f6c031d237a5,034efdea84a97894,040bb5cffd4dcafe,040f257a9a0f8d12,04204cb69ff00ae1,0422a5f20897aa7b,046baa73aeec28dd,04a1e061a57655cb,04b8a2ebeced6e81,04bdbdc199118017,04c89f23387a16d0,04dc72f6206e9954,04ef92977947f245,05027b7b98fe5d30,0523fa23ca5613f4,054e3da628beec58,05918e1097908e5f,05e8622363fbbdca,05f61de20539a111,06115dbd2196f791,06366fc160ca9ee6,064cb47dfd25beae,069e19d5816d5ade,06cb161f14225c05,06d72285f40ea2d8,0710ce664545d8c5,0711701b3a475229,0727cffbe2b27434,073921278e97a27e,075cf52aa5e64161,07bc74fbdcd10021,07d3a4f115f380d8,07efbd511e849e52,07f4fb7f4413c503,0805dc10ecf3426d,082ff4516168369a,083f27f7c979035e,08453297bc4f87fc,0848b20face83413,0867e6711c00f705,08720bf9fd31c59a,08c27d869bbd246d,08c74f6ccc05966f,08ca669ff7bb4f5c +R-pr3469 0006948803e05fc9,0012e630745be5ff,001838027e9029d6,00a3255a9ce2d44a,00b477e674537773,00d7a455130283f6,010dd4adbf80acc0,0116214a60a81616,011da93886f76fbe,0130fb41526f544a,016609380949f16f,017189cf5429e687,021656f435d11779,0227336d02c12673,02275c0ab0baf75a,023e67e6a0b3616d,023f6125d9d78340,02aa1c04fefbc5a8,02e5256a50a746f1,02f89dc6b074efaa,0324f7adc6ed3ee6,032546164129d33d,0339945c3efa6035,038489dca54a7573,0399ea2c450ee7e4,03dde332704bde24,0422a1756dd4b002,0438e4975b7072d9,046515d4d714654f,047ca951ff7785ec,04f37fc9861e71d2,0535ead7615abc0a,057f65b285fe6b33,05e22a3dc2ea9a2d,063353a177c20ce2,068aebd250d40230,06b6caa1c1daf295,06d2677a5e1f2234,06d60312a181da08,06eea42e1946df8e,06f43a28c97084fa,06fa7979d88ad25e,075035bea08a5929,078e49066f545d74,07c19e6e4dca2127,07e9c210684eb5c7,07ed228c3923dab2,07f1260c65aca3e4,084095461514744d,08461ab227913814,08608de778786229,0923d5960c333f4c,0962e07add8d11fc,096fb40d3dfd2f6a,0997d7e58462177a,09a327df704ee1d6,09cde1bedfa401e2,09fa17a1b5be2bb9,0a054fe0694046da,0a2f243b8f303d20,0a30f925756f8921,0a7a59dc51884ee7,0a8215cd89a202c0,0acfb853569bbcbc +R-pr3481 0019670d41b88abf,0025c56febee3c1e,00788e8d381460d8,01085921af69bb5d,013bca984a588ae1,0182e1b43ec91412,01cb96661f7b379a,01ccaa7646113fe8,01e4751e9b8bd8bc,0214edf14e5c4caa,0241ace00023c3e1,0244d81eb1cf7df8,02e5e5bfae58eb6c,02fb3e51262e5bbe,030bd71d498d9f4a,034e95700532a72e,0384472ee790c2c9,042fa3b2826ba57b,0435b1a995776cc4,046cc2c718b6528c,04a205dfbf10419d,04a367224e385395,04cc40602bffc4c6,053531982b8adfc6,057f5d03ff3e57f0,05d4d5e3726e34e2,060e868cfc469e2e,0613cda7474e8688,06193b7f79732539,06321240f05ceb16,06366fc160ca9ee6,069ec35d75ef6f01,0720b99da8a88e81,0747cd138cbddd7e,074d7b2f84301eb6,0847021ac1142824,0861ffef1777e211,08fb20476c465092,09114a13d9a90e4c,0962e07add8d11fc,09636a6a3989b083,0979801c1e506c33,09acfc476a0945df,09b494acf11144f2,09efd89737fd902d,09f47bbd76622410,09fa17a1b5be2bb9,0a2b49c2135ff1fa,0a719fd423c4b892,0a9c1cd928670479,0ab08887572c6a26,0ab9be6807c06e34,0ad49228e993d18b,0bf53c7e6d93fff0,0c0e28c81036cc45,0c143fc96aba0e46,0c7bab9ff098f00c,0d93d04a1bc87b8d,0da76a5968661ae9,0ddbfc9368972e9d,0dddb4a21c485cae,0e40f43d140b9546,0e5a8c058142bb26,0eb226430eb1a835 +R-pr3706 00228f94d1fa58a7,002eda1ccbb589b6,0106b31b3deafed8,01150dec476bf84a,01263884d1f841e6,01b9a45c82226f68,01c44aac1d2f433e,01d43a9122b79597,0224869aeb7de738,026793365a3884a4,027511fcb3c655c3,02772f3deafb6bfe,029b7964d4a1b855,02c3fdff305140c1,02cdc379c829a168,02cee8490c3ddad7,0311a38fba9c8f2e,0314797653f64681,033a3c567f55b5f3,033db279cd21002e,034c7fac88b41a98,03b68d5b0b8accb3,03b6db8ac70e37a6,03bbe92d6958fa2a,03ed05e37c27a685,03f0e444fdd21d05,03fbc2f620c93cf6,04018b28947cdd87,0414a34d81eb2da9,0493f30870e7acde,0495d2c2e0e893c6,04a45cbb6feac706,04b838941d6e978d,04ce560bb174f704,04fdf9f74278dd74,050151741bb81e2a,0532a364839f62a4,0535e905921c53c2,05823efbfbafc743,05c58a28b77e74e0,05ef16ab21f508fa,05f7b84c57193d65,060fabc3b9fee435,062c26906fe1ba39,06366fc160ca9ee6,06556845dc09f5a2,066a7dbb695329d6,06795e8d15fc966b,06972c297c42a2db,06af808f3928207b,06be85d69d5b04dc,06c0c8dcb6cc472c,06c24b4369b706eb,06ecf5878811c6ab,0718238bad800dd3,0728b6546a3a99c3,073361f6bef316f1,0736f832c8113635,0745cd696f572c80,07542c3b925ecbc7,0767c4dcc2f1f4c1,077ba73b7647941b,07802c8551530e03,078a18614778fe96 +R-pr410 0281954ddf265646,02acfd707eca0271,059d755dc32c8117,08297316c9ee2146,09418f8fbb0515dc,0bb540c7e6ceae6a,0d3a28f35f0e92ce,13ba0a82959947af,159b51d4b93fdd88,1890138da69a8c98,1bb45f169f1c8baf,1bef78b242ec1fbc,1cc63bab9bf15e9d,1cd93a2993bc3e7d,1d52a2daf390ba37,1e50db9e924e7cfc,220760523de20ce6,2b3918781f144885,2bc3ec81f8b013ca,2ddc677d5cade068,31a25e568426aeff,31e93c288289f291,328d1b156612f17e,37c1381e5e82d2c1,37d9a7ac7b3d9f5d,383e26d1cf626836,3d97d59349456420,3e06464c36cb5641,446dc15d11143a3a,4a42e9a9075cb408,4b3f01884b33b916,4b400438720ab9cc,4d602f4aaeb19368,4e0a8e94d9a271c9,4e756ea45e327439,4fc361f6628c3224,50c78221a6449f10,541eae15e4030aa6,55148ac6e6d79373,563b98ac379a5a47,57a11377a1d78a78,5d5c4a2325c7f671,5f68af7a94abad89,60368c51bf845293,64581b1211048c86,660272e2094f2d98,6baefd7c1f83fe13,6cd32d59fa7a6d90,6eb2efe4fdd43bb1,6f1552583fb4a09d,7090e682425432ff,7e9a8783ee1a72ee,80086687df462118,857ecd4bd508651c,887ec04a6a5db015,8e309af7d656dc51,8e654b563860104f,9174e8fb242076f8,9686bad31c5704dc,97bb544ce7c6277e,99d183442be952bb,9d1271536b2c5c1f,a07ebce1b585a5cf,abaf22d278434400 +R-pr411 000225fdf97e68e3,000397491624e4c0,000577fbd5435cd5,000b8ca2d59d9090,000d061e08faf9cf,000f3eef78c96c4b,00102e9a21aeab3f,002326560fdcc3c6,002a4ecc191c660d,002aa38d9c99b5d0,002adc8ac35affb3,002e94808c0c14d7,0032d5c491abbcfe,003355484380ced9,003cc64d201a42f8,004be879c75215d5,004d8df16623891e,0051674d31770ac1,00547e800e5a91d6,0054d52c2a37652a,00595bf7945b2a7f,0059b01f12afd7a3,00602d81954341ed,006430a94294546a,00677b08b79d852e,00685867cead7e61,006c1ce19c1e32f9,007093e6fb67f5b7,0071660f8bd5509d,0074a2a4cbe25698,0074f1af57f8b225,0076a9fb096efac6,0076d1efe1f915fc,007adfd44681a49a,0081adc748f0c322,0083b3b48c00cebc,0086f0976c21edb2,008760c133e12b51,008b7d47b1962d7e,00907fc6599b6952,009218be48141ff4,00986b11639d0f07,009931d69b18701a,009ce54f9d530624,00a5c266c1c89ddc,00a657fe0e892a0f,00a849520a8f799b,00ae68cb257e1715,00b3c0e8f3565313,00b3e83624f71565,00b5cf6363c85270,00b88ff4d14b7e04,00b9b6fb9c779999,00bc0950075213ea,00bf6f7949734446,00c2ecfd927b5896,00c3554ac7e689e5,00c45287ea5482e4,00c5bcbaf8ed0862,00c7141a5107a04c,00d5f21b5bb906f2,00d78147b9b5c1e3,00dd2c758255b487,00e109d3367fc646 +R-pr750 0002afe7ec7a17d1,000306f00acefdaf,001dbb6c96eb720c,0020a420afbcfe6c,00501a8c81900f37,007c977a0a045202,009471d2b4945d56,00a49f2d15238619,00af6c049db4b78f,00b658a11b4b703e,00d8035dc5dc7bc9,00f83879b27d7224,011f12b5bd9c9fea,01213880213f6f6e,012a5051e19f36b3,0157d971a83db2fd,015ca00f9963dfd7,0187f14af61e67a8,01c275e21e7716e7,02057794a3980890,0246eace4e4cd098,024d8fe5731a5e2a,025c959dafabb761,02890af4945d9192,028cf8f92d4a8068,02ade39fdceed4b5,02cc18ee46b90583,02cc72e142bcef2a,02d749cf9740be99,02da283eb95ff1d0,02e9a8f476c71d30,03153ab8b9e6fa53,0317c16ee5621e14,0322175bd6ebe8db,032d1a970205a307,0337bc3852bb41cf,0364cc0ce2759782,0366b24b2c879b7a,036b6636c2d90f71,0386d97c753f1a08,0387c3f9fb6e04bd,03c971a5517fb83b,03cae8494e8bf21a,03fa614f60a69a91,0413113c3a802676,04177c9f93dae2de,0427204bfcc645c5,0463e5adf4e2300a,046782b0b0707e3c,046b856bb1211c6f,046f7ac84280c524,04b5bc4036e75aa6,04b6f14727b1fefb,04c401f8dced6259,04cb92881ccedf13,04fc757f905d10e9,052e67191b5927b5,05302fb7243c62e8,056883305a489514,057d1f7ce4fb13a3,059a2629a28f7c4a,059c37ef5ecb56ce,05a58a06ee14eb22,05ac5fcc433b816f diff --git a/docs/audits/rex-001/analysis-plan.md b/docs/audits/rex-001/analysis-plan.md new file mode 100644 index 0000000000..82b09afd4a --- /dev/null +++ b/docs/audits/rex-001/analysis-plan.md @@ -0,0 +1,53 @@ +# REX-001 analysis plan (pre-registered, REX-00) + +Frozen together with spec §2–§5, `crates/aprender-review-experiment/src/stats.rs` +and prompt v1 by `docs/audits/rex-001/prereg.lock`. Any byte change to one of those +files is a new spec version (R-1); data gathered under the old version is `exploratory`. + +## Units + +- **Item**: one corpus diff (class P, R or G; stratum S/M/L; split dev/test). +- **Arm**: (lane, model) — `apr-4b` (primary), `apr-9b` (control), `haiku`, `agy`. +- **Receipt**: one `review-experiment-receipt-v1` row per (item, cell, arm). +- **Correct**: a parsed `FAIL` on a P/R item, a parsed `PASS` on a G item. + `Unparsed`, `NotRun{…}`, `Refused` and `ContextOverflow` are never correct and never a + FAIL (R-6); they are counted and reported per state. + +## Tests (code: `stats.rs`) + +| H | Statistic | Function | Rejection rule | +|---|---|---|---| +| H1 invariance | divergent test items vs reference cell C2 (verdict or output bytes) | `count_rule_p` | count > 0 | +| H2 determinism | divergent items on the 10 % same-cell re-run | `count_rule_p` | count > 0 | +| H3 size | McNemar exact, one-sided, `b` = 9B right & 4B wrong | `discordant`, `mcnemar_exact_one_sided` | Holm-adjusted p ≤ 0.05 | +| H4 usefulness | precision(4B) − min precision(haiku, agy), paired bootstrap, 10 000 resamples, seed 4354 | `paired_bootstrap_diff(Rate::Precision)` | holds iff one-sided bootstrap p (share of resamples < 0) passes Holm at 0.05; the 95 % CI is reported beside it | +| H5 recall | recall(4B) − recall(Claude lane), same | `paired_bootstrap_diff(Rate::Recall)` | as H4 | +| H6 interference | mean(co-located) − mean(alone), 5 + 5 runs | `interference` | slowdown > 2 × sd(alone) | + +Family H1–H6, Holm step-down (`holm`), α = 0.05. H1, H2 and H6 are deterministic rules and +enter the family as p = 0 (rejected) or p = 1 (not rejected). + +For H4 the comparison lane is the voter with the LOWER point precision on the same items; +for H5 it is the Claude lane (Haiku 4.5). Items where either arm is not a parsed verdict are +dropped pairwise and the drop count is reported. + +## Descriptive + +Per cell: p50/p95 review wall-clock (nearest rank, `percentile`) with a bootstrap CI of the +p95 (same seed), TTFT, prefill and decode tok/s, peak anon memory, parse rate, and the ratio +to llama.cpp `d1d3c3396` on the same cell. Recall/precision/false-refute/localization with +Wilson 95 % intervals (`wilson`). + +Sensitivity: every per-cell number is reported twice — all runs (primary) and runs with +loadavg1 < 0.5 × physical cores (secondary). Decisions read the primary column only. + +## Sample-size rule (§2.2) + +After REX-05, project the test-split Wilson half-width on recall at the pilot's point estimate +(`wilson_half_width`). If > 0.12, the corpus grows; hypotheses do not change. + +## Seeds + +- Run order per cell: SplitMix64 seed 4354 (the epic number), Fisher–Yates. +- Determinism re-run subset: the first 10 % of the seed-4354 order of test items. +- Bootstrap: seed 4354. diff --git a/docs/audits/rex-001/prereg.lock b/docs/audits/rex-001/prereg.lock new file mode 100644 index 0000000000..95229c0928 --- /dev/null +++ b/docs/audits/rex-001/prereg.lock @@ -0,0 +1,7 @@ +# REX-001 pre-registration lock (rex-prereg-v1). Written once by REX-00. +# A mismatch against the tree is a new spec version (R-1), never an edit here. +spec_s2_s5=dee8b66cbbaa417a9231d1c1332f91975e32bf2ab7731c5390822f92486617e9 +stats_rs=a892d50abebea3f4f879dc7c99391719da0f60c0557c9daeb74f15f7deec2784 +analysis_plan=8ae9213b8092c853e5ab8379827169712b9297d727ce27eaeeb617fbfe2fc43f +prompt_v1=dd9d67c2df458d92d66aed65fd7f3f6f2e5602499c05e6b0967465b9bd2662e6 +prereg_sha=ef51087dc79bab0ad160e8a14f5b13e2ea43986b30c05dafc63c84c2dc21cdc0 diff --git a/docs/audits/rex-001/prm-04-admission.jsonl b/docs/audits/rex-001/prm-04-admission.jsonl new file mode 100644 index 0000000000..5b7fce7c2f --- /dev/null +++ b/docs/audits/rex-001/prm-04-admission.jsonl @@ -0,0 +1,6 @@ +{"schema":"rex-cell-admission-v1","cell":"C1","host":"intel","backend":"wgpu","apr_tag":"v0.69.3","apr_sha256":"8a67a0103cbb036332908cc184d5a8a8ae422fb9025bd5c78e42d65df86cfbd7","model_id":"Qwen3.5-4B-Q4_K_M","weights_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","prereg_sha":"f0b4e4ac07e605e313e61ce5efa9e43f7339ca5a5e5275b0d5f06b0277011041","at":"2026-09-25T20:01:59Z","status":{"state":"NotRun","reason":"NoDeclaredExecutor"}} +{"schema":"rex-cell-admission-v1","cell":"C2","host":"intel","backend":"cpu","apr_tag":"v0.69.3","apr_sha256":"8a67a0103cbb036332908cc184d5a8a8ae422fb9025bd5c78e42d65df86cfbd7","model_id":"Qwen3.5-4B-Q4_K_M","weights_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","prereg_sha":"f0b4e4ac07e605e313e61ce5efa9e43f7339ca5a5e5275b0d5f06b0277011041","at":"2026-09-25T20:01:59Z","status":{"state":"NotRun","reason":"NoDeclaredExecutor"}} +{"schema":"rex-cell-admission-v1","cell":"C3","host":"lambda-labs","backend":"cpu","apr_tag":"v0.69.3","apr_sha256":"8a67a0103cbb036332908cc184d5a8a8ae422fb9025bd5c78e42d65df86cfbd7","model_id":"Qwen3.5-4B-Q4_K_M","weights_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","prereg_sha":"f0b4e4ac07e605e313e61ce5efa9e43f7339ca5a5e5275b0d5f06b0277011041","at":"2026-09-25T20:01:59Z","status":{"state":"NotRun","reason":"NoDeclaredExecutor"}} +{"schema":"rex-cell-admission-v1","cell":"C5a","host":"mini","backend":"cpu","apr_tag":"v0.69.3","apr_sha256":"8a67a0103cbb036332908cc184d5a8a8ae422fb9025bd5c78e42d65df86cfbd7","model_id":"Qwen3.5-4B-Q4_K_M","weights_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","prereg_sha":"f0b4e4ac07e605e313e61ce5efa9e43f7339ca5a5e5275b0d5f06b0277011041","at":"2026-09-25T20:01:59Z","status":{"state":"NotRun","reason":"NoDeclaredExecutor"}} +{"schema":"rex-cell-admission-v1","cell":"C5b","host":"mini","backend":"metal","apr_tag":"v0.69.3","apr_sha256":"8a67a0103cbb036332908cc184d5a8a8ae422fb9025bd5c78e42d65df86cfbd7","model_id":"Qwen3.5-4B-Q4_K_M","weights_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","prereg_sha":"f0b4e4ac07e605e313e61ce5efa9e43f7339ca5a5e5275b0d5f06b0277011041","at":"2026-09-25T20:01:59Z","status":{"state":"NotRun","reason":"NoDeclaredExecutor"}} +{"schema":"rex-cell-admission-v1","cell":"C4","host":"gx10","backend":"cuda","apr_tag":"v0.69.3","apr_sha256":"2f274f477275c578a511fd5fd90e8c9e36e686df88b8eff0529d80ced5e2508c","model_id":"Qwen3.5-4B-Q4_K_M","weights_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","prereg_sha":"f0b4e4ac07e605e313e61ce5efa9e43f7339ca5a5e5275b0d5f06b0277011041","at":"2026-09-25T20:01:59Z","status":{"state":"NotRun","reason":"Inadmissible"}} diff --git a/docs/audits/rex-001/prm-04-receipt.md b/docs/audits/rex-001/prm-04-receipt.md new file mode 100644 index 0000000000..8ace29ec3a --- /dev/null +++ b/docs/audits/rex-001/prm-04-receipt.md @@ -0,0 +1,63 @@ +# PRM-04 — cell admission re-run under `parity.oracle` (rex-cell-admission-v1 1.1.0) + +**Outcome: receipted refusal. 0 of 6 cells Admitted; S-7 stands** (`rex admission-check` exit 10). +C4 has a passing declared-oneshot parity receipt, but it measured 7 positions, and admission requires 64. + +## What changed + +- `admission::parity_from_receipt` derives an `Admitted` parity block from a receipt's bytes and never from typed numbers. It is covered by FALSIFY-RCA-004..006. + - It reads two receipt shapes: + - the `apr-review-serve parity` oneshot (comparator `apr-cpu…`, recorded as oracle `apr-parity-gpu-cpu`, the §4.1 rule-1 interim oracle); + - `apr-parity-oracle/v1` from `apr parity-oracle` (#4444, 57's branch @03579caa9; oracle `llama.cpp@d1d3c3396`). + - The cosine is the **minimum over the receipt's positions**. The receipt's summary `cosine` is never read. + - The threshold and basis are the **declared** ones. A receipt's self-chosen threshold is never read. + - It refuses on any of these: + - fewer than `min_positions` positions; + - exit ≠ 0, a verdict that is not passing, or failed > 0; + - any position below the threshold, or a missing or non-numeric cosine; + - an undeclared oracle, an empty basis, or a non-finite threshold; + - a binary or weights sha that differs from the row's. +- `rex admit --cell C --parity-receipt R --threshold T --threshold-basis B --min-positions N` appends the row. It admits one cell at a time, never `all`. When the receipt does not admit, it writes nothing, exits 1 and names the reason. + +## The run + +| Cell | State | Why | +|------|-------|-----| +| C1 intel/wgpu, C2 intel/cpu, C3 lambda-labs/cpu, C5a mini/cpu, C5b mini/metal | NotRun{NoDeclaredExecutor} | unchanged since REX-04; resumes on infra#1088 | +| C4 gx10/cuda | NotRun{Inadmissible} | `rex admit` refused: `7 positions < min_positions 64` | + +The C4 evidence is `prm-04/gx10-parity-20260925T124817Z.json`, sha256 `54e2b72a12f2413e8b82ecb1a93b38a7b01a3ea671c5d4702365328b60844b67`. It was read from gx10's declared state dir `~/.local/state/apr-review-serve/parity/`. +- It is the output of the declared oneshot `apr-review-serve parity`. +- Binary: apr v0.69.3 `ad5d07a1`, binary sha `2f274f47…`. +- Weights: Qwen3.5-4B-Q4_K_M, sha `00fe7986…`. +- Result: exit 0, verdict pass, 7/7 positions passed, min cosine **0.99970** against the 0.98 threshold. +- **The only failing condition is the evidence floor.** + - `evidence/parity/thresholds.yaml` sets `min_positions: 64` (basis I8 / CF-4, #1864: "any autoregressive gate validates over >= 64 positions"). + - The 0.98 default was itself measured over ≥ 64 positions. + - A 7-position smoke check cannot use a threshold whose basis is 64 positions. + +Positive control: the same receipt with `--min-positions 7` does admit (cosine 0.9997043609619141, oracle `apr-parity-gpu-cpu`). So the refusal is the floor and nothing else. The row it produced was a scratch file and is not part of this audit. + +## Why not the llama.cpp oracle + +`apr parity-oracle` (#4444) has only been measured on Qwen3.5-**0.8B**. The prereg-locked model is **4B**. Its threshold row `qwen3.5-0.8b@cpu-vs-llama.cpp` deliberately has no `min_cosine`. No 4B llama.cpp reference exists yet, so this oracle cannot admit a cell. + +## To admit C4 (resume conditions) + +1. The declared oneshot must measure at least 64 positions. The fix is a longer parity prompt, such as the 78-token corpus prompt the 0.98 basis used. Then re-run: + ``` + rex admit --cell C4 --parity-receipt --threshold 0.98 --threshold-basis "evidence/parity/thresholds.yaml default.min_cosine" --min-positions 64 --apr-sha --weights-sha 00fe7986… + ``` +2. The serve declaration (infra e872544c) and the parity oneshot (infra 3f66d253/3d59dd3e) are live on gx10, but only through the open infra PRs #1142, #1154 and fold #1054. **They are not on infra main**, and infra#1088 is open. An Admitted C4 still rests on a declaration that has not landed. +3. H1 (verdict invariance) is evaluated in the analysis (PRM-08), not at admission. + +## Note + +The prereg lock the tool reads today is `f0b4e4ac…`, while the REX-04 rows carry `ef51087d…`. This file uses the current lock. REX-04's file is left untouched. + +## Gates + +- `cargo test -p aprender-review-experiment --lib`: 167 passed. +- `cargo clippy -p aprender-review-experiment --all-targets -D warnings`: clean. +- `pv validate contracts/rex-cell-admission-v1.yaml`: valid. +- `cargo mutants --re parity_from_receipt`: 21 caught, 1 unviable, 1 missed. The missed mutant was an unreachable `is_finite` guard, which was removed because JSON carries no NaN or ∞. diff --git a/docs/audits/rex-001/prm-04/gx10-parity-20260925T124817Z.json b/docs/audits/rex-001/prm-04/gx10-parity-20260925T124817Z.json new file mode 100644 index 0000000000..410ac789da --- /dev/null +++ b/docs/audits/rex-001/prm-04/gx10-parity-20260925T124817Z.json @@ -0,0 +1,112 @@ +{ + "host": "gx10-a5b5", + "at": "20260925T124817Z", + "apr_tag": "v0.69.3", + "apr_commit": "ad5d07a182b15e70527d3f33f18ceb1211ba4c80", + "binary_sha256": "2f274f477275c578a511fd5fd90e8c9e36e686df88b8eff0529d80ced5e2508c", + "weights": "/home/noah/data/models/Qwen3.5-4B-Q4_K_M.gguf", + "weights_sha256": "00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4", + "comparator": "apr-cpu (apr parity: GPU vs CPU; no llama.cpp comparator in this apr, aprender#3576)", + "exit": 0, + "verdict": "pass", + "parity": { + "model": "/home/noah/data/models/Qwen3.5-4B-Q4_K_M.gguf", + "tokens": 7, + "passed": 7, + "failed": 0, + "parity": true, + "metrics": [ + { + "position": 0, + "token_id": 3710, + "cpu_argmax": 369, + "gpu_argmax": 369, + "max_abs_diff": 0.2318652868270874, + "mean_abs_diff": 0.03146722540259361, + "cosine_similarity": 0.9998131990432739, + "kl_divergence": 0.00021212672537373304, + "sigma_level": 100.47431010234187, + "cpk": 33.22796605385059, + "verdict": "Pass" + }, + { + "position": 1, + "token_id": 369, + "cpu_argmax": 279, + "gpu_argmax": 279, + "max_abs_diff": 0.14195120334625244, + "mean_abs_diff": 0.02431115135550499, + "cosine_similarity": 0.9999650120735168, + "kl_divergence": 0.000028142869161619233, + "sigma_level": 131.23602099621212, + "cpk": 43.479465434592946, + "verdict": "Pass" + }, + { + "position": 2, + "token_id": 220, + "cpu_argmax": 16, + "gpu_argmax": 16, + "max_abs_diff": 0.24833190441131592, + "mean_abs_diff": 0.03066864237189293, + "cosine_similarity": 0.999754011631012, + "kl_divergence": 0.00010543855908146571, + "sigma_level": 103.40762116359619, + "cpk": 34.20492610853252, + "verdict": "Pass" + }, + { + "position": 3, + "token_id": 17, + "cpu_argmax": 15, + "gpu_argmax": 15, + "max_abs_diff": 0.20449680089950562, + "mean_abs_diff": 0.030058100819587708, + "cosine_similarity": 0.9998959898948669, + "kl_divergence": 0.00007435307399205818, + "sigma_level": 105.21864173972742, + "cpk": 34.80932453478307, + "verdict": "Pass" + }, + { + "position": 4, + "token_id": 10, + "cpu_argmax": 17, + "gpu_argmax": 17, + "max_abs_diff": 0.23898255825042725, + "mean_abs_diff": 0.03911856189370155, + "cosine_similarity": 0.999718189239502, + "kl_divergence": 0.0002524354532544672, + "sigma_level": 82.05736118788325, + "cpk": 27.084956565755913, + "verdict": "Pass" + }, + { + "position": 5, + "token_id": 17, + "cpu_argmax": 30, + "gpu_argmax": 30, + "max_abs_diff": 0.23314523696899414, + "mean_abs_diff": 0.03875268995761871, + "cosine_similarity": 0.9997207522392273, + "kl_divergence": 0.0005191280388814299, + "sigma_level": 82.29276982289848, + "cpk": 27.165167758074453, + "verdict": "Pass" + }, + { + "position": 6, + "token_id": 30, + "cpu_argmax": 271, + "gpu_argmax": 271, + "max_abs_diff": 0.23418569564819336, + "mean_abs_diff": 0.03857100382447243, + "cosine_similarity": 0.9997043609619141, + "kl_divergence": 0.00028003667922200324, + "sigma_level": 82.17311717474466, + "cpk": 27.126914090180225, + "verdict": "Pass" + } + ] + } +} diff --git a/docs/audits/rex-001/rex-00-receipt.md b/docs/audits/rex-001/rex-00-receipt.md new file mode 100644 index 0000000000..bd6e401a55 --- /dev/null +++ b/docs/audits/rex-001/rex-00-receipt.md @@ -0,0 +1,32 @@ +# REX-00 receipt — pre-registration (PMAT-4355, epic #4354) + +- Tree: worktree `/mnt/nvme-raid0/tmp/rex`, branch `rex/001`, base `origin/main` @ `5fa13fe8c` (R-12). +- Spec: operator file sha256 `3e0b5594af99…`; in-tree copy differs only in lines 1 and 3 + (PROMETHEUS rename, cop relay 2026-09-25), both outside §2–§5. +- **prereg_sha = `ef51087dc79bab0ad160e8a14f5b13e2ea43986b30c05dafc63c84c2dc21cdc0`** + (`cargo run -p aprender-review-experiment --example rex -- prereg-check`). +- Components: spec §2–§5 `dee8b66c…`, `stats.rs` `a892d50a…`, analysis plan `8ae9213b…`, + prompt v1 `dd9d67c2…` (full values in `prereg.lock`). +- Prompt v1 = the G1 pilot prompt header byte-for-byte (compared against the pilot's + `prompt.txt`, cop scratchpad `qdemo/`). +- Test-item manifest: placeholder `docs/audits/review-corpus/test-manifest-v1.txt` (sealed by REX-02). +- Ordering (S-1): committed before any REX measurement; no measurement has been taken. + +## Falsifier, planted on disk (RED → GREEN) + +Planted `any count > 0 rejects H1` → `any count > 1 rejects H1` in §3 of the spec file: + +``` +rex-prereg-v1 DRIFT spec_s2_s5: lock dee8b66c… != tree e4c1abd9… +rex-prereg-v1 DRIFT prereg_sha: lock ef51087d… != tree 42db9f76… +planted_rc=1 +falsify_rex_prereg_001: test result: FAILED. 0 passed; 1 failed +``` + +Restored (cmp-identical) → `rex-prereg-v1 OK prereg_sha=ef51087d…`; 12/12 lib tests pass. + +## Recalibration note (R-7) + +§2.2 quotes a half-width of "≈ 0.117" for 70 defect items at p = 0.5. That is the Wald +half-width; the pre-registered Wilson interval gives **0.114** (`stats::wilson_half_width(35, 70)`). +The decision threshold (0.12) is unchanged. diff --git a/docs/audits/rex-001/rex-01-receipt.md b/docs/audits/rex-001/rex-01-receipt.md new file mode 100644 index 0000000000..b680119beb --- /dev/null +++ b/docs/audits/rex-001/rex-01-receipt.md @@ -0,0 +1,31 @@ +# REX-01 receipt — executor issues (PMAT-4356, epic #4354) + +Filed; no measurement. prereg_sha `ef51087dc79bab0ad160e8a14f5b13e2ea43986b30c05dafc63c84c2dc21cdc0`. + +## Phase 0 — does an existing declared label already admit these jobs? No. + +paiml/infra origin/main `3b1b19e8`, `machines/*/forjar.yaml`: + +- **apr binary pin:** present on gx10, lambda-labs and yoga (`apr-pin` + fleet-bins, infra#1027/#1055). + Absent on intel; mini builds from the tag (infra#1035). +- **`apr serve` unit, Qwen3.5 GGUF weights by sha256, review-lane label:** absent on every host. + The lambda-labs `ollama-model-qwen35-*` resources are ollama packages, not the GGUF apr loads. +- gx10 runner labels: `gpu,gx10,cuda,blackwell,gb10`, opt-in `gx10`. None of them admits a serve-backed review job. + +So every cell stays `NotRun{NoDeclaredExecutor}` until infra#1088 lands (R-4). + +## Filed + +| Purpose | URL | +|---|---| +| forjar declarations per host (gx10 P1, intel, lambda CPU-only, mini): apr tag, weights path+sha256, serve unit, cgroup, health, exec_sha256, train-active yield, intel concurrency group + non-clean-room label; checkable acceptance | https://github.com/paiml/infra/issues/1088 | +| 4th shadow lane (`quorum.lane_models`, width stays 3, `Unknown{LaneUnavailable}`, review-ledger-v1 rows) + ARB-APR-001 gx10 revision, extending the existing lane issue rather than duplicating it | https://github.com/paiml/paiml-implement/issues/411#issuecomment-5829799553 | +| ARB-APR-001 row 3 cross-reference | https://github.com/paiml/aprender/issues/3574#issuecomment-5829799814 | + +## Weights (lambda-vector copies, `sha256sum`, 2026-09-25) + +| File | Bytes | sha256 | +|---|---|---| +| Qwen3.5-4B-Q4_K_M.gguf | 2740937888 | 00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4 | +| Qwen3.5-9B-Q4_K_M.gguf | 5680522464 | 03b74727a860a56338e042c4420bb3f04b2fec5734175f4cb9fa853daf52b7e8 | +| Qwen3.5-27B-Q4_K_M.gguf | 16740812704 | 84b5f7f112156d63836a01a69dc3f11a6ba63b10a23b8ca7a7efaf52d5a2d806 | diff --git a/docs/audits/rex-001/rex-02-receipt.md b/docs/audits/rex-001/rex-02-receipt.md new file mode 100644 index 0000000000..b02415e2ba --- /dev/null +++ b/docs/audits/rex-001/rex-02-receipt.md @@ -0,0 +1,63 @@ +# REX-02 receipt — review corpus v1 (PMAT-4357, epic #4354) + +prereg_sha `ef51087dc79bab0ad160e8a14f5b13e2ea43986b30c05dafc63c84c2dc21cdc0` (unchanged; `rex prereg-check` OK). + +- **Corpus version:** `review-corpus-v1@787d2026256cc08b` +- **Sealed manifest:** `test-manifest-v1.txt`, sha256 `787d2026256cc08b2b219c2f315f3ada5ceeca5e5b47ef93081c95e7f0fa1b2b` (105 test items) +- **Item metadata:** `corpus-v1.jsonl`, sha256 `7acf0ef708021a62b55f03393f47ffdee26303a0902ee33bbc6753d3869f37f5` +- **Diffs (outside git):** `/mnt/nvme-raid0/rex-corpus/v1/items-v1.tar` on lambda-vector, sha256 + `70dffe5625e1849661303b82bb1fdf5254ab1d4560e2fe3eacc147e5d2588bba`. Deterministic tar (sorted, + fixed mtime/owner); hosts get it through the infra#1088 declaration. +- **Built by:** + `rex corpus-build prs.json 2026-09-11T00:00:00Z <4 mutant lists>` at base `80437647e`. + Inputs: + - `gh pr list --state merged --limit 1500` snapshot (1468 PRs, sha256 `e185a34fa87f1ee1…`, copied to + `/mnt/nvme-raid0/rex-corpus/v1/prs-snapshot.json`); + - `cargo mutants --list --diff --json` for aprender-core, aprender-serve, apr-cli and + aprender-contracts (76 264 mutants, concatenated sha256 `80e6f2773c0a6127…`). Listed only; nothing was + compiled or run. + +## Counts (class × stratum × split) + +| Class | S dev/test | M dev/test | L dev/test | Total | +|---|---|---|---|---| +| P planted | 15 / 35 | — | — | 50 | +| R real (revert-the-fix) | 7 / 15 | 6 / 14 | 2 / 6 | 50 | +| G good | 5 / 12 | 5 / 12 | 5 / 11 | 50 | +| **Total** | 27 / 62 | 11 / 26 | 7 / 17 | **150 (45 dev / 105 test)** | + +Test defect items (P ∪ R) = **70**. This matches the §2.2 sample-size basis. P is all S, because a +single mutant diff is small; the population does not allow balancing it (§2.2 "where the +population allows"). + +## Construction notes (all `[A]`, fixed before any measurement) + +1. **P source.** No catalogue of 50 committed planted-defect fixtures exists: 9 registered + MUTATION-POINTs and about 47 sed plants are spread across guard scripts. So P is the fleet's + mutation tool (cargo-mutants 27.0.0, the CI `mutants` gate) listing mutants as diffs. The + sample is seeded (4354), one per file, on review paths only. The `+++ replace …` header and the + `/* ~ changed by cargo-mutants ~ */` marker are stripped (FALSIFY-RCV-001); 0 of 150 item + files contain either. + - Caveat: some listed mutants may be equivalent (no behaviour change). This is reported as a + label-noise bound, not corrected. +2. **R.** Merged PRs with a closing-issue link and a fix title. Only code hunks on review paths + (.rs/.sh; not tests, benches or examples) are reverse-applied; comment-only hunks are dropped + (FALSIFY-RCV-003). Defect label = the new-side lines of the reversed hunks. +3. **G.** Merged before 2026-09-11 (14 days before the build), not a fix, not named by a + `Revert` PR, and no `regression` label. Green means `ci / gate` and `workspace-test` + SUCCESS with no FAILURE/TIMED_OUT/ERROR in the rollup; this was checked per PR via gh. +4. **Size.** Strata use a bytes/4 token proxy (not a model tokenizer). Candidates over 32 000 + proxy tokens are dropped at construction. Nothing kept is ever truncated. +5. **Seal.** No test item was read by the author: the builder prints counts only, and the one + dev item inspected was an R dev item. + +## Contamination falsifier, planted on disk (RED → GREEN) + +A test R item was copied (by script, unread) into +`docs/audits/review-corpus/train/leak.jsonl` as a JSONL `diff` field: + +``` +falsify_rcc_003: planted_rc=101 test result: FAILED item: "R-pr1411" how: "hunk" +``` + +Removed by trap → 29/29 lib tests pass. diff --git a/docs/audits/rex-001/rex-03-receipt.md b/docs/audits/rex-001/rex-03-receipt.md new file mode 100644 index 0000000000..166710d5f4 --- /dev/null +++ b/docs/audits/rex-001/rex-03-receipt.md @@ -0,0 +1,65 @@ +# REX-03 receipt — harness, scorer, `review-experiment-receipt-v1` (PMAT-4358, epic #4354) + +prereg_sha `ef51087dc79bab0ad160e8a14f5b13e2ea43986b30c05dafc63c84c2dc21cdc0` (unchanged; `stats.rs` +is frozen and was not edited — the scorer calls it). Corpus `review-corpus-v1@787d2026256cc08b`. + +## What landed (crate `aprender-review-experiment`) + +| Module | Role | +|---|---| +| `receipt.rs` | The row schema (`deny_unknown_fields`), the `VERDICT:` parser (§2.3), localization, admissibility | +| `score.rs` | §2.3 metrics with Wilson CIs over the **expected** item set; H1/H2 divergence, H3 McNemar input, H4/H5 pairwise-parsed pairs | +| `harness.rs` | `/v1/chat/completions` client (ureq), fixed prompt composition, seeded run order, 10 % re-run subset, load snapshot, receipt construction | +| `examples/rex.rs` | `review`, `not-run`, `score` (with `--pubkey` → `minisign -V`, otherwise labelled `unsigned (exploratory)`) | +| `contracts/review-experiment-receipt-v1.yaml` | FALSIFY-RXR-001..003; `pv validate` → 0 errors | + +## Pre-registered choices made here `[A]` (the spec left them open; fixed before any model run) + +- **Decoding:** temperature 0, seed 4354, `max_tokens` 512. +- **Request:** the prompt file verbatim, a blank line, then the diff in a ```diff fence. It is never + truncated; a 4xx that names the context becomes `NotRun{ContextOverflow}`. The sha of the exact + body is recorded (`request_sha256`). +- **Verdict:** the first `VERDICT:` line after the last ``. Markdown decoration is ignored. + The first word must be exactly `PASS`/`FAIL`, and a line naming both is `Unparsed` (an echo of + the prompt). The table has 12 cases in `receipt_tests.rs`. +- **Localization:** a finding after the verdict names the defect file by its full path or by its + last two components. A bare file name does not count (`mod.rs`). +- **Scoring denominator:** the expected item set. An item without exactly one admissible row + (raw output present at the recorded sha, and re-parsing to the recorded verdict) is + `NotRun{Inadmissible}`. Dropping a row can therefore never raise a score. +- **Hosted arms** (Haiku, agy) record `hosted` for the apr/weights shas plus their exact model id. +- **A row whose serve reply has no `usage`** is `NotRun{ServeError}`; it cannot be admitted as a run. +- **Nullable by design:** `timings.server` (TTFT/tok/s is then `NotRun{NoSensor}`, never 0), + `load.co_running` and `load.peak_anon_bytes` (executor-reported, infra#1088). + +## Falsifiers, planted RED → restored GREEN (mutations in `receipt.rs`, restored from a sha-checked copy) + +| Falsifier | Plant | Planted | Restored | +|---|---|---|---| +| RXR-001 missing model sha → inadmissible | `#[serde(default)]` on `weights_sha256` + drop its hex check | rc=101 "missing weights sha must be inadmissible" | 1 passed | +| RXR-002 NotRun counted as correct → rejected | `NotRun(_) => true` in `Verdict::correct` | rc=101 "NotRun counted as correct", left (10,10) right (0,10) | 1 passed | +| RXR-003 Unparsed counted as PASS → rejected | `Unparsed => !defect` | rc=101 "Unparsed counted as PASS on a good item", left (2,3) right (0,3) | 1 passed | + +Scorer fixtures are hand-computed (`score_tests.rs`): +- 6 defects / 4 good → recall 3/6 with Wilson (0.18762, 0.81238) by hand; precision 3/4; + false-refute 1/4; localization 2/3; parse 6/8. +- McNemar b=5, c=1 → p = 7/64. +- `rfc3339` checked against `date -u`. +- A loopback HTTP test drives `post` end to end. + +Totals: 47/47 lib tests; clippy `-D warnings` clean on lib, examples and tests. + +## CLI smoke on the real corpus (no serve, no GPU) + +- `rex not-run --why NoDeclaredExecutor --split test` (C2, apr-4b, v0.69.3 `~/.cargo/bin/apr` + sha) wrote 105 rows. +- `rex score` gave: items 105, recall 0/70 (the NotRun items stay in the denominator), + correct 0/105, `not_run {NoDeclaredExecutor: 105}`, rejected rows 0. +- Signed with a throwaway minisign key: `provenance: signed`. +- After a one-byte edit to the receipts file: `signature does not verify` → exit 1. + +## Not done here, by design + +- No model run: cells need forjar-declared executors (R-4, infra#1088). REX-05/06 run through + `rex review` on those executors. +- Receipt push/transport is executor-side (infra#1088). diff --git a/docs/audits/rex-001/rex-04-admission.jsonl b/docs/audits/rex-001/rex-04-admission.jsonl new file mode 100644 index 0000000000..f29acc529f --- /dev/null +++ b/docs/audits/rex-001/rex-04-admission.jsonl @@ -0,0 +1,6 @@ +{"schema":"rex-cell-admission-v1","cell":"C1","host":"intel","backend":"wgpu","apr_tag":"v0.69.3","apr_sha256":"8a67a0103cbb036332908cc184d5a8a8ae422fb9025bd5c78e42d65df86cfbd7","model_id":"Qwen3.5-4B-Q4_K_M","weights_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","prereg_sha":"ef51087dc79bab0ad160e8a14f5b13e2ea43986b30c05dafc63c84c2dc21cdc0","at":"2026-09-25T09:35:32Z","status":{"state":"NotRun","reason":"NoDeclaredExecutor"}} +{"schema":"rex-cell-admission-v1","cell":"C2","host":"intel","backend":"cpu","apr_tag":"v0.69.3","apr_sha256":"8a67a0103cbb036332908cc184d5a8a8ae422fb9025bd5c78e42d65df86cfbd7","model_id":"Qwen3.5-4B-Q4_K_M","weights_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","prereg_sha":"ef51087dc79bab0ad160e8a14f5b13e2ea43986b30c05dafc63c84c2dc21cdc0","at":"2026-09-25T09:35:32Z","status":{"state":"NotRun","reason":"NoDeclaredExecutor"}} +{"schema":"rex-cell-admission-v1","cell":"C3","host":"lambda-labs","backend":"cpu","apr_tag":"v0.69.3","apr_sha256":"8a67a0103cbb036332908cc184d5a8a8ae422fb9025bd5c78e42d65df86cfbd7","model_id":"Qwen3.5-4B-Q4_K_M","weights_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","prereg_sha":"ef51087dc79bab0ad160e8a14f5b13e2ea43986b30c05dafc63c84c2dc21cdc0","at":"2026-09-25T09:35:32Z","status":{"state":"NotRun","reason":"NoDeclaredExecutor"}} +{"schema":"rex-cell-admission-v1","cell":"C4","host":"gx10","backend":"cuda","apr_tag":"v0.69.3","apr_sha256":"8a67a0103cbb036332908cc184d5a8a8ae422fb9025bd5c78e42d65df86cfbd7","model_id":"Qwen3.5-4B-Q4_K_M","weights_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","prereg_sha":"ef51087dc79bab0ad160e8a14f5b13e2ea43986b30c05dafc63c84c2dc21cdc0","at":"2026-09-25T09:35:32Z","status":{"state":"NotRun","reason":"NoDeclaredExecutor"}} +{"schema":"rex-cell-admission-v1","cell":"C5a","host":"mini","backend":"cpu","apr_tag":"v0.69.3","apr_sha256":"8a67a0103cbb036332908cc184d5a8a8ae422fb9025bd5c78e42d65df86cfbd7","model_id":"Qwen3.5-4B-Q4_K_M","weights_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","prereg_sha":"ef51087dc79bab0ad160e8a14f5b13e2ea43986b30c05dafc63c84c2dc21cdc0","at":"2026-09-25T09:35:32Z","status":{"state":"NotRun","reason":"NoDeclaredExecutor"}} +{"schema":"rex-cell-admission-v1","cell":"C5b","host":"mini","backend":"metal","apr_tag":"v0.69.3","apr_sha256":"8a67a0103cbb036332908cc184d5a8a8ae422fb9025bd5c78e42d65df86cfbd7","model_id":"Qwen3.5-4B-Q4_K_M","weights_sha256":"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4","prereg_sha":"ef51087dc79bab0ad160e8a14f5b13e2ea43986b30c05dafc63c84c2dc21cdc0","at":"2026-09-25T09:35:32Z","status":{"state":"NotRun","reason":"NoDeclaredExecutor"}} diff --git a/docs/audits/rex-001/rex-04-receipt.md b/docs/audits/rex-001/rex-04-receipt.md new file mode 100644 index 0000000000..1b6cf9c643 --- /dev/null +++ b/docs/audits/rex-001/rex-04-receipt.md @@ -0,0 +1,63 @@ +# REX-04 receipt — per-cell admission, `rex-cell-admission-v1` (PMAT-4359, epic #4354) + +prereg_sha `ef51087dc79bab0ad160e8a14f5b13e2ea43986b30c05dafc63c84c2dc21cdc0` (unchanged). + +## Result: 6/6 cells resolved, 0 silent; every cell `NotRun{NoDeclaredExecutor}` → **S-7** + +Admission file: `docs/audits/rex-001/rex-04-admission.jsonl` (6 rows). Every row carries: +- apr `v0.69.3`, released binary sha256 `8a67a0103cbb036332908cc184d5a8a8ae422fb9025bd5c78e42d65df86cfbd7`; +- `Qwen3.5-4B-Q4_K_M`, weights sha256 `00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4`. + +| Cell | Host / backend | State | Why | +|---|---|---|---| +| C1 intel-wgpu | intel / wgpu | NotRun{NoDeclaredExecutor} | infra#1088 open | +| C2 intel-cpu (reference) | intel / cpu | NotRun{NoDeclaredExecutor} | infra#1088 open | +| C3 lambda-cpu | lambda-labs / cpu | NotRun{NoDeclaredExecutor} | infra#1088 open | +| C4 gx10-cuda | gx10 / cuda | NotRun{NoDeclaredExecutor} | infra#1088 open (gx10 is P1) | +| C5a mini-cpu | mini / cpu | NotRun{NoDeclaredExecutor} | infra#1088 open | +| C5b mini-metal | mini / metal | NotRun{NoDeclaredExecutor} | infra#1088 open | + +`rex admission-check --file docs/audits/rex-001/rex-04-admission.jsonl` → +`{"admitted":[],"refused":[],"not_run":["C1","C2","C3","C4","C5a","C5b"],"s7":true}`, exit **10**. + +The spec expects C1 and C5b to be "likely `Refused` before 0.71". They are still recorded +as NotRun: a refusal is a result (§0.5), so it has to be observed on the declared executor, +never assumed. R-4 forbids ad-hoc SSH or hand-installed `apr`, and the REX-01 Phase 0 check +found no existing declared label that admits these jobs. No cell can therefore run until +infra#1088 lands. + +## Premise check (§0.6 genchi genbutsu) + +§7 REX-04 says "`apr parity` per cell against llama.cpp `d1d3c3396`". This premise is +**false at the released tag**: `apr parity` in v0.69.3 is a GPU-vs-CPU check +(`apr parity --help`: "GPU/CPU parity check"), and no released apr verb compares logits +against llama.cpp. The admission schema therefore records the oracle explicitly +(`parity.oracle`, e.g. `llama.cpp@d1d3c3396` or `apr-parity-gpu-cpu`), with a cosine, a +threshold and a cited `threshold_basis` (R-7). The llama.cpp oracle binary must be a +declared executor input like the weights (added to infra#1088). + +## Falsifiers, planted RED → restored GREEN (`admission.rs`, restored from a sha-checked copy) + +| Falsifier | Plant | Planted | Restored | +|---|---|---|---| +| RCA-001 a silent cell rejects the file | silent-cell arm made a no-op | rc=101 `admission_tests.rs:80` "a silent cell must reject the file" | 1 passed | +| RCA-002 Admitted needs a passing parity receipt | drop `cosine >= threshold` | rc=101 `admission_tests.rs:106` "below threshold" | 1 passed | +| RCA-003 no admitted cell raises S-7 | `s7 = false` | rc=101 `admission_tests.rs:122` "all NotRun is S-7" | 1 passed | + +Source was byte-identical after restore (`restored-ok`). 52/52 lib tests pass; clippy +`-D warnings` is clean on lib, examples and tests; `pv validate` reports 0 errors. + +## S-7 → STOP + +§8 S-7: "Every cell is `Refused` or `NotRun`, so there is no admissible cell for (A)". This +is a STOP: write the §9 report and do not work around it. REX-05..REX-12 all need a running +cell or the gx10 declaration, so none proceeds. **Resume condition:** infra#1088 lands at +least one declared executor. Then re-run `rex admit` for that cell with a parity receipt, +and `rex admission-check` exits 0. + +## Operator ruling recorded here (the prereg is frozen) + +S-6 was resolved by the operator (verbatim, 2026-09-25, relayed by aprender-77 as +"OPERATOR, verbatim"): **"yes it can train on claude/agy outputs"**. B2 may train on silver +labels (Claude/agy outputs) alongside gold labels and the 27B teacher. Silver and gold are +kept in separate files, and the label source is recorded per row. diff --git a/docs/findings/2026-09-26-aprender-84.jsonl b/docs/findings/2026-09-26-aprender-84.jsonl new file mode 100644 index 0000000000..4281922811 --- /dev/null +++ b/docs/findings/2026-09-26-aprender-84.jsonl @@ -0,0 +1,2 @@ +{"id":"FND-20260926-qwen35-cpu-q4k-fwd-parity","title":"apr CPU forward on Qwen3.5-0.8B-Q4_K_M differs from llama.cpp by 0.19-1.65 nats on top-20 logprobs and flips one top-1","evidence":"apr 0.69.3 (6f909ca30) serve --no-gpu (log backend=cpu) vs llama-server build 11050 (60b06ab9a) -ngl 0, same GGUF sha256 bd258782...dc517, official template enable_thinking:false, teacher-forced on apr ids: 29 steps/6 prompts, top-1 28/29, overlap>=16 28/29, max|dlogprob|<=0.25 on only 4/29; p1 step0 apr \" Paris\" -0.506 vs oracle \"Paris\" -0.390; positive control oracle-vs-itself dlogprob 0 (positive_control.sh, p{2,3,5}.posctl.json). Attribution untested (Q4_K/Q8_K dot, accumulation order, SSM/linear-attn kernels). Receipt docs/audits/prm-001/c11-4026-live/RECEIPT.md","repro":"bash docs/audits/prm-001/c11-4026-live/c4026_live.sh Qwen3.5-0.8B-Q4_K_M.gguf ; jq .steps /p1.cmp.json","suspected_epic":"unknown (forward-parity contract; not #4026)","severity":"P2","found_at_sha":"6f909ca30c924c9a18e08a9503feebeb260d2d59"} +{"id":"FND-20260926-serve-tokenize-splits-im-end","title":"apr serve POST /tokenize splits <|im_end|> into 6 literal pieces while parsing <|im_start|> (248045) as one special","evidence":"apr 0.69.3 (6f909ca30), Qwen3.5-0.8B-Q4_K_M: /tokenize on the p1 templated prompt returns 28 ids; the chat path (usage.prompt_tokens) and llama.cpp /tokenize parse_special:true both give 24. In the returned ids 248045 appears twice but <|im_end|> is [15294,92811,76,6018,91,29]. File docs/audits/prm-001/c11-4026-live/p1.apr_tok.json. The chat path is unaffected (count matches)","repro":"apr serve run --no-gpu; POST /tokenize with the p1 templated prompt (p1.ids.json oracle = 24 ids); compare num_tokens and look for 248046","suspected_epic":"unknown","severity":"P3","found_at_sha":"6f909ca30c924c9a18e08a9503feebeb260d2d59"} diff --git a/docs/findings/2026-09-26-aprender-fb.jsonl b/docs/findings/2026-09-26-aprender-fb.jsonl new file mode 100644 index 0000000000..3ac9c73953 --- /dev/null +++ b/docs/findings/2026-09-26-aprender-fb.jsonl @@ -0,0 +1 @@ +{"id":"FND-20260926-serve-qwen35-gpu-no-timings","title":"apr serve omits the server timings block on the Qwen3.5 hybrid CUDA chat path, so replay TTFT/prefill/decode are null for apr","evidence":"PRM-S1 review-replay-v1 set b8a58011 on gx10 (GB10, gpu-layers resolved=32/32 backend=cuda, server log 'qwen35 hybrid forward, #3090'), apr 0.69.5-rc.1 (5660b0877), Qwen3.5-4B-Q4_K_M sha 00fe7986: 200/200 /v1/chat/completions replies have usage (input/output tokens) but no timings, so prompt_ms, prompt_tps and decode_tps are null in all 200 rows. llama.cpp d1d3c3396 on the same set and cell returns timings on 200/200. Timings::from_phases exists and cuda_chat_backend.rs builds PhaseTimings; the hybrid path apparently does not call it (suspected, not traced)","repro":"apr serve run Qwen3.5-4B-Q4_K_M.gguf --gpu-layers all --context-length 36864; POST /v1/chat/completions non-streaming; the reply JSON has no timings key","suspected_epic":"PRM-001 (#4354)","severity":"P2","found_at_sha":"5660b08775c1115e9951960638e54a6d83baabd6"} diff --git a/docs/roadmaps/entries/PMAT-4354.yaml b/docs/roadmaps/entries/PMAT-4354.yaml new file mode 100644 index 0000000000..ad899bd56c --- /dev/null +++ b/docs/roadmaps/entries/PMAT-4354.yaml @@ -0,0 +1,18 @@ +- id: PMAT-4354 + github_issue: 4354 + item_type: task + kind: code + title: 'PRM-L: land PRM-001 v3 (PROMETHEUS) + rex/001 to main' + status: in_progress + priority: high + assigned_to: aprender-84 + created: 2026-09-25T18:38:33Z + updated: 2026-09-25T18:38:33Z + spec: docs/specifications/PRM-001-prometheus.md + acceptance_criteria: + - Refs paiml/aprender#4354. Spec PRM-001 v3 byte-exact, rex-prereg-v2 lock, v1 lock frozen, superseded banners, v3 figure. + phases: [] + subtasks: [] + estimated_effort: null + labels: [] + notes: null diff --git a/docs/roadmaps/entries/PMAT-4446.yaml b/docs/roadmaps/entries/PMAT-4446.yaml new file mode 100644 index 0000000000..3323407258 --- /dev/null +++ b/docs/roadmaps/entries/PMAT-4446.yaml @@ -0,0 +1,16 @@ +- id: PMAT-4446 + github_issue: 4446 + item_type: task + title: PRA-001 T12 terms/provider tagging (P-TERMS) + status: planned + priority: medium + assigned_to: null + created: 2026-09-25T17:08:31Z + updated: 2026-09-25T17:08:31Z + spec: null + acceptance_criteria: [] + phases: [] + subtasks: [] + estimated_effort: null + labels: [] + notes: null diff --git a/docs/roadmaps/entries/PMAT-4447.yaml b/docs/roadmaps/entries/PMAT-4447.yaml new file mode 100644 index 0000000000..d62f697fd3 --- /dev/null +++ b/docs/roadmaps/entries/PMAT-4447.yaml @@ -0,0 +1,16 @@ +- id: PMAT-4447 + github_issue: 4447 + item_type: task + title: PRM-S1 review-replay-v1 benchmark + runner + status: planned + priority: medium + assigned_to: null + created: 2026-09-25T17:46:46Z + updated: 2026-09-25T17:46:46Z + spec: null + acceptance_criteria: [] + phases: [] + subtasks: [] + estimated_effort: null + labels: [] + notes: null diff --git a/docs/specifications/PRM-001-prometheus.md b/docs/specifications/PRM-001-prometheus.md new file mode 100644 index 0000000000..6eeec4fdb8 --- /dev/null +++ b/docs/specifications/PRM-001-prometheus.md @@ -0,0 +1,441 @@ +# PRM-001 — PROMETHEUS: make local Qwen 3.5 fast enough, and good enough, to break quorum ties + +**Spec id:** `PRM-001` v3 (Prometheus). **Supersedes:** PRM-001 v2, REX-001 / PRM-001 v1 (experiment protocol), and PRA-001 (trace-corpus design). Old row ids are kept in the §7 `was` column. +**Target:** `paiml/aprender` → `docs/specifications/PRM-001-prometheus.md`. This is the single source of truth. +- The infra copy becomes a one-line pointer. +- The v1 file `review-experiment-protocol.md` stays byte-frozen in §2–§5, so its v1 lock still verifies. It gains one superseded banner above §0. + +**Runner:** the aprender traffic cop, sole minter. Tracking is one sub-epic under E4 (#4381) per APR-EPIC-001 §4d, with its phases replaced by §7 here. Epic #4354 (rows #4355–#4367) is collapsed into its checklist. +**Launch:** from `~/src/aprender`, run `Implement docs/specifications/PRM-001-prometheus.md autonomously.` +**Figure:** `docs/specifications/figures/prometheus-flow.png`. Re-export it: title "PROMETHEUS", remove the silver pool, and add the tie-breaker rung. +**Related:** +- E2 Verbs Are Fast (0.71) and E6 llama.cpp Parity (0.73): Prometheus is their real-workload acceptance test. +- ARB-APR-001, ARB-SELF-001, EXT-001, FLOW-001, APR-PERF-GATE-001 (J3 receipts). + +**Provenance marks:** `[V]` verified · `[C]` computed · `[A]` asserted · `[O]` operator ruling · `[U]` unmeasured · `[X]` third-party. No invented thresholds. + +--- + +## §0 Operating assumptions + +1. **One goal.** Qwen 3.5 4B on `apr` becomes the **4th-lane tie-breaker**. That needs three things: + - **fast:** it never lengthens a quorum; + - **available:** it answers ≥ 95% of rounds; + - **measured:** its accuracy on sealed gold is at least that of the voters it would overrule. + Speed is necessary; it is not sufficient. +2. **Speed is the critical path.** Review is prefill-heavy: a diff of 2k–32k tokens goes in, and a verdict plus at most 5 bullets comes out `[U until PRM-S2]`. The binding constraints are prefill and TTFT, not decode. +3. **The shadow has zero weight until promoted** `[O 2026-09-25]`. + - Every quorum receipt carries an apr row (`Verdict | Refused | NotRun{…}`); a receipt without one is RED. + - The apr verdict never blocks, breaks a tie or escalates until §5.4 grants a rung. +4. **Hosted-model outputs are never training targets** `[O 2026-09-25]`. + - Claude and agy/Gemini outputs are captured and stored (you own your outputs). They are used only as **replay inputs, latency baselines and evaluation comparators**. + - Training uses gold labels plus locally generated rows only: Qwen's own outputs and the local 27B teacher. + - This makes the program clean under Anthropic's help-center rule on Outputs, Anthropic Consumer §3(2) and Commercial §D.4, and the Google ToS ban on "using AI-generated content … to develop machine learning models" (§1 G19). It replaces the S-6 ruling. +5. **Capture first.** A quorum whose I/O is not captured at dispatch is lost for replay, evaluation and outcome joins. The order is capture → measure → speed → promote. +6. **A scientific experiment.** §2–§5 are frozen by the pre-registration lock `rex-prereg-v2` (PRM-00b), which locks **this v3**, before any measurement. v1 collected 0 data rows `[V]`. +7. **Producer is never the gate.** + - Released, sha-verified `apr` tags and sha256-pinned weights only. + - The writer of a trace never labels it. +8. **A refusal is a result.** `Refused{removed_by}` is recorded and filed, never retried under the same cell name. +9. **Genchi genbutsu.** Re-measure §1 at HEAD before relying on it. A false premise is recorded as `premise-falsified`. +10. **Foreign work is transferred.** + - Capture at dispatch → paiml-implement #436. + - Executors, the almacen store, the second copy and transcript retention → paiml/infra. + - Speed defects are aprender issues in E2/E6; Prometheus links them and does not own them. +11. **The operator has no pre-steps.** Pure Rust + bashrs-clean shell, no Python; provisioning by `forjar apply` or make only. + +--- + +## §1 Ground truth (frozen baseline; never quote as current) + +| # | Fact | Value | Mark | Source | +|---|---|---|---|---| +| G1 | n=1 pilot | 4B Q4_K_M, apr v0.69.3, lambda CPU at load 33–40: **175.8 s**; Haiku 4.5: **17.8 s** (≈ 10×) | [V] | diff `0c6932fd5` | +| G2 | Recall and precision of every lane, the voters included | unknown | [C] | — | +| G3 | apr v0.69.3 sha256 | `8a67a0103cbb036332908cc184d5a8a8ae422fb9025bd5c78e42d65df86cfbd7` | [V] | REX-04 receipt | +| G4 | Weights sha256 | 4B `00fe7986…69ef11a4` · 9B `03b74727…2b7e8` · 27B `84b5f7f1…d2a806` | [V] | REX-01 receipt | +| G5 | Train themes | 0.70 fast (`--json-schema`, resident serve) · 0.71 wider + training (qwen35 finetune/distill; WGPU/Metal) | [O 2026-09-20] | `06x-release-schedule.md` | +| G6 | `apr parity` v0.69.3 | GPU-vs-CPU only; no released llama.cpp comparison | [V] | REX-04 premise check | +| G7 | rc.1 lane on gx10 | F2 guard (cos 0.8425) → CPU fallback (#4313); `--context-length` ignored, 262144 used (#4443); decode ≈ 0.5× ollama; **32k prefill 0.26×**; temperature-0 divergence at ≈ 55 tokens | [V] cop report | #4313, #4443, `evidence/4252-promotion.md` | +| G8 | TTFT at 0.68.2 on a 4090 | 7.7× slower than llama.cpp; end-to-end 4.9× | [V] | #3596 | +| G9 | Epic speed targets | E2: TTFT ≤ 2× llama.cpp · E6: decode, prefill, TTFT ≥ 1.0× llama.cpp across the certified matrix | [O] | APR-EPIC-001 §2 | +| G10 | Executors | apr pin on gx10, lambda-labs, yoga; no `apr serve` unit, weights or review label anywhere | [V] | infra#1088 | +| G11 | paiml-implement | v1.8.1 has no advisory lane; #420 unreleased | [V] | PRM report | +| G12 | Ledger | findings stored as a count; `weights_sha256: None` hardcoded; free-text `LaneUnavailable` | [V] | `crates/aprender-review-experiment/src/ledger.rs` | +| G13 | Sealed corpus | `review-corpus-v1@787d2026256cc08b`: test = 35 P (size S, mutants) + 35 R + 35 G | [V] | REX-02 receipt | +| G14 | Corpus diffs | single copy on a 94%-full raid (`items-v1.tar`, sha `70dffe56…8bba`) | [V] | REX-02 receipt | +| G15 | Branch | `rex/001` @ `da7b6386f` folded into **closed** #4317; nothing on `main` | [V] | review package README | +| G16 | Volume | 50–200 quorums/day; ≤ ~0.45 TB/yr | [O] / [U] | PRM-C14 | +| G17 | Storage | almacen 3×16 TB RAID 5; re-pull time and scrub policy unmeasured | [V] / [U] | `almacen-storage-integration.md` | +| G18 | Transcripts | Claude Code deletes after `cleanupPeriodDays` (default 30) | [X] | Claude Code docs | +| G19 | Terms | Anthropic: training allowed only for non-competing specialised tools; "using Outputs as training targets" is prohibited. Google ToS: flat ban on developing ML models from AI-generated content. Claude Code under Max = Consumer Terms. **Not legal advice.** | [X] | support.claude.com #12326764; anthropic.com/legal; policies.google.com/terms | + +--- + +## §2 Design + +### §2.1 Lane dispatch and states + +Order `[O 2026-09-25, R-9 option b]`: `gx10-cuda → lambda-cuda → intel-wgpu → mini-metal → intel-cpu → NotRun{NoExecutor}`. + +- **lambda-cuda is rung 2 and shadow only.** + - Dispatch time only, and only when rung 1 returned NotRun **and** `/tmp/apr-gpu.lock` is free. + - Never a resident serve, VRAM preload, experiment cell, teacher or finetune. + - If the lock is taken mid-round, the row is `NotRun{Busy}` and the ladder continues. + - **Once the lane holds tie-breaker or vote, lambda-cuda leaves the ladder.** A deciding lane runs only on declared, non-contended cells (§4.1). +- **Rows record the backend that actually ran** (`gx10-cpu` on the F2 fallback). +- **train-active falls through; it never blocks.** +- **One typed state enum**, shared by receipt-lint (#436) and the ledger: `Verdict{PASS|FAIL}` · `Refused{removed_by}` · `NotRun{NoExecutor|Busy|Timeout|ContextOverflow|TrainActive}`. +- **Timeout per rung** = 2 × that cell's p95 once it has ≥ 10 rows; until then 300 s `[U]`. + +### §2.2 Capture (`agent-trace-v1`): the substrate for replay, evaluation and outcomes + +- **What:** a `TraceSpan` in the quorum rail (#436) hashes the exact wire bytes and streams the full reply into a blob, for every lane of every round. +- **On failure:** `trace_status = capture_failed` raises an andon. The quorum never waits on capture. + +| Field group | Contents | +|---|---| +| identity | `trace_id` (ULID), `quorum_id`, `round`, `lane`, `counted`, `provider`, exact `model_id` or weights sha, `access_channel`, `apr_tag`, `served_by`, `lane_state` | +| work unit | `repo`, `pr`, `head_sha`, `base_sha`, `diff_sha256`, `input_sha`, `input_parts`, `prompt_version` | +| result | `output_sha` (full, never truncated), `verdict`, `findings[{file,line_start,line_end,severity,category,text}]`, `parse_status` | +| **timing (first-class)** | `tokens{input,output,cached?}`, `latency_ms{queue,ttft,prefill,decode,total}`, `prefill_tps`, `decode_tps`, `load_snapshot` | +| local only | `logits_sha` → `sparse-logits-v1` (k=20 `[A]` + `logsumexp_full` + `topk_mass`); used only for local teacher → student | +| lifecycle | `outcome` ∈ {pending, merged, reverted_le14d, regression_escape, closed_unmerged}; `split_guard{repo#PR, split, dedup_cluster}`; `secret_scan`; `train_eligible` (computed by admission, §4.2) | + +- Second stream: Claude Code transcripts (including `subagents/`) and agy logs, harvested hourly `[A]`. +- These are stored for audit and replay only. They are never training data (§0.4). + +### §2.3 Storage + +``` +almacen:/traces/blobs/sha256/ab/cd/.zst write-once; address = sha of uncompressed bytes +almacen:/traces/index/agent-trace-v1/YYYY/MM/DD.jsonl append-only +almacen:/traces/transcripts/{claude-code,agy}//.zst +almacen:/traces/replay/review-replay-vN.jsonl frozen replay sets (§2.4) +almacen:/rex-corpus/v1/items-v1.tar the sealed corpus leaves the 94% raid +``` + +- Keep forever; quarantine only, never delete. +- **Second copy:** nightly, incremental by sha, to separate media. +- **Weekly restore drill:** 100 random blobs `[A]`, 0 mismatches. RAID 5 is not a backup `[C: P(≥1 URE) ≈ 0.92 on a degraded 32 TB read at 10⁻¹⁴]`. +- **Transcript retention:** a forjar-managed Claude Code setting, `cleanupPeriodDays: 36500` `[A]`. + +### §2.4 Replay benchmark (`review-replay-v1`): the speed instrument + +- **Source:** real quorum diffs from the trace index. Inputs only; no hosted output enters the benchmark. +- **Stratification:** by input tokens, 2k / 8k / 16k / 32k. +- **Set size:** 200 items per version `[A]`. Frozen per version; a new version each train. +- **Exclusions:** sealed-test hashes and clusters. +- **Run on every `apr` tag,** on the primary cell and on llama.cpp `d1d3c3396` with the same GGUF and cell. Output is `review-replay-receipt-v1` rows. +- **Measures:** review wall-clock p50/p95, TTFT, prefill tok/s per stratum, decode tok/s, peak anon memory, parse rate, and the verdict-identity rate vs the prior tag. +- **Queue budget reference:** the p95 of the **slowest counted voter on the same diffs**, taken from captured `latency_ms.total`. The speed target is measured, not asserted. +- **Workload profile (PRM-S2):** the measured distribution of input and output tokens per lane. It confirms or falsifies §0.2 and weights the strata. +- **Consumers:** this receipt is the **acceptance workload for E2 and E6**. Their `done_when` numbers are read on replay-v1, not on synthetic pp512/tg128 alone. + +### §2.5 Sealed evaluation corpus (`review-corpus-v1`) + +- As built (G13). +- At most 20 promotion decisions per sealed version `[A]`, then 50% rotation. +- Escapes feed the next version, never test v1. +- `review-corpus-contamination-v1` covers exact hashes and near-dup clusters. + +### §2.6 Experiment cells and arms + +| Cell | Host | Backend | Notes | +|---|---|---|---| +| C1 `intel-wgpu` | intel | wgpu | `Refused` expected before 0.71; must be observed | +| C2 `intel-cpu` | intel | cpu | **reference cell** | +| C3 `lambda-cpu` | lambda-labs | cpu | the GPU is never an experiment cell | +| C4 `gx10-cuda` | gx10 | cuda sm_121 | primary candidate; `gx10-cpu` on F2 fallback | +| C5a/C5b | mini | cpu / metal | C5b `Refused` expected before 0.71 | + +- **Arms:** 4B primary; 9B control (test split, reference cell plus top cell). +- **Evaluation comparators on the reference cell:** Haiku 4.5 and agy, exact ids. Evaluation only; no training. +- **Fixed:** prompt sha, greedy decoding, `max_tokens`. Overflow is `NotRun{ContextOverflow}`, never truncated. + +### §2.7 Metrics (deterministic; no model judges) + +| Metric | Definition | +|---|---| +| **Review wall-clock p50/p95** | request → parsed verdict, per item; primary speed metric | +| **TTFT; prefill tok/s by stratum; decode tok/s** | from `apr` timing and the serve metadata | +| **Speed ratio** | apr ÷ llama.cpp on the same cell and GGUF, per metric | +| Availability | Verdict rows ÷ rounds, rolling 7 d, by cause | +| Parse rate | parsed ÷ attempted | +| Recall, per class | FAIL on P ÷ \|P\|; FAIL on R ÷ \|R\| (reported separately) | +| False-refute rate | FAIL on G ÷ \|G\| (the HRQ cost; does not depend on prevalence) | +| Precision | descriptive | +| Localization | a FAIL finding names the defect's file | +| Cross-cell invariance | verdict identity with the reference cell (bytes are descriptive) | +| Determinism | byte-identical on a 10% same-cell re-run | +| **κ_err** | Cohen's κ on binary error indicators vs gold, for every lane pair (voter–voter and qwen–voter) | +| Interference | slowdown of a reference workload when the serve is co-located vs alone | + +--- + +## §3 Pre-registered hypotheses (locked by `rex-prereg-v2`; Holm family H1–H6, α = 0.05) + +| # | Hypothesis | Test | Consequence | +|---|---|---|---| +| H1 | **Verdict invariance** across admitted cells vs reference | count of divergent verdicts > 0 rejects | the cell is inadmissible; one issue per item | +| H2 | **Determinism** on the same cell | byte count on the 10% re-run > 0 rejects | the cell is inadmissible until fixed | +| H3 | **Size:** 9B correctness exceeds 4B | McNemar exact, one-sided | if rejected, §5.4 weighs 9B's gain against its wall-clock | +| H4 | **Precision ≥ min(Haiku, agy)** | paired bootstrap of `p4B − min(pH,pA)`, **min inside each resample**, 10k resamples, seed 4354; Holm-adjusted one-sided p decides; CI reported only | the tripwire gate | +| H5 | **R-class recall ≥ the Claude lane** | same machinery, R items only | the tie-breaker gate (with H4) | +| H6 | **Interference** | 5 runs alone vs 5 co-located; slowdown > 2 × sd(alone) rejects | a rejected cell ranks below every other cell | +| H7 | **Independence** (outside the Holm family; a gate) | κ_err(qwen, each voter) ≤ **max κ_err over voter–voter pairs** on the same items | the tie-breaker gate. It is δ-free: a tie-breaker may not be more correlated with a voter than the voters already are with each other | + +- **Sensitivity:** all runs (primary) vs loadavg1 < 0.5 × physical cores (secondary). Decisions read the primary column. +- **Sample size:** after PRM-05, if the Wilson half-width on R-recall > 0.12, grow R; hypotheses unchanged. + +## §4 Decision rules + +### §4.1 Hardware (lexicographic) + +1. **Admissible:** a parity receipt passes against its declared `parity.oracle` (`llama.cpp@d1d3c3396` once it is a declared input; until then `apr-parity-gpu-cpu` plus H1; `ds.yaml` min_cosine 0.98); H1 and H2 hold; the parse rate equals the reference cell's; the executor is forjar-declared; for tie-breaker or vote, the cell is **not** lambda-cuda. +2. Rank by replay p95 under natural load. +3. H6-rejected cells rank last. +4. Ties go to the larger 7-day idle fraction, then to a host outside the clean-room pool. +5. **Queue budget:** replay p95 ≤ the slowest voter's p95 on the same diffs (§2.4). A cell failing it is shadow-only. +6. **Output:** a primary cell and a failover cell. When both are down: `NotRun{NoExecutor}`. The quorum keeps width 3; a tie then goes to HRQ, as today. +7. Committed as `docs/audits/prm-001/hardware-ruling.md`. + +### §4.2 Admission (`trace-admit`, a separate binary) + +| Gate | Rule | Target | +|---|---|---| +| **G-PROV** | `train_eligible` only if provider = local (qwen, teacher-27b) **or** the row is a gold label (outcome, HRQ ruling, sealed-corpus label). Hosted outputs are never eligible | hosted rows in any training pool = **0** `[O]` | +| G-SEC | Rust scanner plus the gitleaks ruleset ported as data; a hit quarantines the row, with no redaction | 0 hit rows in a pool; 100% canary recall `[A]` | +| G-CON | 0 sealed hashes or near-dup clusters in a pool or replay set | 0 `[A]` | +| G-ID | provider, exact model id, access_channel, input/output sha, served_by, weights sha all present | 100% `[A]` | +| G-DUP | split by `repo#PR`, time-ordered, ≥ 14 d embargo, clusters closed to their earliest member | 0 cross-split clusters `[A]` | +| G-MAT | gold only after the 14 d outcome window; pending is never a negative | 0 immature gold `[A]` | +| G-HASH | blob re-hash equals its address | 0 mismatches `[A]` | + +- **Gold** = sealed-corpus labels, matured outcomes, and HRQ rulings (from tripwire onward). +- **Public release** (EXT-001) = base weights plus artifacts trained on G-PROV-eligible rows only. + +--- + +## §5 Improvement loop and promotion + +### §5.1 Speed loop (first priority; every aprender tag) + +| Lever | Owner | Expected effect | +|---|---|---| +| Keep the GPU path on gx10 (F2 guard) | #4313 | CPU → GPU; the largest single step | +| Honour `--context-length` | #4443 | right-sized KV | +| Prefill kernels and batched prefill | E2 → E6 | the dominant term (§0.2) | +| Resident `apr serve` plus a prefix cache for the fixed system prompt | E2 | TTFT | +| `--json-schema` | 0.70 | bounded output; parse rate → 1.0 | +| Knob sweep (threads; Q4_K_M vs Q8_0; KV) once per train | PRM-10 | per-cell optimum; a new quant goes through §5.4 | + +- **Ratchet:** replay p95 on the primary cell must never rise after 3 tags. A +10% planted regression turns RED (paired Harrell–Davis p95, landed). +- Every tag files one aprender issue per stratum where the apr/llama.cpp ratio worsened. + +### §5.2 Quality loop (no hosted-output targets) + +| Tier | Method | Available | +|---|---|---| +| Q1 | prompt versions | now | +| Q2 | retrieval few-shot from **gold** train-pool specimens (never the test split) | now | +| Q3 | constrained decoding (`--json-schema`) | 0.70 | +| Q4 | **local logit distillation:** Qwen3.5-27B on gx10 writes `sparse-logits-v1` over G-PROV-eligible inputs; the 4B student trains with forward-KL plus tail-mass correction; reverse-KL/GKD as challengers, using Qwen's own shadow outputs as on-policy samples | qwen35 `apr distill`, per D-7 | +| Q5 | QLoRA on gold labels only (`apr finetune`); merge (`apr merge`); re-quantize (`apr quantize`, now) | per D-7 / now | + +- Q4/Q5 are aprender acceptance tests: a qwen35 distill/finetune ticket is done only when its challenger passes §5.4. +- Until then the row reads `NotRun{VerbRefused(removed_by)}`. + +### §5.3 Champion/challenger (pre-registered) + +A challenger replaces the champion only if all of these hold on the current sealed version: +1. McNemar exact, one-sided, α 0.05: correctness improves. +2. The false-refute rate is not higher. +3. H1/H2 hold on the primary cell. +4. H7 holds. +5. G-CON and G-PROV hold on its training data. +6. Replay p95 meets the §4.1.5 queue budget. + +- Promotion is a forjar pin bump (weights and adapter sha); rollback is the previous sha. Each decision uses 1 of the 20. +- **Demotion:** 2 consecutive gold escapes where the champion's verdict decided the round → automatic HRQ and re-evaluation. + +### §5.4 Promotion ladder (`rex ladder`, contract `rex-promotion-ladder-v1`, extended by PRM-09) + +| Rung | Authority | Speed gate | Quality gate (sealed gold) | +|---|---|---|---| +| **shadow** (default; anything unreadable) | none | — | — | +| **tripwire** | a dissent against a 3–0 opens an HRQ, capped at D-8/day | replay p95 ≤ queue budget | H4 | +| **tie-breaker** | decides only a **1–1** split (a counted lane NotRun, or a 2-counted quorum) | queue budget **and** availability ≥ 95% over 7 d `[O cop crit. 4]`, on a non-lambda cell | H4 **and** H5 **and** H7, with H1/H2 on the primary cell | +| **vote** | counted voter | same | same, plus ≥ 20 decided tie-breaks `[A]` whose 14 d outcome agreement ≥ the voters' agreement on the same rounds | + +- The ladder reads only `prm-001-report-v1` and §5.3 reports carrying the `rex-prereg-v2` sha. +- It never skips a rung. +- A demotion drops exactly one rung. + +--- + +## §6 Hard rules (each has a §9 falsifier) + +- **R-1 Pre-registration is immutable.** `rex-prereg-v2` locks v3 §2–§5, `stats.rs`, analysis plan v2 and prompt v1. Any change is a new version, and earlier data is `exploratory`. +- **R-2 Sealed test.** Only the scorer reads it; it never appears in replay sets or pools. +- **R-3 Released artifacts only.** apr tag and weights by sha256. +- **R-4 Declared executors only.** No ad-hoc SSH or hand-installed apr. +- **R-5 Identity.** 0 `unknown` identity fields in receipts and ledger rows; `Coverage::holds()` requires `identity_gaps == 0`. +- **R-6 NotRun is never a pass,** and never in an accuracy numerator. +- **R-7 No invented thresholds.** `[A]` values are recalibrated after the first test version. +- **R-8 No Python.** +- **R-9 Lambda GPU.** + - 0 minutes for experiment cells, teacher, finetune and resident serve. + - Rung-2 shadow dispatch only (§2.1), with its minutes measured. + - 0 minutes once the lane holds tie-breaker or above. +- **R-10 train-active falls through, never blocks.** +- **R-11 No authority without a rung.** The lane decides nothing beyond its §5.4 rung. +- **R-12 Measurement states its tree.** +- **R-13 Capture is complete.** One `agent-trace-v1` row per lane call, NotRun included, with the full output and findings text. +- **R-14 Producer never labels.** +- **R-15 No secret in any pool or replay set;** no in-place redaction. +- **R-16 Blobs are write-once,** with a second copy within 24 h. +- **R-17 One spelling per lane state.** +- **R-18 Hosted outputs are never training targets.** G-PROV = 0 at every admission, in every pool manifest and in every release artifact. + +--- + +## §7 Tickets (EV-ordered) + +**Phases** (these replace APR-EPIC-001 §4d P0–P6): + +| Phase | Train | Content | +|---|---|---| +| **P0** | now | capture + shadow | +| **P1** | 0.71 (E2) | speed instrument + ratchet | +| **P2** | 0.71 | experiment + hardware ruling | +| **P3** | 0.72 | tripwire | +| **P4** | 0.73 (E6) | tie-breaker; prefill parity | +| **P5** | D-7 | local distill (Q4/Q5) | +| **P6** | 0.75 (E8) | vote | + +| EV | Row | Was | Phase | Work | Contract | Done when | K̂ | Status | +|---|---|---|---|---|---|---|---|---| +| 0 | PRM-00 | REX-00 | — | v1 prereg | `rex-prereg-v1` | — | — | **DONE** `5f0ae10ac` | +| 0 | **PRM-00b** | new | P0 | lock v3: §2–§5, `stats.rs` (min inside resample, R-class H5, H7 max-pair rule), analysis plan v2 | `rex-prereg-v2` | sha recorded; a planted §3 edit fails; 0 rows exist under v1 | 60 | todo, before any measurement | +| 0 | **PRM-L** | new | P0 | land `rex/001` plus this spec on `main` (own PR); banner on v1 file | — | merged; `rex prereg-check` green on `main` | 45 | todo | +| 0 | PRM-01 | REX-01 | P0 | split infra#1088: **gx10 slice** (4B weights by sha, `apr-review-serve` unit, cgroup, health, `exec_sha256`, train-active yield) as P0; corpus tar and replay sets to almacen | — | slice issue filed with checkable criteria | 20 | DONE → split todo | +| 0 | PRM-C1 | PRA T1 | P0 | capture at dispatch, incl. `latency_ms{queue,ttft,prefill,decode,total}` | `agent-trace-v1` | **transferred → pi#436**; F-R13 green there | — | foreign | +| 0 | **PRM-C2** | PRA T2 | P0 | `review-ledger-v2`: findings text, trace shas, weights sha, typed states | `review-ledger-v2` | FALSIFY: no weights sha → `holds()=false`; free-text reason → parse error; v1 fixtures parse | 120 | todo | +| 0 | PRM-C3–C5 | PRA T3–T5 | P0 | almacen store, second copy, transcript retention | `trace-blob-store-v1`, `trace-backup-v1`, `transcript-retention-v1` | **transferred → infra**; drill 0 mismatches; 31-day canary survives | — | foreign | +| 0 | **PRM-07** | REX-07 | P0 | shadow activation | `review-ledger-v2` | on gx10 slice + pi release: 100% of receipts carry an apr row; planted gx10-down → width 3, `NotRun{NoExecutor}` | 90 | active on its 2 deps | +| 1 | **PRM-S1** | new | P1 | replay benchmark `review-replay-v1` + runner (apr vs llama.cpp, same cell and GGUF) | `review-replay-v1` | 200 items, 4 strata, 0 sealed overlap; receipt per tag; queue budget derived from captured voter p95 | 150 | todo | +| 1 | **PRM-S2** | new | P1 | workload profile: input/output token distribution per lane | extends `agent-trace-v1` | §0.2 confirmed or falsified with receipt; strata weights set | 60 | todo | +| 1 | PRM-10 | REX-10 | P1 | perf ratchet wired to replay; per-tag issue per worsened stratum; knob sweep per train | `review-lane-perf-ratchet-v1` | 3 real tags recorded; planted +10% RED | 60 | gate landed `634916cbd` | +| 1 | PRM-02 / 03 | REX-02 / 03 | — | corpus v1; harness + scorer | — | — | — | **DONE** `8185ad1c3` / `0ab6ce731` | +| 2 | PRM-04 | REX-04 | P2 | cell admission re-run under the `parity.oracle` field | `rex-cell-admission-v1` | ≥ 1 cell `Admitted` or a receipted refusal | 30 | DONE (S-7) → re-run | +| 2 | PRM-C6 | PRA T6 | P2 | secret gate plus backfill | `trace-admission-secret-v1` | AWS-key literal plus 20 canaries quarantined | 180 | todo | +| 2 | PRM-C7–C9 | PRA T7–T9 | P2 | contamination on clusters; MinHash dedup; `repo#PR` time split | `trace-dedup-v1`, `trace-split-guard-v1` | perturbed sealed item refused; 0 cross-split clusters | 360 | todo | +| 2 | PRM-C10 | PRA T10 | P2 | outcome join (14 d) plus HRQ rulings as gold | `trace-outcome-join-v1` | 3-day-old gold refused; weekly matured count | 150 | todo | +| 2 | **PRM-C13** | PRA T12–T13 | P2 | G-PROV gate plus κ_err probe (all pairs) | `lane-independence-v1`, `trace-admission-prov-v1` | a planted hosted row in a pool → RED; κ matrix `[V]` | 120 | todo | +| 2 | PRM-C14 | PRA T14 | P2 | datacard (Croissant+RAI) plus weekly yield/bytes receipt | `trace-datacard-v1` | G16 `[U]` → `[V]` | 120 | todo | +| 3 | PRM-05 | REX-05 | P2 | pilot | extends receipt | sample-size rule on R-recall applied | 90 | parked (prep `469bf0c98`) | +| 3 | PRM-06 | REX-06 | P2 | full run plus comparators | extends receipt | 100% receipts or explicit NotRun; 0 unknown identity | 240 | parked | +| 4 | PRM-08 | REX-08 | P2 | analysis plus hardware ruling | `prm-001-report-v1` | H1–H7 verdicts; primary/failover or shadow-only with reason | 150 | parked | +| 5 | **PRM-09** | REX-09 | P3/P4 | ladder: add the **tie-breaker** rung; wire pi (#428) and arbiter (infra#1095) | `rex-promotion-ladder-v2` | planted reports: H4 only → tripwire; H4+H5+H7 → tie-breaker; lambda primary → refused | 60 | gate landed `4a4318f1c` → extend | +| 6 | PRM-11 | REX-11 | P3 | Q1–Q3 loop (prompt, gold few-shot, json-schema) | `review-champion-challenger-v1` | ≥ 1 challenger end-to-end; counter increments | 120 | gate landed `d8696650e` | +| 7 | PRM-C11 | PRA T11 | P5 | `sparse-logits-v1` in `apr serve` rows (local only) | `sparse-logits-v1` | header + residual mass round-trip | 180 | todo | +| 7 | PRM-12 | REX-12 | P5 | Q4/Q5: 27B teacher on gx10 → 4B student; gold QLoRA | `review-b2-loop-v1` | teacher dataset receipted (0 test hashes, 0 hosted rows); `NotRun{VerbRefused}` until D-7's train | 180 | gate landed `da7b6386f` | + +**Budget:** remaining aprender K̂ = **2,585 min `[A]`** `[C: sum of the rows above]` · **K = 2,840** (1.1 × K̂) · **andon at 2,275** (0.88 × K̂). Foreign rows are outside K. + +--- + +## §8 STOP conditions (exhaustive) + +- **S-1** A measurement before the `rex-prereg-v2` lock. +- **S-2** An undeclared `apr`, or a weights sha mismatch, on an executing host. +- **S-3** Any sealed hash or cluster in a pool or replay set. +- **S-4** Lambda GPU work outside rung-2 shadow, or any lambda work at tie-breaker or above. +- **S-5** A run overlaps train-active on a clean-room/CUDA pool without yielding. +- **S-6** **A hosted-model output appears in any training pool, manifest or release artifact (R-18).** +- **S-7** No admissible cell for the hardware ruling. Capture, shadow and P0/P1 rows continue. +- **S-8** A harness hook refuses writes for a reason other than a missing ticket. +- **S-9** K is reached, or the andon is crossed with more than one row incomplete. +- **S-10** Two consecutive non-passing quorums on the same PR. +- **S-11** A secret must be purged from all copies. +- **S-12** Almacen free < 20%, or the second-copy target is missing from forjar inventory. +- **S-13** Any change grants authority outside §5.4. +- **S-14** Transcripts contain third-party personal data beyond author identities. + +### Operator decisions + +| ID | Question | Status / default | +|---|---|---| +| R-9 | lambda GPU | **decided: option b**, shadow only (§2.1) | +| Training data | hosted outputs as targets | **decided: never** (§0.4, R-18) | +| D-7 | qwen35 finetune/distill: 0.71 (G5) or 0.75 (E8)? | default 0.71 under E4 as the PRM-12 acceptance test; E8 keeps multi-engine CRUX | +| D-8 | tripwire HRQ cap per day | none; blocks P3 only | +| D-10 | move the counted Claude lanes from Max OAuth to API keys (Commercial Terms; Max limits assume "ordinary, individual usage") | none; does not block Prometheus | + +--- + +## §9 Final report schema and scoreboard + +```yaml +spec: PRM-001 +version: 3 +prereg_sha: +session: {date_utc, host, tree: {head, origin_main, worktree}, model} +selected_row: PRM-NN +ticket: PMAT-NNNN +outcome: merged | stopped | already-done | premise-falsified | foreign-linked +stop: {id: S-n, evidence, resume_when} +baseline_remeasured: [{row: G#, value, mark, command}] +lane: {rung: shadow|tripwire|tie-breaker|vote, rows_7d, availability_7d, by_state: {}, by_served_by: {}, lambda_gpu_min} +speed: # PRM-S1 / PRM-10, per tag + replay_version: review-replay-vN + apr_tag: vX.Y.Z + cell: + p50_s: ; p95_s: ; p95_ci: [] + queue_budget_p95_s: ; within_budget: bool + ttft_ms: {apr: , llama_cpp: , ratio: } + prefill_tps: {"2k": {apr: , llama_cpp: , ratio: }, "8k": {}, "16k": {}, "32k": {}} + decode_tps: {apr: , llama_cpp: , ratio: } + parse_rate: ; verdict_identity_vs_prev_tag: +workload: {input_tokens_p50_p95: {}, output_tokens_p50_p95: {}} +corpus: {rows_7d, completeness_7d, capture_failed_open, gold_matured_7d, hosted_rows_in_pools: 0, quarantined_7d, restore_drill_mismatch} +cells: [{cell, status, reason, apr_tag, apr_sha, weights_sha, parity: {oracle, cosine, receipt}}] +results: {hypotheses: [{id: H1..H7, statistic, ci, p_holm, verdict}], + per_cell: [{cell, n, recall_P, recall_R, precision, false_refute, localization, parse_rate, p50_s, p95_s, ttft_ms, prefill_tps, decode_tps, peak_anon_gb, llama_cpp_ratio, interference}], + comparators: [{lane, model_id, recall_P, recall_R, precision, false_refute, p95_s}], + kappa_err: [{pair, kappa, n}]} +hardware_ruling: {primary, failover, rung_eligible, reason} +promotion: {champion: {weights_sha, adapter_sha, prompt_sha}, challengers: [{id, verdict, mcnemar_p, false_refute_delta, kappa_err}], test_version, evaluations_used, tie_breaks_decided, tie_break_outcome_agreement} +issues_filed: [{repo, url, purpose}] +budget: {k_hat_row, k_actual_row, cumulative, andon_crossed} +next_row: PRM-NN +``` + +| Scoreboard metric | Target | Rendered by | +|---|---|---| +| **Replay p95 on the primary cell** | ≤ slowest-voter p95 (measured); never up after 3 tags | PRM-S1 / PRM-10 | +| **TTFT ratio vs llama.cpp** | ≤ 2× by E2 (0.71) `[O]` | replay receipt | +| **Prefill ratio vs llama.cpp, per stratum** | ≥ 1.0× by E6 (0.73) `[O]` | replay receipt | +| Decode ratio vs llama.cpp | ≥ 1.0× by E6 `[O]` | replay receipt | +| Verdict availability, rolling 7 d | ≥ 95% before tie-breaker `[O]` | ledger | +| Receipts carrying an apr row | **100%** | receipt-lint | +| Closes decided by apr above its rung | **0** | receipt audit | +| Rows whose cell ≠ the backend that ran | **0** | `served_by` audit | +| Capture completeness; `capture_failed` open > 24 h | **100% / 0** | trace index | +| Hosted rows in any training pool or release | **0** | G-PROV | +| Secrets / sealed hashes / cross-split clusters in pools or replay | **0 / 0 / 0** | §4.2 | +| Restore mismatches; blobs without a second copy after 24 h | **0 / 0** | backup | +| Lambda GPU minutes outside rung-2 shadow | **0** | host receipts | +| Champion sealed-test correctness | never down across promotions | §5.3 | +| κ_err(qwen, voter) | ≤ max voter–voter κ_err | H7 | +| Python lines in files this spec touches | **0** | diff scan | + +**Five whys (why Qwen cannot break a tie today):** +- A review takes minutes, not seconds (G1: ≈ 10× Haiku). +- The GPU path falls back to CPU (#4313), and prefill runs at 0.26× (G7). +- Prefill was optimised against synthetic pp512, never against the real review workload, and the lane waited on a 4-host executor batch. + +Terminal causes, as mechanisms: +- **no real-workload speed gate** → fixed by PRM-S1 as the E2/E6 acceptance test; +- **batch-coupled executor dependency** → fixed by PRM-01, one slice per executor. diff --git a/docs/specifications/figures/prometheus-flow.png b/docs/specifications/figures/prometheus-flow.png new file mode 100644 index 0000000000..353732a8d3 Binary files /dev/null and b/docs/specifications/figures/prometheus-flow.png differ diff --git a/docs/specifications/research-design-pr-agent-qwen-3.5.md b/docs/specifications/research-design-pr-agent-qwen-3.5.md new file mode 100644 index 0000000000..b660cba1c4 --- /dev/null +++ b/docs/specifications/research-design-pr-agent-qwen-3.5.md @@ -0,0 +1,483 @@ +# research-design-pr-agent-qwen-3.5.md — PRA-001: Capture-First Trace Corpus for Distilling a Local Qwen 3.5 PR Reviewer + +> **SUPERSEDED** by [`PRM-001-prometheus.md`](PRM-001-prometheus.md) (PRM-001 v3, "PROMETHEUS"), which folds PRA-001 and REX-001 into one programme. This document is kept unchanged below as the design record. Where the two differ, PRM-001 wins. + +Every quorum is currently throwing away its training pairs. The fix is to capture every lane's exact input bytes, full raw output and findings text at dispatch, under a content-addressed and provenance-tagged `agent-trace-v1` contract, before any training happens. Silver and gold labels stay separable on every row, and a measured lane-independence guard (error-kappa) controls whether silver distillation is allowed at all. + +## TL;DR +- **Capture before training.** `review-ledger-v1` stores findings as a count, so each of the 50–200 quorums/day [A] loses its (input, output, findings) triples. PRA-001 moves capture to dispatch inside the quorum rail (paiml-implement #436). Each row stores the exact input bytes, the full raw reply, decoding params, top-k=20 [A] logits for local lanes, provider, exact model id, access channel and terms reference. Rows go into content-addressed zstd blobs on almacen, with a nightly second copy on separate media. +- **Silver is allowed but tagged and fenced.** The literature supports a conservative design. Top-k logit caches give biased teacher estimates (Anshumann et al., ACL 2025: top-K caching gives "biased estimates of teacher probability distribution"), and so does sequence-level KD alone. Code-review corpora carry heavy label noise: Liu et al. found only 64% of the original CodeReviewer comments were valid. They also carry near-duplicate/temporal leakage: Lee et al. (ACL 2022) found train-test overlap in over 4% of standard validation sets, and Allamanis found code-duplication inflates metrics by up to 100%. LLM errors are strongly correlated across providers: Kim et al. (ICML 2025) found model pairs agree about 60% of the time when both are wrong on HELM, against 1/3 by chance. So every row carries `label_tier`, `provider`, `model_id` and `terms_ref`. Splits are grouped by repo#PR and ordered by time, and outcome labels mature only after the 14-day [A] revert window. +- **The dissent lane has to stay independent.** Pairwise Cohen's κ on errors against gold is measured before and after any silver training. If κ rises above the pre-registered baseline + margin [A], the silver-trained candidate is refused as the Qwen shadow lane; it may still be promoted elsewhere via PRM-001 §5.4. Anthropic's and Google's current terms both contain competing-model/ML-development restrictions (summarised in §2.10 and Appendix A.7, not legal advice). The operator has ruled S-6, so the design's job is to make any provider's rows filterable later. + +--- + +## Header + +| Field | Value | +|---|---| +| Spec id | **PRA-001** (PR-Review Agent trace corpus) | +| Title | Research design: collecting PR-review agent data to fine-tune and distill a local Qwen 3.5 reviewer | +| Author | Noah Gift, Pragmatic AI Labs (paiml), Sovereign AI Stack | +| Date | 2026-09-25 | +| Target repo / path | `paiml/aprender` → `docs/specifications/research-design-pr-agent-qwen-3.5.md` | +| Runner | the aprender traffic cop | +| Launch line | `Implement docs/specifications/research-design-pr-agent-qwen-3.5.md autonomously.` | +| Related specs | REX-001 / PRM-001 "Prometheus" (H1–H6, champion/challenger §5.4, sealed-test rotation after 20 [A] promotion decisions); ARB-SELF-001 (dogfood loop, outcome ledger, escape join); ARB-APR-001 (arbiter on apr); FLOW-001 (fleet flow rules); EXT-001 (continuous HF release of the dogfooded model) | +| Contracts introduced | `agent-trace-v1`, `review-ledger-v2`, `trace-blob-store-v1`, `trace-backup-v1`, `transcript-retention-v1`, `trace-admission-secret-v1`, `trace-dedup-v1`, `trace-split-guard-v1`, `trace-outcome-join-v1`, `sparse-logits-v1`, `lane-independence-v1`, `trace-datacard-v1` | +| Contracts reused | `review-corpus-contamination-v1` (sealed test, corpus `review-corpus-v1`) | + +**Provenance marks (apply to every number in this document)** + +| Mark | Meaning | Obligation | +|---|---|---| +| **[V]** | Verified by a measurement command in this repo/fleet | The command is quoted beside the number | +| **[C]** | Computed from other marked numbers | The arithmetic is shown; it inherits the weakest input mark | +| **[A]** | Asserted (operator decision or design choice) | Changing it is a spec change, not a tuning knob | +| **[U]** | Unmeasured estimate | Must be replaced by [V] at the first scheduled probe (§3), or the row goes to andon | +| **[X]** | Third-party figure (paper, vendor, docs) | Cited in Appendix A; never used as a gate threshold without local re-measurement | + +Rule: there are no invented thresholds. A gate threshold is either [V] (a measured baseline plus a declared ratchet) or [A] (an explicit policy choice such as 0 or 100%). + +--- + +## §0 Operating assumptions + +1. **Capture-first.** Anything not captured at dispatch is lost forever. Capture comes before curation, curation before training, and training before promotion. No sampling and no TTL [A]. +2. **Silver and gold are separable per row, forever.** `label_tier ∈ {silver, gold, pending, quarantined}` together with `provider` and `model_id` is enough to filter any row out of any pool without re-deriving anything. +3. **The sealed test is inviolable.** The sealed test (105 items, 70 defects [A], corpus `review-corpus-v1`) is never in any training pool. Zero sealed hashes in any pool [A]. Rotation follows PRM-001 only. +4. **The shadow lane is zero-weight.** The Qwen row is mandatory in every receipt, but it never blocks, breaks a tie or escalates. PRA-001 changes what is *recorded*, not how votes are counted. +5. **The producer is never the gate.** The process that writes a trace never admits it to the training pool. Admission (§4) is a separate binary with its own contract. +6. **The operator has no pre-steps.** Anything only the operator can do (credentials, media purchase, ToS decisions, sealed-corpus changes) is a §8 STOP, not a hidden prerequisite. +7. **Pure Rust + bashrs-clean shell. No Python anywhere.** That includes training-data tooling, scanners and dedup. Provisioning uses only `forjar apply` or make targets, with released and sha-verified `apr` tags. Weights are pinned by sha256. +8. **Legal posture.** Provider terms are recorded as row metadata (§2.10). This document is **not legal advice**. The operator's S-6 ruling governs; PRA-001 only guarantees that rows can be filtered later. + +--- + +## §1 Ground truth (frozen baseline, 2026-09-25) + +Every operator-supplied fact is [A] until ticket T0 converts it to [V] by running the listed command. If a measured value disagrees with the [A] value, that is a §8 STOP (baseline drift). + +| # | Fact | Mark | Source of truth | Verify command (T0) | +|---|---|---|---|---| +| G1 | Quorums/day across paiml repos: 50–200 | [A] | quorum receipts | `make trace-yield WINDOW=7d` | +| G2 | Counted lanes: Claude Sonnet 5, Claude Haiku 4.5, agy (Antigravity / Gemini 3.1 Pro); shadow: Qwen3.5-4B-Q4_K_M via `apr serve` | [A] | paiml-implement rail config | `pmat quorum lanes --json` | +| G3 | Qwen row states: Verdict \| Refused \| NotRun{NoExecutor\|Busy\|Timeout\|ContextOverflow\|TrainActive} | [A] | receipt schema | receipt schema test | +| G4 | Dispatch ladder: gx10-cuda → lambda-cuda (shadow-only, dispatch-time, GPU-lock free, never resident) → intel-wgpu → mini-metal → intel-cpu → NotRun{NoExecutor}; row records the backend that actually ran | [A] | rail dispatcher | `make trace-served-by-audit` | +| G5 | `review-ledger-v1` keeps {family, model, verdict, findings: count} only; text/inputs/outputs discarded | [A] | `crates/aprender-review-experiment/src/ledger.rs` | `grep -n 'findings' crates/aprender-review-experiment/src/ledger.rs` | +| G6 | S-6: Claude and agy outputs MAY train the local model (silver) | [A] | operator ruling S-6 | n/a (ruling) | +| G7 | Gold = sealed corpus + outcome joins (merged / reverted ≤ 14 days / `regression`-labelled escape) | [A] | ARB-SELF-001 outcome ledger | `make trace-outcome-join --dry-run` | +| G8 | Sealed test: 105 items, 70 defects, corpus `review-corpus-v1` | [A] | `review-corpus-contamination-v1` | `pv check review-corpus-contamination-v1` | +| G9 | Storage: almacen UNAS Pro, 3×16 TB RAID 5, 32 TB usable, 10G | [A] | forjar inventory | `forjar inventory almacen --json` | +| G10 | Raw growth ≤ ~0.45 TB/yr | [U] | estimate | `make trace-bytes WINDOW=7d` (replace in week 1) | +| G11 | A real AWS-key literal has already landed in a diff | [A] | incident record | `make trace-secret-scan-backfill` | +| G12 | `apr finetune/distill/merge` refuse qwen35 until aprender 0.71; `apr quantize` available; `--json-schema` planned 0.70 | [A] | aprender release notes | `apr --version && apr finetune --model-arch qwen35 --dry-run` (expect refusal) | +| G13 | Qwen3.5-27B teacher on gx10 can write top-k logits | [A] | apr capability | `make teacher-logit-smoke` | +| G14 | Claude Code stores transcripts in plaintext under `~/.claude/projects/` "for 30 days by default" (Claude Code docs); Desktop/Cowork sessions exempt by default | [X] | Claude Code docs (App. A.8) | `make transcript-retention-audit` | + +--- + +### §1.1 Amendment (2026-09-25, T0): verify commands that exist + +T0 (`evidence/pra-001/t0-baseline.yaml`) found that none of the §1 verify commands exist as written: the seven `make` targets, `pmat quorum`, `pv check` and `apr --model-arch`. Cop ruling (2026-09-25): a missing instrument is not a contradiction. G1, G7, G10, G11 and G13 are **[U]-pending-instrument**, each tied to the ticket that builds it. G2 and G4 are paiml-implement#436 scope (T1, R-9). This table replaces the §1 "Verify command" column. A `make` target named in §1 lands with its ticket and replaces the interim command here. + +| # | Verify command (real, today) | Until then | +|---|---|---| +| G1 | lands with T1: `trace::weekly` row count over the index | [U]-pending-instrument (T1) | +| G2 | `jq '.quorum' ~/.claude/skills/paiml-implement/config.json` | STOP S-BASE → #436 scope | +| G3 | `cargo test -p aprender-review-experiment --lib every_wire_form_round_trips` | — | +| G4 | the #436 dispatcher's row `served_by` audit; no dispatcher exists yet | STOP S-BASE → #436 R-9 | +| G5 | `grep -n 'findings' crates/aprender-review-experiment/src/ledger.rs` | — | +| G6 | n/a (ruling) | — | +| G7 | lands with T10 (`trace-outcome-join-v1`) | [U]-pending-instrument (T10) | +| G8 | `pv validate contracts/review-corpus-contamination-v1.yaml && cargo test -p aprender-review-experiment --lib contamination` | — | +| G9 | `df -B1` on the almacen mount, on a host that mounts it | — | +| G10 | lands with T1: `trace::weekly` `bytes_raw`/`bytes_zstd` over week 1 | [U]-pending-instrument (T1) | +| G11 | lands with T6 (secret-scan backfill over the index) | [U]-pending-instrument (T6) | +| G12 | `. scripts/apr_bin.sh && "$APR" --version && "$APR" finetune --plan` (expect a refusal; `apr` has no `--model-arch`) | STOP S-BASE: `--plan` exits 0 with no refusal on 0.69.3 | +| G13 | lands with T16 (27B teacher on gx10, infra#1132) | [U]-pending-instrument (T16) | +| G14 | per host: `jq .cleanupPeriodDays ~/.claude/settings.json` and `find ~/.claude/projects -mtime +30 \| wc -l` | — | + +--- + +## §2 Design + +### §2.1 Capture points + +| Stream | Where | When | Why here | +|---|---|---|---| +| **S1: lane traces** | Inside the quorum rail (paiml-implement, issue #436), at the dispatch boundary of each lane call | Synchronously, before the verdict is parsed and before the receipt is written | This is the only point where the exact input bytes, the raw reply and the actual backend co-exist. Parsing is lossy and receipts are summaries. | +| **S2: session transcripts** | Claude Code `~/.claude/projects/**/*.jsonl` (including `subagents/`), agy logs | Harvested by a forjar-declared timer on each workstation into almacen; never edited in place | For later agentic (multi-turn, tool-use) distillation. Retention is set by config, not by hope (§2.8). | + +Mechanism for S1: the rail wraps every lane call in `TraceSpan::open(lane, round)`. The span hashes the request bytes *as serialised to the wire*, not a reconstruction. It streams the response into a blob writer while the lane client reads it. It closes with the parsed verdict and findings. If the blob write fails, the receipt row is written with `trace_status = CaptureFailed` and andon fires. The quorum itself is never blocked (jidoka on the capture line, not on the review line). + +### §2.2 `agent-trace-v1` row schema (one row per lane per round) + +| Field | Type | Required | Notes | +|---|---|---|---| +| `schema` | const `"agent-trace-v1"` | ✔ | versioned; readers reject unknown majors | +| `trace_id` | ULID | ✔ | time-sortable | +| `quorum_id` | string | ✔ | joins to receipt | +| `round` | u16 | ✔ | re-review rounds within a quorum | +| `lane` | enum {sonnet, haiku, agy, qwen-shadow, teacher-27b, …} | ✔ | | +| `counted` | bool | ✔ | false for qwen-shadow and teacher | +| `provider` | enum {anthropic, google, local} | ✔ | filter key for terms (§2.10) | +| `model_id` | string (exact, e.g. dated snapshot id or weights sha256) | ✔ | never a family alias | +| `access_channel` | enum {anthropic-api, claude-code-cli, antigravity, gemini-api, vertex, apr-serve} | ✔ | terms differ by channel (App. A.7) | +| `terms_ref` | {url, effective_date, fetched_at} | ✔ | snapshot of governing terms at capture | +| `repo`, `pr`, `head_sha`, `base_sha` | string/u64 | ✔ | | +| `diff_sha256` | hex | ✔ | sha of the exact diff bytes sent | +| `input_sha` | hex | ✔ | sha of the full wire request (system prompt + prompt version + diff + tools) | +| `input_parts` | [{role, blob_sha, bytes}] | ✔ | enables dedup of shared system prompts/diffs across lanes | +| `prompt_version` | semver | ✔ | | +| `output_sha` | hex | ✔ | full raw reply, including any returned reasoning; never truncated | +| `decoding` | {temperature, top_p, top_k, max_tokens, seed?, json_schema_sha?} | ✔ | provider-reported where available, otherwise as sent | +| `tokens` | {input, output, reasoning?, cached?} | ✔ | provider-reported; `null` is not the same as 0 | +| `latency_ms` | {ttft?, total} | ✔ | | +| `served_by` | string (cell that actually ran, e.g. `gx10-cpu`) | ✔ | records parity-guard fallback | +| `lane_state` | enum Verdict \| Refused \| NotRun{…} | ✔ | NotRun rows carry no output blob but are still rows | +| `verdict` | enum {approve, request_changes, comment, abstain} | if Verdict | | +| `findings` | [{file, line_start, line_end, severity, category, text}] | if Verdict | **text, not count** | +| `parse_status` | enum {ok, repaired, failed} | ✔ | failed parses keep raw output | +| `logits_sha` | hex \| null | local lanes | `sparse-logits-v1` blob (§2.7) | +| `label_tier` | enum {silver, gold, pending, quarantined} | ✔ | set by admission, never by producer | +| `outcome` | enum {merged, reverted_le14d, regression_escape, closed_unmerged, pending} | ✔ | via `trace-outcome-join-v1` | +| `split_guard` | {group_key: "repo#pr", split: train\|val\|test\|sealed, assigned_at, dedup_cluster} | ✔ | §2.9 | +| `secret_scan` | {scanner_versions, hits, status} | ✔ | §4 | +| `trace_status` | enum {ok, capture_failed, partial} | ✔ | | +| `producer` | {binary, version, git_sha} | ✔ | provenance of the writer | + +### §2.3 Blob layout on almacen + +``` +/almacen/traces/ + blobs/sha256/ab/cd/.zst # content-addressed, zstd, immutable, write-once + index/agent-trace-v1/YYYY/MM/DD.jsonl # uncompressed JSONL index, append-only + transcripts/claude-code//.zst + transcripts/agy//.zst + quarantine/sha256/… # secret-hit blobs, mode 0400, excluded from pool + manifests/YYYY-MM-DD.sha256 # daily Merkle-style manifest of new blobs + datacard/ # Croissant + RAI metadata (§2.11) +``` + +- The address is the sha256 of the **uncompressed** bytes. The compression level is recorded in the index, never implied by the path. This allows recompression, or later dictionary training, without re-addressing. +- Blobs are write-once (`O_EXCL`), and existing addresses are skipped, which gives free dedup of shared system prompts and diffs across the 4 lanes. The zstd level defaults to 19 [A] for archival blobs and 3 [A] for hot index rotation; replace both with the measured ratio from §3 M8. +- The index is uncompressed so that `grep`/`jq`-class Rust tools and the mining index (§2.9) can read it without decompression. + +### §2.4 Second copy (RAID 5 is not a backup) + +- Nightly `trace-backup-v1`: every blob named in `manifests/.sha256` is copied to separate media on a separate host. This is 3-2-1 in shape: primary almacen, second media, and a third offsite copy as a §8 STOP pending operator media. +- Weekly restore drill: sample N=100 [A] random blobs from the second copy, decompress them, re-hash them, and compare. Target mismatch = 0 [A]. +- Rationale (computed, worst case): a degraded 3×16 TB RAID 5 rebuild reads the two surviving 16 TB members, 32 TB ≈ 2.56×10¹⁴ bits [C]. At a consumer URE spec of 1 per 10¹⁴ bits [X], the expected UREs are ≈ 2.56 [C] and P(≥1) ≈ 1 − e^−2.56 ≈ 0.92 [C]. At an enterprise spec of 1 per 10¹⁵ [X], P ≈ 0.23 [C]. Spec rates are warranty floors, and field rates are lower [X]. But "kept forever" plus a single-parity array is not a durability story. The second copy is mandatory. + +### §2.5 Index schema (JSONL, one object per row) + +This is the full `agent-trace-v1` row minus blob contents, plus the mining columns: + +| Column | Definition | +|---|---| +| `agreed` | all counted lanes returned the same verdict in this round | +| `lane_disagreement` | bitmask of lanes that differ from the counted majority (Qwen included, as a measurement only) | +| `outcome` | as §2.2 | +| `label_tier` | as §2.2 | +| `unique_diff` | true if this `diff_sha256` is not in any earlier row and not in a near-dup cluster already seen (§2.9) | +| `dedup_cluster` | MinHash-LSH cluster id over diff shingles | +| `outcome_matured_at` | timestamp when the 14-day [A] revert window closed; null means pending | + +### §2.6 Retention + +Keep forever. No sampling, no TTL [A]. A row is never deleted. The only operation is to *quarantine* it (move the blob, flip `label_tier`). Deletion requests (for example, a leaked secret that must be purged from all copies) are a §8 STOP handled by the operator. The index records the tombstone. + +### §2.7 Logit storage format (`sparse-logits-v1`) + +The design choice is to store top-k **plus the residual mass**, and to record the sampling policy. Future distillation objectives (forward KL, reverse KL, on-policy) need more than the top-k ids. + +Per output token t: + +``` +struct TokenLogits { + token_id_sampled: u32, + k: u8, // 20 [A] + ids: [u32; k], // u32 because Qwen vocab > 2^16 [A: verify via `apr inspect`] + logprobs: [f16; k], // log-softmax at temperature 1.0, pre-sampling-filters + logsumexp_full: f32, // normaliser over full vocab + topk_mass: f16, // Σ exp(logprobs) = retained mass M ≤ 1 +} +``` + +- Header per blob: `{model_sha256, tokenizer_sha256, apr_tag, backend(served_by), temperature_of_record, k, n_tokens}`. +- Encoding: columnar (all ids, then all logprobs), then zstd. Size ≈ 20 × (4+2) + 8 ≈ 128 B/token raw [C]. That is ≈ 256 KB [C] for a 2,000-token [U] review. +- Why residual mass: naive top-k caching gives biased estimates of the teacher distribution and hurts calibration [X, App. A.1]. A fused forward-KL objective over a sparse top-K teacher explicitly uses retained mass M ≤ 1 [X]. Storing `topk_mass` and `logsumexp_full` keeps the "tail bucket" correction available. +- Option reserved: `sparse-logits-v1` has a `mode` field {topk, rs-sample}. It allows importance-sampled (RS-KD-style) token sets to be written by the 27B teacher later without a schema change [X, App. A.1]. +- Frontier lanes (Claude, agy) provide no full logits. Their rows carry `logits_sha = null`; sequence-level KD only. + +### §2.8 Session transcripts (S2) + +- `transcript-retention-v1`: `forjar apply` writes Claude Code **managed settings** fleet-wide with `cleanupPeriodDays` set to a large value (36500 [A]). The value must be ≥1 [X]; 0 fails validation in current versions [X], and in an earlier version it silently disabled persistence [X]. Managed settings take precedence over user settings [X]. +- The harvester copies `projects/**/*.jsonl` **and** `projects/**/subagents/*.jsonl` to almacen hourly [A]. These are content-addressed, so re-harvesting an appended file stores a new blob, and the index links versions by `session_id`. +- Transcripts are plaintext and include tool outputs, file contents and anything a command printed [X]. They are the highest secret-risk stream, so every transcript blob passes §4 G-SEC before it is eligible for any pool. +- Falsifier: plant a canary transcript older than 31 days [A] on a test host. Run a Claude Code session start. Assert the canary survives. + +### §2.9 Split policy (`trace-split-guard-v1`) + +1. **Group key = `repo#pr`.** Every row, round and re-review of a PR goes to one split [A]. Rows are never split individually. Re-reviews are near-duplicates. +2. **Time-ordered.** Splits are assigned by PR *open time* with forward-chaining cutoffs: train < val < test in time [A]. There is an embargo gap ≥ 14 days [A] between train cutoff and val start, so that outcome labels in val cannot leak through revert chains. +3. **Cross-PR near-dup closure.** Cherry-picks, backports and re-opened PRs produce near-identical diffs under different keys. Diff shingles are MinHash-LSH clustered. A union-find closure assigns every cluster to the split of its **earliest** member [A]. Target cross-split cluster count = 0 [A]. +4. **Sealed firewall.** Any row whose `diff_sha256` or cluster matches `review-corpus-v1` sealed items gets `split = sealed` and is never in a pool (G-CON, §4). +5. **Outcome maturity.** A row is eligible for gold only after `outcome_matured_at` is set. Pending is never treated as negative [A]. + +Mining at training time: oversample `lane_disagreement ≠ 0` and `outcome ∈ {reverted_le14d, regression_escape}`. The oversampling factor is chosen per run and logged; it is not a spec constant. + +### §2.10 Provider-terms provenance (tagging requirement, not advice) + +Every row carries `provider`, `model_id`, `access_channel` and `terms_ref`. The governing terms differ by channel (App. A.7). The pool builder takes a `--exclude-provider` / `--exclude-channel` filter as a first-class argument. Rebuilding any pool without a given source is a single command, and it is falsified in §9 (P-TERMS). **Not legal advice; S-6 governs.** + +### §2.11 Dataset documentation (`trace-datacard-v1`) + +- Machine-readable: a Croissant JSON-LD file (dataset metadata, resources, record sets, semantics) with the Croissant-RAI extension fields: collection process, labelling (silver/gold provenance), limitations, intended use, PII/secret handling. It is generated by a Rust binary from the index, never hand-edited. +- Human-readable: a Datasheet-style card (motivation, composition, collection, preprocessing, uses, distribution, maintenance) under `datacard/`. It is regenerated per snapshot. +- Publication to HF (EXT-001) is gold-and-local-only by default [A]. Rows with `provider ∈ {anthropic, google}` are excluded from any public release unless the operator overrides this at a §8 STOP. + +--- + +## §3 Research questions and pre-registered measurements + +All measurements run from make targets delivered by the tickets in §7. The first run sets the [V] baseline, and later gates ratchet from it. + +| ID | Question | Metric (definition) | Command | Baseline mark now | +|---|---|---|---|---| +| M1 | Yield | admitted rows/week by `label_tier` × lane | `make trace-yield WINDOW=7d` | [U] | +| M2 | Capture completeness | rows with all ✔ fields ÷ lane calls dispatched | `make trace-completeness` | target 100% [A] | +| M3 | Disagreement rate | fraction of rounds with `lane_disagreement ≠ 0` among counted lanes | `make trace-disagreement` | [U] | +| M4 | Error overlap | Cohen's κ on binary error indicators vs gold, per lane pair (incl. Qwen); CAPA as secondary | `make lane-kappa SPLIT=val` | [U] | +| M5 | Near-dup rate | fraction of rows in a cluster of size > 1; cross-split clusters | `make trace-dedup-report` | cross-split = 0 [A]; rate [U] | +| M6 | Bytes/round | raw and compressed bytes per lane-round, split by blob kind (input, output, logits) | `make trace-bytes WINDOW=7d` | [U] (G10 ≤ ~0.45 TB/yr) | +| M7 | Secret hit rate | hits ÷ rows scanned, by stream (S1 diff, S1 output, S2 transcript) | `make trace-secret-report` | [U] | +| M8 | Compression ratio | raw ÷ zstd bytes by blob kind and level | `make trace-zstd-bench` | [U]; literature suggests 6–7× for JSONL at level 3 [X] | +| M9 | Outcome label yield | gold rows matured/week; revert and escape counts | `make trace-outcome-join` | [U] | +| M10 | Shadow availability | Qwen `lane_state` distribution by `served_by` | `make trace-served-by-audit` | [U] | +| M11 | Token lengths | p50/p95 input/output/reasoning tokens by lane | `make trace-tokens` | [U] | +| M12 | Silver noise | fraction of silver findings contradicted by gold outcome (on matured rows) | `make silver-audit` | [U] | + +Pre-registered hypotheses (they extend REX-001 H1–H6 and do not replace them): +- **PRA-H1:** Silver-trained Qwen increases κ_err(Qwen, counted lane) by more than the pre-registered margin [A] relative to base Qwen. +- **PRA-H2:** Gold-only fine-tuning does not increase κ_err beyond the margin [A]. +- **PRA-H3:** Disagreement-oversampled training improves sealed-test defect recall over uniform sampling at an equal token budget. +- **PRA-H4:** Grouped time-ordered splits lower val score relative to random row splits. If they do, the size of the drop is the leakage measure, and it is reported. + +--- + +## §4 Admission gates to the training pool + +Admission is a separate binary (`trace-admit`). It is never the producer. A row enters the pool only if **all** gates pass. Failure is recorded; there is no silent drop. + +| Gate | Rule | Target | Falsifier (planted RED → GREEN) | +|---|---|---|---| +| **G-SEC** secret scan | Union of two independent scanners over input blobs, output blobs and transcripts: (a) an in-process Rust entropy+pattern scanner (ripsecrets-class); (b) the gitleaks public ruleset ported **as data** (TOML rules → Rust `regex` set), with no Go/Python runtime. Any hit sends the row to `quarantined`, blobs to `quarantine/`. **No in-place redaction** [A]. | quarantined-row leakage into pool = 0 [A] | Plant the known AWS-key-literal diff plus 20 [A] synthetic canaries (AWS, GitHub PAT, private key, JWT, high-entropy generic) in a fixture PR. RED: the pool contains the canary. GREEN: all canaries quarantined, pool count 0. | +| **G-SEC-FN** scanner recall | Measured recall on the canary set plus a held-out labelled fixture | 100% on canaries [A]; real-world recall [U] | Remove one rule; the canary for it must go RED. | +| **G-CON** contamination | 0 sealed-test hashes (exact `diff_sha256`) and 0 sealed near-dup clusters in any pool (`review-corpus-contamination-v1` extended to MinHash clusters) | 0 [A] | Insert one sealed item with whitespace/rename perturbation into a candidate pool. RED on exact-hash-only; GREEN once cluster check lands. | +| **G-ID** identity completeness | `provider`, `model_id` (exact), `access_channel`, `terms_ref`, `input_sha`, `output_sha`, `served_by`, `label_tier`, `split_guard` all non-null | 100% [A] | Row with `model_id = "claude-sonnet"` (alias) must be refused. | +| **G-DUP** dedup and split | Row's cluster is not in a different split; exact duplicates of admitted rows are linked, not re-admitted | cross-split = 0 [A] | Backport PR with identical diff under new PR number assigned to val while original in train: RED → GREEN after union-find. | +| **G-MAT** outcome maturity | `label_tier = gold` requires `outcome_matured_at` ≠ null | 0 immature gold rows [A] | Gold row 3 days old must be refused. | +| **G-HASH** blob integrity | Re-hash of every referenced blob equals its address | mismatch = 0 [A] | Flip one byte in a test blob. | + +--- + +## §5 How the data feeds the improvement loop + +| Phase | When | What | Data used | +|---|---|---|---| +| **B1: prompt / few-shot** | now (apr 0.69+) | Mine disagreement and escape rows into retrieval few-shot exemplars for the Qwen prompt, then version the prompt (`prompt_version`). After 0.70, use `--json-schema` constrained decoding to drive `parse_status = failed` toward 0. | gold first; silver exemplars only when tagged in the prompt manifest | +| **B2a: sequence-level distill / SFT** | apr 0.71 | `apr finetune` on (input, output) pairs. Silver and gold are pooled per run manifest, and every manifest lists its tiers and providers. | admitted pool | +| **B2b: logit distill from 27B teacher** | apr 0.71 | The 27B teacher re-scores diffs, writing `sparse-logits-v1`. The student is trained with a forward-KL objective that uses the tail-mass correction, with reverse-KL / on-policy (GKD-style) as challengers. On-policy needs the student's own samples, which the Qwen shadow rows already are. | local rows only (teacher and student both local) | +| **Promotion** | per PRM-001 §5.4 | Champion/challenger on the sealed test. Rotation after 20 [A] promotion decisions. | sealed (evaluation only) | + +**Preserving dissent independence (`lane-independence-v1`).** +1. **Measure before training.** κ_err for every (Qwen, counted-lane) pair on matured gold val rows goes into the scoreboard as the [V] baseline. +2. **Two heads, two pools.** The *dissent* checkpoint that fills the shadow lane is trained on **gold only**. A *general* checkpoint may use silver. Only the dissent checkpoint may occupy the Qwen shadow lane. +3. **Gate.** A candidate for the shadow lane is refused if κ_err(candidate, any counted lane) > baseline + δ, with δ [A] pre-registered in the run manifest before training. Refusal is an andon, not a warning. +4. **Diversity option (research, not default).** Down-weight silver examples where all three counted lanes agree and the outcome is still pending. These are exactly the examples that teach "agree with the voters", with no evidence behind them. + +--- + +## §6 Hard rules (each falsified in §9) + +| # | Rule | Falsifier id | +|---|---|---| +| R1 | Every lane call produces exactly one `agent-trace-v1` row, NotRun included | F-R1 | +| R2 | `output_sha` covers the full raw reply; truncation anywhere is a defect | F-R2 | +| R3 | Findings are stored as text; a count-only findings field fails schema validation | F-R3 | +| R4 | The producer never sets `label_tier` other than `pending` | F-R4 | +| R5 | No row with a secret hit ever enters a pool; no in-place redaction | F-R5 | +| R6 | 0 sealed hashes or sealed clusters in any pool | F-R6 | +| R7 | Splits are by `repo#pr` group, time-ordered, cluster-closed | F-R7 | +| R8 | Every row has exact `model_id`, `provider`, `access_channel`, `terms_ref` | F-R8 | +| R9 | Blobs are write-once and re-hash equal to their address | F-R9 | +| R10 | Nightly second copy exists and the weekly restore drill mismatch is 0 | F-R10 | +| R11 | Transcript retention is declared fleet-wide by forjar; no host runs the default 30 | F-R11 | +| R12 | Qwen shadow lane weight stays 0 regardless of trace content | F-R12 | +| R13 | A shadow-lane candidate failing the κ_err gate is not deployed | F-R13 | +| R14 | No Python in any PRA-001 artifact; shell is bashrs-clean | F-R14 | +| R15 | `apr` is a released, sha-verified tag; weights are pinned by sha256; `served_by` reflects the actual backend | F-R15 | + +--- + +## §7 EV-ordered tickets + +Each ticket: `pmat work add`, feature branch, PR, protected main, `ci / gate`, TDD, one `pv` contract with its falsifier planted RED then GREEN. **K̂** = estimated minutes [A]; **K** = measured minutes (recorded at close); **andon** if K > 2·K̂ [A] or any falsifier fails to go RED first. + +| EV rank | Ticket | Contract | Done when | K̂ [A] | +|---|---|---|---|---| +| 1 | T1 Capture at dispatch in quorum rail (paiml-implement #436) | `agent-trace-v1` | F-R1, F-R2, F-R3 GREEN; 7 days of 100% completeness (M2) | 240 | +| 2 | T2 Ledger v2: findings text + trace links; v1 readers keep working | `review-ledger-v2` | v1 fixtures parse; v2 row carries findings text; migration test | 120 | +| 3 | T3 Content-addressed blob store on almacen | `trace-blob-store-v1` | F-R9 GREEN; O_EXCL write-once test; forjar-declared mount | 120 | +| 4 | T4 Transcript retention + harvester | `transcript-retention-v1` | F-R11 GREEN on every host in inventory; subagents/ harvested | 90 | +| 5 | T5 Nightly second copy + weekly restore drill | `trace-backup-v1` | F-R10 GREEN; first drill mismatch = 0 | 120 | +| 6 | T6 Secret admission gate (Rust scanner + ported ruleset) + backfill | `trace-admission-secret-v1` | F-R5 GREEN; AWS-key literal quarantined in backfill | 180 | +| 7 | T0 Baseline freeze: run §1 verify commands, convert [A]/[U]→[V] | (report only) | §1 table all [V] or STOP raised | 60 | +| 8 | T7 Contamination gate extended to clusters | `review-corpus-contamination-v1` (extend) | F-R6 GREEN incl. perturbed sealed item | 90 | +| 9 | T8 MinHash-LSH dedup index (pure Rust) | `trace-dedup-v1` | M5 report produced; cluster ids in index | 150 | +| 10 | T9 Group/time split assigner with union-find closure | `trace-split-guard-v1` | F-R7 GREEN; cross-split clusters = 0 | 120 | +| 11 | T10 Outcome join (merged/revert ≤14d/regression escape) | `trace-outcome-join-v1` | G-MAT falsifier GREEN; M9 produced | 150 | +| 12 | T11 Qwen top-k logits capture in `apr serve` rows | `sparse-logits-v1` | header + residual mass round-trip test; M6 logits bytes measured | 180 | +| 13 | T12 Terms/provider tagging + exclusion filter | `agent-trace-v1` (terms fields) | P-TERMS GREEN (pool rebuild excluding a provider) | 60 | +| 14 | T13 Lane κ_err probe + scoreboard | `lane-independence-v1` | M4 baseline [V] recorded for every pair | 120 | +| 15 | T14 Datacard: Croissant + RAI + datasheet generator | `trace-datacard-v1` | generated file passes Croissant validation; regenerated per snapshot | 90 | +| 16 | T15 B1 few-shot miner from disagreement/escape rows | (uses `agent-trace-v1`) | prompt_version bump with manifest of exemplar tiers | 120 | +| 17 | T16 27B teacher re-score job (gx10) | `sparse-logits-v1` (mode topk) | teacher blobs for all admitted local rows of week N | 180 | + +EV rationale: T1–T5 stop irreversible loss, which is ongoing and compounds daily. T6–T10 make the data *admissible*. T11–T17 make it *useful*. Nothing training-related (B2) is ticketed until aprender 0.71 ships; that is a §8 STOP. + +--- + +## §8 STOP conditions (exhaustive) + +The runner halts, writes the §9 report with `status: STOPPED`, and waits for the operator when: + +1. **S-BASE:** any §1 verify command disagrees with its [A] value (baseline drift). +2. **S-CRED:** a step needs credentials, API keys, or account settings the runner does not hold. +3. **S-MEDIA:** the second-copy media or an offsite target does not exist in forjar inventory. +4. **S-CAP:** almacen free space < 20% [A], or the M6 projection exceeds 50% of usable capacity within 5 years [A]. +5. **S-SECRET-PURGE:** a secret hit requires purging from *all* copies (rotation is the operator's job; the runner only quarantines). +6. **S-SEALED:** any change to `review-corpus-v1`, sealed rotation outside PRM-001, or any sealed hash found in a pool. +7. **S-TERMS:** any change in provider terms detected by `terms_ref` refresh, any publication including non-local rows, or any request to change S-6. +8. **S-APR:** a required verb is missing from the released `apr` tag (for example B2 before 0.71), or a tag's sha fails verification. HEAD is never a fallback. +9. **S-PY:** a dependency would introduce Python, or a non-forjar install would be needed. +10. **S-KAPPA:** the κ_err gate refuses a shadow-lane candidate (the operator decides whether δ was right; the runner never widens δ). +11. **S-WEIGHT:** any change would give the Qwen lane non-zero weight, tie-break or escalation power. +12. **S-ANDON:** K > 2·K̂ [A] on any ticket, or a falsifier that cannot be made RED first. +13. **S-PII:** a transcript contains third-party personal data beyond author identities (scrubbing policy is an operator decision). +14. **S-LOSS:** any blob mismatch in the restore drill, or `capture_failed` rows > 0 for 24 h [A]. + +--- + +## §9 Final report schema and scoreboard + +```yaml +report: PRA-001 +version: 1 +generated_at: +status: COMPLETE | STOPPED | IN_PROGRESS +stop: {code: |null, evidence: } +apr_tag: # sha-verified +baseline: # §1 after T0 + - {id: G1, value: , mark: V, command: "make trace-yield WINDOW=7d"} +tickets: + - {id: T1, contract: agent-trace-v1, pr: , khat_min: 240, k_min: , andon: false, + falsifier: {id: F-R1, red_commit: , green_commit: }} +measurements: # §3 + - {id: M4, pair: [qwen-shadow, sonnet], kappa_err: , n: , split: val, mark: V} +gates: # §4 + - {id: G-SEC, pool_leaks: 0, canaries_quarantined: /} +falsifiers: # §6 + - {id: F-R5, state: GREEN, evidence: } +storage: + bytes_raw_per_week: + bytes_zstd_per_week: + projected_tb_per_year: # replaces G10 [U] + restore_drill_mismatch: 0 +``` + +**Scoreboard (targets and probes only)** + +| Probe | Target | Mark | +|---|---|---| +| Capture completeness (M2) | 100% | [A] | +| Rows lost to `capture_failed` | 0 | [A] | +| Secret-hit rows in any pool | 0 | [A] | +| Sealed hashes/clusters in any pool | 0 | [A] | +| Cross-split near-dup clusters | 0 | [A] | +| Rows missing identity fields | 0 | [A] | +| Immature gold rows | 0 | [A] | +| Restore-drill mismatches | 0 | [A] | +| Hosts with default transcript retention | 0 | [A] | +| Qwen lane weight | 0 | [A] | +| κ_err(shadow candidate, counted lane) | ≤ baseline + δ | [V] baseline, [A] δ | +| Bytes/year | ≤ measured M6 projection; ratchet | [U]→[V] | +| Yield/week by tier | ratchet up from week-1 baseline | [U]→[V] | +| P-TERMS: pool rebuild excluding one provider | succeeds, 0 rows of that provider | [A] | + +Falsifier catalogue (F-R1…F-R15): each is a `pv` contract test that is committed RED (failing on a planted defect) before the implementing commit turns it GREEN. Examples: F-R1 kills a lane mid-call and asserts a NotRun row. F-R2 sends a 1 MB reply and asserts the byte-exact `output_sha`. F-R12 injects a Qwen verdict that disagrees with all counted lanes and asserts the quorum outcome is unchanged. F-R14 runs a repo-wide scan for `*.py`, `python`, `pip` in PRA-001 paths and expects 0 hits. + +--- + +## Appendix A: Research findings and how each changed the design + +**A.1 Distillation (sequence vs logit, sparse top-k).** +- Anshumann et al., *Sparse Logit Sampling* (ACL 2025 oral, arXiv 2503.16870), prove that caching top-K probabilities gives "biased estimates of teacher probability distribution to the student, resulting in suboptimal performance and calibration". Their importance-sampling RS-KD gives unbiased estimates while using only 0.01% of precomputed teacher logits. A secondary summary reports parity with full KD using about 12 tokens/step and ≈3.6 TB per 100B tokens versus ≈90 TB for top-300 [X, secondary]. +- A 2026 preprint (arXiv 2608.03796) frames sparse top-K KD as forward KL against a teacher with retained mass M ≤ 1. +- MiniLLM (Gu et al., ICLR 2024) argues reverse KL suits generative students. GKD (Agarwal et al., ICLR 2024) trains on student-generated sequences with teacher feedback. +- **Design changes:** store `topk_mass` + `logsumexp_full`, reserve `mode: rs-sample`, and keep Qwen's own outputs as on-policy samples to be re-scored by the local 27B teacher. Frontier lanes are sequence-level only, because no logits are available. + +**A.2 Code-review datasets and label quality.** +- CodeReviewer (Li et al., ESEC/FSE 2022) built a multilingual corpus (nine languages) of real diffs and review comments with three tasks: quality estimation, comment generation, refinement. Its schema is diff-in / label-or-comment-out, with human reviewer comments as labels. +- *Too Noisy To Learn* (Liu, Lin & Thongtanunam, arXiv 2502.02757) shows substantial vague or non-actionable noise remains after heuristic and SVM cleaning. Only 64% of the original CodeReviewer comments were valid, and LLM cleaning raised that "to up to 85%", with 66–85% precision in detecting valid comments. +- **Design change:** findings text alone is not a label. Gold comes from outcomes and the sealed corpus, silver is tagged, and M12 measures silver noise against matured outcomes. + +**A.3 Outcome signals as labels.** +- SZZ-derived bug-inducing labels are notoriously noisy. Herbold et al. (arXiv 1911.08938) report that "about one quarter of the links to defects detected by SZZ is wrong", and that for every correctly labelled bug issue there are 0.74 mislabelled ones. Afric et al. report 14.3% mislabelled even after refinement (via ReDef, arXiv 2509.09192). +- **Design change:** gold outcomes are restricted to direct, observable events (merge, revert ≤14 days [A], explicit `regression` label). No blame-tracing heuristics in v1. Outcomes mature before use (G-MAT), and pending is never negative. + +**A.4 Near-duplicates and leakage.** +- Allamanis (Onward! 2019) showed code duplication inflates reported metrics by up to 100%. +- Lee et al. (ACL 2022) found near-duplicates via MinHash (e.g. 3.04% of C4) and train-test overlap affecting over 4% of validation sets. +- **Design change:** group-by-PR and time-ordered splits, MinHash-LSH cluster closure across PRs, contamination extended from exact hash to cluster (G-CON), and PRA-H4 to quantify leakage. + +**A.5 Ensemble diversity and error correlation.** +- Kim et al. (*Correlated Errors in LLMs*, ICML 2025) evaluated 350+ LLMs. On HELM, model pairs agree about 60% of the time when both are wrong, against 1/3 for uniformly random wrong answers. Shared provider and architecture raise correlation, and larger, more accurate models are highly correlated even across providers. +- Goel et al. (*Great Models Think Alike*, ICML 2025) introduce CAPA and show judges favour similar models. They find weak-to-strong gains are larger when supervisor and student are functionally different. +- **Design change:** κ_err is measured before and after training (M4, CAPA as secondary). The dissent checkpoint is gold-only. A pre-registered δ gates the shadow lane (§5). + +**A.6 Secret detection.** +- Basak et al. (arXiv 2307.00714) measured tools on SecretBench (15,084 true secrets among 97,479 reported). Gitleaks had the best recall (88%) at 46% precision and the top F1 (60%), and TruffleHog reached 52% recall. GitHub's scanner had the best precision (75%) but only 6% recall. "No current tool has the coveted high precision and high recall." +- On a different benchmark (AssetHarvester, arXiv 2403.19072), Gitleaks recall fell to 0.02 and TruffleHog reached 0.40. Recall is dataset-dependent. +- ripsecrets is a Rust, local-only scanner that claims to be ≥95× faster [X, vendor claim]. +- **Design change:** union of two independent rule sources, implemented in Rust (the gitleaks ruleset ported as data). Quarantine, not redaction. Recall is measured locally with planted canaries rather than trusting published numbers. + +**A.7 Provider terms (factual summary; not legal advice; S-6 governs).** +- **Anthropic Commercial Terms of Service** (effective June 17, 2025), §D.4: "Customer may not and must not attempt to (a) access the Services to build a competing product or service, including to train competing AI models or resell the Services except as expressly approved by Anthropic…". §B also states that the customer owns its Outputs and that Anthropic may not train on Customer Content. +- **Gemini API Additional Terms** (effective March 23, 2026): "You may not use the Services to develop models that compete with the Services (e.g., Gemini API or Google AI Studio)." +- **Google Antigravity Additional Terms** (undated page) incorporate the Google Terms of Service (effective July 30, 2026). Those forbid "using AI-generated content from our services to develop machine learning models or related AI technology." Antigravity §8 binds users who select an Anthropic model to Anthropic's commercial terms. +- Google Cloud service terms exempt Gemini Enterprise Agent Platform (formerly Vertex AI) when no Google pre-trained model is used. +- Consumer terms for subscription-based Claude Code were not verified here. +- **Design change:** `access_channel` and `terms_ref` are mandatory. Pools can be rebuilt excluding any provider or channel (P-TERMS). Public releases default to local/gold rows. Terms changes are S-TERMS. + +**A.8 Claude Code transcripts.** +- Official docs: transcripts under `~/.claude/projects/` are plaintext, include everything that passed through tools, and are deleted after `cleanupPeriodDays`. The default is 30 and the minimum is 1; 0 fails validation. Managed settings govern. +- A February 2026 issue (#23710) reported that 0 had silently disabled persistence in an earlier version. Another issue (#58154) reports that `subagents/` transcripts are not swept. +- **Design change:** forjar-managed settings with a large explicit value, harvesting that includes `subagents/`, a survival canary, and a secret scan on every transcript. + +**A.9 Dataset documentation.** +- Croissant (Akhtar et al., NeurIPS 2024 D&B) defines four layers (metadata, resources, structure, semantics). Croissant-RAI builds on Datasheets for Datasets and Data Cards. +- NeurIPS 2026's Evaluations & Datasets track now requires RAI metadata in the Croissant file. Hugging Face auto-generates core Croissant. +- **Design change:** `trace-datacard-v1` generates Croissant + RAI + a datasheet from the index on each snapshot. + +**A.10 Storage sizing.** +- A consumer drive URE spec of 1 per 10¹⁴ bits (≈12.5 TB read) is a worst-case warranty floor. Field studies show lower rates [X]. +- zstd at level 3 is reported at 6–7× on JSONL [X, blog], and dictionaries help small similar files [X]. +- **Design change:** compression level is recorded per blob and not implied, M8 measures our own ratio, the second copy plus restore drill is mandatory, and G10 stays [U] until week 1. + +## Caveats +- Some literature figures come from secondary summaries (the RS-KD storage numbers, and the Afric et al. 14.3% figure, which comes via ReDef) and are marked [X, secondary] where applicable. None is used as a gate threshold. +- Secret-scanner recall differs by an order of magnitude between benchmarks. Local canary recall is necessary, not sufficient, and a zero-hit scan is not proof of absence. +- Provider terms change: Anthropic updates take effect 30 days after posting. `terms_ref` must be refreshed on a schedule, and nothing here is legal advice. +- κ_err needs matured gold rows. Until M9 produces enough of them, the independence gate cannot be evaluated, and B2 shadow-lane deployment must wait (S-KAPPA). \ No newline at end of file diff --git a/docs/specifications/review-experiment-protocol.md b/docs/specifications/review-experiment-protocol.md new file mode 100644 index 0000000000..e296b83315 --- /dev/null +++ b/docs/specifications/review-experiment-protocol.md @@ -0,0 +1,362 @@ +# PRM-001 (was REX-001): PROMETHEUS — review-lane experiment protocol (Qwen 3.5 4B on apr, across fleet devices) + +**Spec id:** `PRM-001` (was `REX-001`; row ids unchanged) · **Rows:** `REX-00..REX-12` (the traffic cop mints one `pmat` ticket per row; single-minter rule applies) +**Target repo:** `paiml/aprender` (`docs/specifications/review-experiment-protocol.md`) +**Runner:** the aprender traffic cop (`aprender-traffic-cop-prompt.md`). It mints the tickets, directs the workers through issues, and owns the pre-registration commit and the final report. +**Launch:** from `~/src/aprender`, run `Implement docs/specifications/review-experiment-protocol.md autonomously.` +**Related:** ARB-APR-001 (arbiter on apr), ARB-SELF-001 (dogfood feedback loop), `stack-30-day-plan.md` §4.4 (trustworthy review), #3558 (Pareto model-size selection rule), APR-PERF-GATE-001 (perf arms, receipt transport J3). +**Status:** spec, not implemented. Dated 2026-09-25. + +**Provenance marks:** `[V]` verified at the cited sha/time · `[C]` computed · `[A]` asserted · `[U]` unverified/unmeasured · `[X]` third-party. + +--- + +> **SUPERSEDED** by [`PRM-001-prometheus.md`](PRM-001-prometheus.md) (PRM-001 v3, 2026-09-25). This file is kept +> byte-frozen in §2–§5 so its `rex-prereg-v1` lock (`docs/audits/rex-001/prereg.lock`, +> prereg_sha `ef51087d…`) still verifies. Do not edit it; amend PRM-001 instead. + +## §0 Operating assumptions + +1. **This is a scientific experiment, not a benchmark run.** + - Hypotheses, metrics, analysis plan and decision rules are frozen in REX-00 before any measurement. + - Data gathered under an earlier version of §3–§6 is labelled `exploratory` and never feeds a decision. +2. **The question has two parts:** + - (A) which device cell should host the Qwen 3.5 4B review lane now, and how its speed improves each release; + - (B) how the model improves continuously: prompt, decoding, fine-tune, distill, merge. + - The experiment answers (A) and builds the measuring instrument for (B). +3. **Dogfood starts on day 0, alongside the experiment.** + - The 4B lane joins the quorum as a **4th, non-voting shadow lane** on a provisional cell (gx10, per the 2026-09-25 ruling). + - The experiment's hardware ruling (REX-08) supersedes the provisional cell. +4. **Producer is never the gate.** + - The review lane runs a **released, sha-verified `apr` tag**, never aprender HEAD. + - The model weights are pinned by sha256. + - The lane's vote stays asymmetric until it is promoted: an apr dissent sends a 2–0 to human review (HRQ), and apr never breaks a 1–1. +5. **A refusal is a result, not an error.** + - A cell whose backend refuses the model (for example WGPU or Metal before 0.71) is recorded as `Refused{removed_by}` in the report and filed as an aprender issue. + - It is never retried under another backend while keeping the same cell name. +6. **Genchi genbutsu.** Every §1 value is re-measured at HEAD in Phase 0 before any row relies on it. A row whose premise is false at HEAD is recorded `premise-falsified` and skipped. +7. **Cross-repo work is filed, not done.** + - Executor declarations (infra/forjar) and quorum wiring (paiml-implement) are filed as issues with exact acceptance criteria. + - Cells that depend on them are `NotRun{NoDeclaredExecutor}` until those issues land. + - Rows that don't depend on them proceed. +8. **The operator has no pre-steps.** Anything only Noah can do is a §8 STOP condition. + +--- + +## §1 Ground truth (baseline, frozen; never quote as current) + +| # | Fact | Value | Mark | Source of truth | +|---|---|---|---|---| +| G1 | n=1 pilot, same diff and prompt | Qwen3.5-4B Q4_K_M on `apr` v0.69.3 release binary (checksum `8a67a010`), lambda CPU at load 33–40: **175.8 s**, PASS. Haiku 4.5 via `claude -p`: **17.8 s**, PASS. GPU rerun skipped (GPU lock held > 60 s) | [V] (operator-pasted report, 2026-09-25) | the pasted report; diff `0c6932fd5` | +| G2 | What G1 can establish | A PASS on a correct diff says nothing about defect recall. Precision and recall of the 4B lane, and of every existing voter, are **unknown** | [C] | — | +| G3 | Audited `apr` surface relevant to this spec | `apr run --backend wgpu` q=2, hw UNKNOWN · `apr serve run` q=4 · `apr finetune` q=2 · `apr distill` q=9 · `apr merge` q=6 · `apr quantize` q=4 · `apr parity` q=9 · `apr train apply` q=1 · `mcp:apr.serve` q=1 | [C] at audit snapshot; [U] at HEAD | `docs/audits/surface_audit.csv` | +| G4 | Train themes | 0.69 = verbs that are true; 0.70 = fast (TTFT, resident serve, `--json-schema`); finetune/distill on qwen35 refuse until 0.71; WGPU/Metal refuse until 0.71 | [A] operator ruling 2026-09-20; [U] at HEAD | `docs/specifications/06x-release-schedule.md`, milestones | +| G5 | Parity oracle | llama.cpp pin `d1d3c3396` (`scripts/llama_pin.toml` `build_commit`); `ds.yaml` min_cosine 0.98 | [V] 2026-09-20; [U] at HEAD | `scripts/llama_pin.toml`, `ds.yaml` | +| G6 | TTFT gap | apr 0.68.2 TTFT 7.7×, end-to-end 4.9× slower than llama.cpp on Qwen3.5-4B, 4090 | [V] 2026-09-20 | #3596 | +| G7 | Host roles | lambda-labs = agent host, primary x86 CUDA (scarce); **its GPU is excluded from this experiment by construction** (shadow rung 2 only, R-9). intel = clean-room runner (contended), dual AMD GPU (Vulkan). gx10 = GB10, 120 GB unified, clean-room pool. mini = M4, macOS, `rust-neutral` pool | [A] | `infra/machines/*/forjar.yaml` | +| G8 | Undeclared apr | lambda-labs `~/.local/bin/apr` was a hand-installed 0.64.0 | [V] 2026-09-20; [U] at HEAD | `command -v apr`; `fleet-bins.tsv` | +| G9 | Reviewer identity | 301/301 historical quorum receipts record the reviewer model as `unknown` | [V] 2026-09-10 | PV-LEAN-AUDIT | +| G10 | Decode ceilings (bandwidth ÷ bytes/token) | 4B Q4_K_M ≈ 2.5 GB/token → 4090 ≈ 400, GB10 ≈ 110 tok/s | [C] theoretical, [U] for apr | — | + +--- + +## §2 Design + +### §2.1 Factors + +**Device cell** (the primary factor). The same model, apr tag, prompt and decoding settings run in every cell. + +| Cell | Host | Backend | Notes | +|---|---|---|---| +| C1 `intel-wgpu` | intel | wgpu (Vulkan, dual AMD) | likely `Refused` before 0.71 (G4). The refusal is recorded and filed | +| C2 `intel-cpu` | intel | cpu (AVX-512) | clean-room contention: shares the intel concurrency group (APR-PERF-GATE-001 §4.9.2) | +| C3 `lambda-cpu` | lambda-labs | cpu | GPU excluded. Co-located with arbiter and agent sessions | +| C4 `gx10-cuda` | gx10 | cuda (sm_121) | provisional shadow host. Must yield when `train-active` is set | +| C5a `mini-cpu` | mini | cpu (NEON) | | +| C5b `mini-metal` | mini | metal | likely `Refused` before 0.71 (G4) | + +**Reference cell:** C2 `intel-cpu`. It is the cross-cell invariance reference, and its logits are also checked against llama.cpp `d1d3c3396` CPU `[X]`. + +**Model arm:** +- Primary: `Qwen3.5-4B-Q4_K_M`, sha pinned. +- Control: `Qwen3.5-9B-Q4_K_M`, **test split, run only on the reference cell and the top-ranked cell.** This control makes the "4B is enough" claim falsifiable (H3). + +**External baselines**, reference cell only, once per test item: +- Haiku 4.5 (`claude -p`); +- the agy lane. + +Both record their exact model ids. They measure the existing voters on the same corpus. **This is the first recall and precision measurement of the current quorum.** + +**Fixed across all runs:** +- a versioned prompt file, sha recorded; initially the G1 prompt ("VERDICT: PASS or FAIL, then at most 5 short bullet findings"); +- greedy decoding (temperature 0, fixed seed); +- a fixed `max_tokens`; +- the context policy: diffs are never truncated silently. A diff that exceeds the context is `NotRun{ContextOverflow}` and counted as such. + +### §2.2 Corpus (`docs/audits/review-corpus/`) + +| Class | Construction | Label source | Target n `[A]` | +|---|---|---|---| +| **P** planted | the fleet's existing mutation RED fixtures, applied as diffs | known defect plus its file:line | 50 | +| **R** real | **revert-the-fix**: take a merged fix PR that has a linked defect issue and reverse-apply its fix hunks, which reintroduces a real, historical defect | the fix hunk's file:line | 50 | +| **G** good | merged PRs with a green clean-room, no revert, and no `regression` label for 14 days `[A]` | clean | 50 | + +- **Strata:** diff size S (< 2k tokens), M (2–8k), L (> 8k), balanced across classes where the population allows. The per-stratum n is reported. +- **Splits:** + - `dev`: 30%, stratified. Prompt iteration is allowed on it. + - `test`: 70%, **sealed**. Its item hashes are committed in REX-00. Its contents are read only by the scoring harness, never by a prompt author, a training job or a model-selection loop. +- **Sample-size basis `[C]`:** 70 defect items in test give a Wilson 95% CI half-width of ≈ 0.117 on recall at p = 0.5. If the pilot (REX-05) projects a half-width > 0.12, the pre-registered response is to grow the corpus. The hypotheses are not changed. +- **Contamination contract:** + - The test-item hashes must not appear in any training set (§6). + - A diff hash overlap is a hard FAIL (`review-corpus-contamination-v1`). +- **Test-set rotation:** a sealed test version supports **at most 20 promotion decisions `[A]`**. Then 50% of its items are rotated out and replaced from new R/G specimens. The rotation is logged, and the old version is kept for trend comparability. + +### §2.3 Metrics (all deterministic; no model judges) + +| Metric | Definition | +|---|---| +| Verdict | parsed from the first `VERDICT: (PASS\|FAIL)` line. Unparsable → `Unparsed`, which counts as incorrect and is reported separately | +| Parse rate | parsed ÷ attempted | +| Recall | FAIL verdicts on P∪R ÷ \|P∪R\| | +| Precision | FAIL verdicts on P∪R ÷ all FAIL verdicts | +| False-refute rate | FAIL on G ÷ \|G\| | +| Localization | on P∪R FAILs: a finding names the defect's file path (string match). This is the "useful dissent" rate | +| Cross-cell invariance | the fraction of items whose verdict and output bytes equal the reference cell's | +| Determinism | a 10% random re-run on the same cell gives byte-identical output | +| Load, TTFT, prefill tok/s, decode tok/s | taken from `apr` timing output (`--json` timing, or the serve response metadata) | +| Review wall-clock | request sent → verdict parsed, per item | +| Peak memory | cgroup anon, not `memory.peak` (09-18 measurement method) | +| Host load | loadavg1, busy runners (via the `fleet-bin.sh` oracle), co-running jobs; recorded per run | +| Interference cost | the slowdown of a fixed reference workload co-located with the serve, compared with running alone (§3 H6) | +| Energy | RAPL (intel, lambda) / `nvidia-smi` (gx10) / `powermetrics` (mini) where available, otherwise `NotRun{NoSensor}`. Reported only, never used in a decision | + +### §2.4 Execution discipline + +- **Run order:** randomized per cell with a fixed seed. Cells run concurrently on their own hosts. +- **Cold vs warm:** + - The first request after the unit starts is recorded as `cold` and reported separately. + - All other items are measured warm against a resident `apr serve`. +- **Receipts:** + - One JSONL row per (item, cell, arm), conforming to `review-experiment-receipt-v1`. + - Required fields: item id+sha, class, stratum, split, cell, host, backend, apr tag+binary sha, model id+weights sha, prompt sha, decoding params, token counts, all timings, verdict, raw-output path+sha, load snapshot, corpus version, prereg sha. + - A missing field makes the receipt inadmissible, and an inadmissible receipt is never a pass. +- **Transport:** hosts push signed receipts (APR-PERF-GATE-001 §4.9.1, J3). The analysis job runs anywhere and verifies the signature and freshness. + +--- + +## §3 Pre-registered hypotheses and tests + +The family for Holm correction is H1–H6, overall α = 0.05. All CIs are 95%. + +| # | Hypothesis | Test | Decision consequence | +|---|---|---|---| +| H1 | **Invariance:** on every parity-admissible cell, 4B verdicts and output bytes equal the reference cell's on every test item | a count of divergent items; any count > 0 rejects H1 | each divergence becomes an aprender parity issue with the item attached. **A divergent cell is not admissible for (A)** | +| H2 | **Determinism:** same cell, same item → byte-identical output | a count on the 10% re-run; > 0 rejects | a non-deterministic cell is inadmissible until fixed | +| H3 | **Size:** 9B per-item correctness exceeds 4B's on the test split | McNemar exact, paired, one-sided | if not rejected, 4B stays the lane model. If rejected, the §5 promotion rule decides whether 9B replaces it, weighing its measured wall-clock | +| H4 | **Usefulness vs voters:** 4B precision ≥ the lowest precision among the existing voting lanes (Haiku, agy) on the same items | a paired bootstrap CI of the difference (10k resamples, seeded); H4 holds if the CI lower bound ≥ 0 | this is the gate from shadow to **tripwire** (dissent may trigger HRQ) | +| H5 | **Recall vs voters:** 4B recall ≥ the Claude lane's recall | same test | this is the gate from tripwire to **full vote** | +| H6 | **Interference:** the serve on cell X slows the host's reference workload by more than run-to-run noise | the reference workload runs 5× alone and 5× co-located; noise = 2 × sd(alone) `[C at measurement]` | a cell whose serve slows a gated CI job or the arbiter host beyond noise is ranked below every cell that doesn't | + +Descriptive (not tested): the per-cell p50/p95 wall-clock with bootstrap CIs, TTFT, and tok/s, each compared against llama.cpp `d1d3c3396` running the same model on the same cell `[X]`. + +**Sensitivity analysis:** the primary analysis uses all runs, because that is the deployment reality. A secondary analysis restricted to loadavg1 < 0.5 × physical cores is reported beside it. G1's "unfair CPU load" objection is answered by that second column, not by excluding data. + +--- + +## §4 Pre-registered decision rule: hardware for the review lane (Goal A) + +This rule is lexicographic and fixed in REX-00. + +1. **Admissible:** + - a parity receipt passes (`apr parity` vs llama.cpp `[X]`, `ds.yaml` min_cosine 0.98); + - H1 and H2 not rejected for that cell; + - parse rate equal to the reference cell's; + - the cell's executor is forjar-declared. +2. **Among admissible cells, rank by p95 review wall-clock** under natural load (primary analysis). +3. **Interference override:** a cell whose H6 is rejected ranks below every cell where it isn't. +4. **Ties** (overlapping p95 bootstrap CIs) go to the host with the larger measured 7-day idle fraction. If that also ties: not in the clean-room pool > in it. +5. **Queue budget:** the chosen cell's p95 must be ≤ the slowest existing voting lane's measured p95 on the same items, so the local lane never lengthens the quorum. + - If no cell meets it, the fastest admissible cell is selected **as shadow only**. + - Every cell's gap vs llama.cpp is filed as an aprender perf issue. +6. **Output:** a **primary cell and a failover cell.** + - If both are down, the lane refuses by name (`Unknown{LaneUnavailable}`). A quorum never silently runs below its width. + - When `train-active` is set on the primary's host, the lane emits `NotRun{TrainActive}` and blocks rather than passing. + +The ruling is committed as `docs/audits/rex-001-hardware-ruling.md` with the receipts attached. It supersedes the 2026-09-25 provisional gx10 placement only if it differs, and then through the infra issue that moves the declared unit. + +--- + +## §5 Continuous improvement loop (Goal A and Goal B) + +### §5.1 Performance ratchet (A, every aprender tag) + +- On each tag, rerun the perf arm on the primary cell (the full `dev` split plus 30 fixed test items for timing only; the verdicts on them are already known): + - with the new `apr` tag; + - with llama.cpp `[X]` on the same cell. +- **Ratchet:** review p95 on the primary cell is never allowed to rise after 3 recorded tags. + - A regression is an andon. It opens an aprender issue carrying both receipts. +- **Runtime knob sweep** on the primary cell only, once per train (0.70, 0.71, …): + - thread count ∈ {physical cores, physical/2}; + - quant ∈ {Q4_K_M, Q8_0}; + - context/KV setting as exposed. + - The winner is chosen by the §4 rule restricted to that cell. A quant change also requires the §5.3 quality test, because a new quant is a new model. + +### §5.2 The data flywheel (B0; starts on day 0) + +- **Every quorum, from every lane, writes a `review-ledger-v1` row:** + - diff hash, repo, PR, lane verdicts and findings; + - model ids, apr tag, weights sha; + - final outcome: merged / reverted ≤ 14 d `[A]` / `regression`-labelled escape. +- **Label tiers:** + - **Gold:** corpus labels, plus reverts and escapes joined back to the PR that introduced them. Gold is the only tier allowed in evaluation. + - **Silver:** Claude + agy consensus verdicts and findings. Used for training only, and only if §8 S-6 is resolved. Never used for evaluation. +- The ledger joins ARB-SELF-001's outcome ledger (ASL-09) and escape join (ASL-12). The escape join files regression escapes as new R-class specimens, **so the corpus grows from real misses.** + +### §5.3 Candidate improvements, in EV order + +| Tier | Method | Available | Gate | +|---|---|---|---| +| B1a | Prompt versions (structure, rubric, output schema) | now | dev split, then sealed test (§5.4) | +| B1b | Retrieval few-shot: the k most similar gold specimens from the train pool (never the test split) | now | same | +| B1c | Constrained decoding once `apr` ships `--json-schema` (0.70) | 0.70 | parse rate → 1.0 is the expected effect; confirmed by the same test | +| B2a | **Logit distillation from a local teacher:** Qwen3.5-27B on gx10 writes top-k logits at m = 1 over the train pool; the 4B student trains with `apr distill` | when `apr distill`/`finetune` accept qwen35 (0.71 per G4) | §5.4 | +| B2b | QLoRA fine-tune on gold-labelled train-pool items (`apr finetune`) | same | §5.4 | +| B2c | Merge: combine successive adapters or checkpoints (`apr merge` / `aprender-train-lora merge`) | same | §5.4 | +| B2d | Re-quantization of a promoted checkpoint (`apr quantize`) | now for existing weights | §5.4 plus §5.1 | + +- **B2 rows are aprender acceptance tests.** An `apr finetune`/`distill`/`merge` ticket on qwen35 counts as done only when it produces a challenger that passes §5.4 on the sealed test. +- That makes the review lane a standing consumer of the training verbs, which is the dogfood the 30-day goal I asks for. +- Until those verbs accept qwen35, B2 rows are `NotRun{VerbRefused(removed_by)}`, never skipped silently. + +### §5.4 Champion/challenger promotion (pre-registered) + +A challenger (any §5.3 candidate) replaces the champion only if all of these hold on the current sealed test version: + +1. **Per-item correctness improves:** McNemar exact, paired, one-sided, α = 0.05. +2. **Precision point estimate is not lower** than the champion's, because precision is what spends Noah's HRQ time. +3. **Invariance (H1) and determinism (H2) hold** on the primary cell. +4. **Contamination contract green:** 0 test hashes in its training data. +5. **Wall-clock:** its p95 on the primary cell still meets the §4 queue budget. + +- Promotion is a forjar pin bump of the weights sha (and the adapter sha, if any). Rollback is the previous sha. +- Each decision consumes one of the test version's 20 evaluations (§2.2). +- **Demotion:** 2 consecutive gold-labelled escapes in which the champion said PASS and a voter said FAIL trigger an automatic HRQ and re-evaluation. + +--- + +## §6 Hard rules (each has a §9 falsifier) + +- **R-1 Pre-registration is immutable.** + - REX-00 commits §2–§5 and the analysis code; its sha goes into every receipt. + - Any change creates a new spec version, and data collected under the old version is `exploratory`. +- **R-2 Sealed test.** + - The test split is read only by the scorer. + - A diff that reads test items from anywhere else (prompt building, retrieval, training, selection) fails CI (`review-corpus-contamination-v1`). +- **R-3 Released artifacts only.** + - The lane and every cell run a released `apr` tag verified by sha256, and weights verified by sha256. + - Never HEAD, never a hand-built binary. +- **R-4 Declared executors only.** + - Every cell runs through a forjar-declared unit or a declared runner label. No ad-hoc SSH, and no hand-installed `apr` on any host. + - An undeclared `apr` on an executing host is a STOP (S-2). +- **R-5 Receipts carry identity.** + - 0 receipts may record `unknown` for model, weights sha, apr tag, host, backend or prompt sha. +- **R-6 NotRun is never a pass.** + - `Refused`, `NotRun{…}`, `Unparsed` and `ContextOverflow` are each counted and reported. + - None of them counts toward accuracy numerators. +- **R-7 No invented thresholds.** + - Every numeric gate cites its measurement command or is marked `[A]`/`[U]`. + - The `[A]` values in this spec (corpus sizes, 14-day revert window, 20-evaluation budget) are recalibrated after the first test version and logged. +- **R-8 No Python** in any file this spec creates, harness or analysis. Shell passes `bashrs`; everything else is Rust (an aprender workspace crate or xtask). +- **R-9 Lambda GPU is excluded, except as shadow rung 2 at dispatch time** (operator ruling 2026-09-25, option b; `cop-inbox/handoff/ruling-r9-b.md`). + - No REX experiment cell, teacher or finetune job, resident `apr serve` or VRAM-preloaded weights may place work on the lambda-labs 4090. Every other R-9 exclusion stands. + - The single exception: the apr SHADOW lane may run on `lambda-cuda` as rung 2, only when rung 1 (`gx10-cuda`) returned `NotRun` and the lambda GPU lock is free at dispatch time. + - `lambda-cuda` yields to agent sessions and `train-active`. A lock taken before or during the round makes the row `NotRun{Busy}`, never a Verdict, and the ladder continues. The round is not retried on the same cell. + - Fallback order: `gx10-cuda` → `lambda-cuda` → `intel-wgpu` → `mini-metal` → `intel-cpu` → `NotRun{NoExecutor}`. Each row records the backend that actually ran; the `gx10-cpu` labelling rule applies to lambda too. + - The rung-2 routing lives in paiml-implement#436. Falsifier: a fixture holding the lambda GPU lock produces a `lambda-cuda` row of `NotRun{Busy}`. +- **R-10 The release train wins.** When `train-active` is set, the lane and the experiment jobs yield on clean-room/CUDA pools. +- **R-11 The asymmetric vote holds** until H4 and H5 pass through the §5.4 machinery. Before that, apr never casts a deciding vote. +- **R-12 Measurement states its tree:** HEAD vs `origin/main`, the host, and a per-session worktree. + +--- + +## §7 Tickets (EV-ordered) + +`K̂` is minutes `[A]`, recalibrated after the first three rows. + +| EV | Row | Work | Contract | Done when (all must hold) | K̂ | +|---|---|---|---|---|---| +| 0 | **REX-00** pre-register | Commit this spec, the analysis plan (§3 tests as code signatures), the test-item hash manifest (empty placeholder until REX-02), prompt v1 and its sha | `rex-prereg-v1` | prereg sha recorded; a planted edit to §3 after the commit makes the prereg check fail | 45 | +| 0 | **REX-01** executor issues | File the infra issue: forjar declarations per host (intel, lambda-labs, gx10, mini) for the exact `apr` tag, weights (4B, 9B; 27B on gx10 only) by path+sha256, an `apr serve` unit with its own cgroup budget and health endpoint, `exec_sha256`, `train-active` yield, the intel concurrency group shared with clean-room, a non-clean-room label for intel. **gx10's declaration is marked priority 1 (day-0 shadow).** File the paiml-implement issue for the 4th shadow lane (`quorum.lane_models`) and ARB-APR-001's gx10 revision. Phase 0 first checks whether any existing declared label already admits these jobs | — | issue URLs recorded; each issue carries checkable acceptance criteria | 60 | +| 1 | **REX-02** corpus | Build P/R/G per §2.2, strata, splits, the sealed hash manifest, the contamination contract with fixtures | `review-corpus-v1`, `review-corpus-contamination-v1` | counts per class/stratum/split receipted; a planted test-item leak into a train file fails CI | 180 | +| 1 | **REX-03** harness | A Rust bench binary drives a resident `apr serve` over HTTP, applies the fixed prompt, parses verdicts deterministically, writes `review-experiment-receipt-v1` rows, and pushes signed receipts. The scorer implements §2.3 and §3 (Wilson, McNemar exact, seeded bootstrap, Holm) | `review-experiment-receipt-v1` (pv; falsifiers: missing model sha → inadmissible; `NotRun` counted as correct → rejected; `Unparsed` counted as PASS → rejected) | scorer unit-tested against hand-computed fixtures; each falsifier planted RED→GREEN | 240 | +| 2 | **REX-04** parity admission | `apr parity` per cell against llama.cpp `d1d3c3396` `[X]` on the exact weights; refusals recorded with `removed_by` | `rex-cell-admission-v1` | 6/6 cells are `Admitted`, `Refused{removed_by}` or `NotRun{NoDeclaredExecutor}`, each with a receipt; 0 cells silent | 120 | +| 2 | **REX-05** pilot | 10 dev items on every admitted cell, plus cold-start runs and the H6 reference workload alone ×5 | extends REX-03 | variance and runtime projection receipted; the sample-size rule from §2.2 applied and logged | 90 | +| 3 | **REX-06** full run | Test split on all admitted cells; 9B control on the reference cell; Haiku and agy baselines on the reference cell; 10% determinism re-run; H6 co-located runs | extends REX-03 | 100% of (item × cell × arm) have an admissible receipt or an explicit `NotRun`; 0 `unknown` identity fields | 240 | +| 3 | **REX-07** day-0 shadow lane | As soon as gx10's declaration lands: the 4B lane runs as a non-voting 4th lane on every aprender quorum and writes `review-ledger-v1` | `review-ledger-v1` | 100% of quorums after activation carry a shadow row (0 missing); a planted gx10-down leaves the quorum's width at 3, and the shadow row is written as `Unknown{LaneUnavailable}` | 90 | +| 4 | **REX-08** analysis + hardware ruling | Run the §3 tests and apply the §4 rule; commit `rex-001-hardware-ruling.md`; file one aprender issue per H1/H2 divergence and one per cell perf gap vs llama.cpp; file an infra issue if the primary cell ≠ gx10 | `rex-001-report-v1` | every §3 hypothesis has a verdict with its statistic and CI; primary and failover cells named, or "shadow only" with its reason | 150 | +| 5 | **REX-09** promotion ladder | Encode H4/H5 as the shadow→tripwire→vote gates in paiml-implement (filed issue) and arbiter `decide` (filed issue) | extends ARB-APR-001 | the gates read only `rex-001-report-v1` and later §5.4 reports; a planted report with H4 failing keeps the lane in shadow | 60 | +| 6 | **REX-10** perf ratchet | Per-tag §5.1 job on the primary cell; ratchet file; knob sweep per train | `review-lane-perf-ratchet-v1` | 3 consecutive tags recorded; a planted +10% p95 regression turns the andon RED | 90 | +| 7 | **REX-11** B1 loop | Prompt versions and retrieval few-shot evaluated via §5.4 | `review-champion-challenger-v1` | ≥ 1 challenger evaluated end-to-end with its receipt, promoted or rejected by rule; the test-version evaluation counter increments | 120 | +| 8 | **REX-12** B2 loop | Teacher logit generation (27B on gx10, top-k, m = 1) over the train pool; the B2a/B2b/B2c acceptance tests wired to the qwen35 finetune/distill/merge tickets | extends REX-11 | the teacher dataset is receipted (sha, count, 0 test hashes); B2 rows read `NotRun{VerbRefused}` until the verbs land, then run §5.4 automatically | 150 | + +**K̂ = 1635 `[A]` · K = 1800 · andon at 1440.** + +--- + +## §8 STOP conditions (exhaustive; stop, write the §9 report, do not work around) + +- **S-1** REX-00 cannot be committed before any measurement (pre-registration out of order). +- **S-2** An executing host resolves `apr` to an undeclared binary, or its weights sha mismatches the declaration. +- **S-3** The sealed-test contamination check fails anywhere. +- **S-4** Any change would place work on the lambda-labs GPU outside the R-9 shadow rung-2 exception. +- **S-5** A run overlaps `train-active` on a clean-room/CUDA pool without yielding. +- **S-6** **Training on hosted-model outputs.** Using Claude or agy (Gemini) outputs as training data for B2 may conflict with those providers' terms. It is an operator decision, and until Noah rules, silver labels stay evaluation-irrelevant and unused for training. B2 proceeds with the local 27B teacher and gold labels only. +- **S-7** Every cell is `Refused` or `NotRun`, so there is no admissible cell for (A). +- **S-8** A harness hook refuses the session's writes for a reason other than a missing ticket. +- **S-9** Budget K is reached, or the andon is crossed with more than one row incomplete. +- **S-10** Two consecutive non-passing quorums on the same PR. + +--- + +## §9 Final report schema (one per session) + +```yaml +spec: REX-001 +session: {date_utc, host, tree: {head, origin_main, worktree}, model} +prereg_sha: +selected_row: REX-NN +ticket: PMAT-NNNN +outcome: merged | stopped | already-done | premise-falsified +stop: {id: S-n, evidence: "..."} # only when stopped +baseline_remeasured: [{row: G#, value, mark, command}] +cells: + - {cell: C1..C5b, status: Admitted|Refused|NotRun, reason, apr_tag, apr_sha, weights_sha, parity: {cosine, receipt}} +results: # REX-08 onward + hypotheses: [{id: H1..H6, statistic, ci, p_holm, verdict}] + per_cell: [{cell, n, recall, precision, false_refute, localization, parse_rate, + p50_s, p95_s, p95_ci, ttft_ms, prefill_tps, decode_tps, peak_anon_gb, + llama_cpp_ratio, interference: {slowdown, noise, rejected}}] + baselines: [{lane, model_id, recall, precision, p95_s}] +hardware_ruling: {primary, failover, mode: vote|tripwire|shadow, reason} +promotion: {champion: {weights_sha, prompt_sha}, challengers: [{id, verdict, mcnemar_p, precision_delta}], test_version, evaluations_used} +issues_filed: [{repo, url, purpose}] +scoreboard_moved: [{metric, before, after, command}] +budget: {k_hat_row, k_actual_row, cumulative, andon_crossed: bool} +next_row: REX-NN +``` + +### §9.1 Scoreboard (targets and probes only; state is rendered, never written here) + +| Metric | Target | Rendered by | +|---|---|---| +| Cells with an admissible receipt, a refusal or an explicit NotRun | **6/6**, 0 silent | `rex-001-report-v1` | +| Cross-cell verdict divergence on admitted cells | **0** | H1 | +| Receipts with `unknown` identity | **0** | receipt lint | +| Quorums carrying a shadow row after activation | **100%** | `review-ledger-v1` | +| Sealed-test contamination | **0** hashes | contamination contract | +| Review p95 on the primary cell | never-up after 3 tags | perf ratchet | +| Gap to llama.cpp on the primary cell | measured every tag; filed when it grows | perf ratchet | +| Champion correctness on the sealed test | never-down across promotions | §5.4 reports | +| Lane precision / recall | measured, then ratcheted never-down | §5.4 reports | +| Lambda GPU minutes consumed by this spec outside the R-9 shadow rung 2 | **0** | host receipts | +| Python lines in files this spec touches | **0** | `grep -rl python3` over the diff | diff --git a/evidence/pra-001/report.yaml b/evidence/pra-001/report.yaml new file mode 100644 index 0000000000..dbcf898242 --- /dev/null +++ b/evidence/pra-001/report.yaml @@ -0,0 +1,42 @@ +# PRA-001 §9 final report. Lead: aprender-a2. Updated on every ticket close. +# The cop reads this file; it does not run measurements. +report: PRA-001 +version: 1 +generated_at: 2026-09-25T17:40:00Z +status: IN_PROGRESS +stop: {code: null, evidence: null} +apr_tag: null # set when a ticket measures against a sha-verified tag +spec: {path: docs/specifications/research-design-pr-agent-qwen-3.5.md, handoff_sha256_prefix: e8764b72adbe, amended: "§1.1 (T0 verify commands)"} +baseline: # §1 after T0; full rows in evidence/pra-001/t0-baseline.yaml + source: evidence/pra-001/t0-baseline.yaml + verified: [G3, G5, G8] + partial: [G9, G14] + ruling: [G6] + pending_instrument: {G1: T1, G7: T10, G10: T1, G11: T6, G13: T16} + stop_s_base: {G2: "paiml-implement#436 (T1)", G4: "paiml-implement#436 R-9", G12: "apr finetune --plan exits 0 with no refusal on 0.69.3"} +tickets: + - {id: T0, owner: aprender-60, contract: null, state: DONE, evidence: evidence/pra-001/t0-baseline.yaml} + - {id: T1, owner: paiml-implement-21, contract: agent-trace-v1, state: IN_PROGRESS} + - {id: T2, owner: "aprender-a2 + aprender-84", contract: review-ledger-v2, state: IN_PROGRESS, khat_min: 120} + - {id: T3, owner: infra-27, contract: trace-blob-store-v1, state: IN_PROGRESS} + - {id: T4, owner: infra-27, contract: transcript-retention-v1, state: IN_PROGRESS, note: "retention done 5/5 hosts, infra#1135"} + - {id: T5, owner: infra-27, contract: trace-backup-v1, state: IN_PROGRESS} + - {id: T6, owner: aprender-cb, contract: trace-admission-secret-v1, state: IN_PROGRESS} + - {id: T7, owner: aprender-cb, contract: review-corpus-contamination-v1, state: BLOCKED_ON, on: T6} + - {id: T8, owner: aprender-a2, contract: trace-dedup-v1, state: BLOCKED_ON, on: T2, khat_min: 150} + - {id: T9, owner: aprender-a2, contract: trace-split-guard-v1, state: BLOCKED_ON, on: T8, khat_min: 120} + - {id: T10, owner: aprender-a2, contract: trace-outcome-join-v1, state: BLOCKED_ON, on: T9, khat_min: 150} + - {id: T11, owner: aprender-84, contract: sparse-logits-v1, state: OPEN} + - {id: T12, owner: aprender-fb, contract: agent-trace-v1 (terms fields), state: OPEN} + - {id: T13, owner: aprender-36, contract: lane-independence-v1, state: OPEN, note: "idle until rc.2"} + - {id: T14, owner: aprender-88, contract: trace-datacard-v1, state: IN_PROGRESS} + - {id: T15, owner: null, state: NOT_STARTED, note: "not before T1-T10 GREEN"} + - {id: T16, owner: null, state: NOT_STARTED, note: "needs gx10 (infra#1132)"} +measurements: [] +gates: [] +falsifiers: [] +storage: + bytes_raw_per_week: null # trace::weekly over week 1 (T1) + bytes_zstd_per_week: null + projected_tb_per_year: null + restore_drill_mismatch: null diff --git a/evidence/pra-001/t0-baseline.yaml b/evidence/pra-001/t0-baseline.yaml new file mode 100644 index 0000000000..b9993405cc --- /dev/null +++ b/evidence/pra-001/t0-baseline.yaml @@ -0,0 +1,43 @@ +# PRA-001 T0 — baseline freeze (§1 G1–G14), report-only. Owner aprender-60, 2026-09-25 17:06Z, host lambda-labs. +# For aprender-a2's §9 report (evidence/pra-001/report.yaml `baseline:`). +# Finding that applies to all rows: NONE of the §1 "Verify command (T0)" commands exists as written. +# - make targets trace-yield, trace-served-by-audit, trace-outcome-join, trace-bytes, trace-secret-scan-backfill, +# teacher-logit-smoke, transcript-retention-audit: 0 of 7 in aprender origin/main Makefile +# - `pmat quorum lanes`: no `quorum` subcommand; `forjar inventory almacen`: no such argument form; +# `pv check`: no `check` subcommand; ledger.rs is on origin/rex/001 only, not main; +# `apr finetune --model-arch`: no such flag in apr 0.69.3. +# Where a fact could be measured another way it is [V] with that command; otherwise STOP S-BASE, not guessed. +apr_tag: v0.69.3 # x86_64 cpu asset, sha256 of tarball verified; apr binary a83d8c8e0829c9fa…580f6 +baseline: + - {id: G1, mark: U-pending-instrument, ticket: T1, ruling: "cop 2026-09-25: missing instrument, not a contradiction", value: null, + evidence: "quorum artifacts carry no timestamp (keys: agreed,author,…,head,lanes,ticket,width); file mtimes are checkout-contaminated (581 of 640 dated today); make trace-yield absent"} + - {id: G2, mark: STOP, code: S-BASE, scope: "paiml-implement#436 (T1)", value: "installed ~/.claude/skills/paiml-implement/config.json quorum.lane_models = [gemini-3.1-pro-high, gemini-3.8-flash-high, gemini-3.7-flash-high]; quorum.advisory_lane = null; quorum.local_lane = null", + command: "jq '.quorum' ~/.claude/skills/paiml-implement/config.json", + evidence: "DRIFT vs [A]: counted lanes sonnet-5+haiku-4-5+agy are passed per-round via --lane-model, not the rail config; the Qwen shadow is not wired in the installed rail at all"} + - {id: G3, mark: V, value: "NotRun{NoExecutor|Busy|Timeout|ContextOverflow|TrainActive} + Verdict + Refused", + command: "git grep -n 'enum NotRun' origin/rex/001 -- crates/aprender-review-experiment/src/ledger.rs", + note: "matches [A]; lives on origin/rex/001 (not main). receipt.rs has a different NotRun (NoDeclaredExecutor|ContextOverflow|Refused|ServeError|Inadmissible)"} + - {id: G4, mark: STOP, code: S-BASE, scope: "paiml-implement#436 R-9", value: "ledger row schema records cell+backend (gx10-cuda, intel-wgpu in tests); no ladder dispatcher exists", + evidence: "no gx10-cuda→lambda-cuda→intel-wgpu→mini-metal→intel-cpu dispatcher in paiml-implement origin/main or aprender origin/main; the only deployed executor is the gx10 watcher apr_dogfood_lane_4252.py (availability ≤60.8%, aprender evidence/4252-promotion.md)"} + - {id: G5, mark: V, value: "LaneRow {family, model, verdict, findings: usize}", + command: "git show origin/rex/001:crates/aprender-review-experiment/src/ledger.rs | grep -n findings", + note: "matches [A]; file is on origin/rex/001, not main"} + - {id: G6, mark: A, value: "operator ruling S-6", note: "a ruling, not measurable (spec: n/a)"} + - {id: G7, mark: U-pending-instrument, ticket: T10, ruling: "cop 2026-09-25: missing instrument, not a contradiction", value: null, + evidence: "make trace-outcome-join absent; no merged/reverted≤14d/regression outcome join found in infra origin/main arbiter sources"} + - {id: G8, mark: V, value: "Test split 105 items, 70 defects (35 P + 35 R + 35 G); Dev 45; corpus 150", + command: "git show origin/rex/001:docs/audits/review-corpus/corpus-v1.jsonl | jq -r '[.split,.class,((.defect|length)>0)]|@tsv' | sort | uniq -c", + note: "matches [A]; pv check absent, measured by count; corpus on origin/rex/001 only"} + - {id: G9, mark: V-partial, value: "32 TB usable (df: 31.98e12 B, 4.3T used, 15%)", + command: "df -B1 /mnt/unas # //192.168.1.156/media, cifs", + evidence: "S-BASE for the unverifiable sub-claims: RAID 5, 3×16 TB, 10G link are not visible from a client and no forjar inventory entry states them (stack-architecture.yaml: 'UNAS Pro storage' only)"} + - {id: G10, mark: U-pending-instrument, ticket: T1, ruling: "cop 2026-09-25: missing instrument, not a contradiction", value: null, evidence: "make trace-bytes absent; no capture exists yet to measure — stays [U] until T1 captures a week"} + - {id: G11, mark: U-pending-instrument, ticket: T6, ruling: "cop 2026-09-25: missing instrument, not a contradiction", value: null, + evidence: "make trace-secret-scan-backfill absent; no incident record found (gh search 'AWS key' → only pmat#1129, an AWS-key-SHAPED literal the scanner missed, not a landed real key). Not re-scanned here: T6 owns the scanner"} + - {id: G12, mark: STOP, code: S-BASE, note_cop: "verify command wrong: apr has no --model-arch; see spec §1.1", value: "apr 0.69.3: finetune/distill/merge reject --model-arch (usage, rc=2); `apr finetune Qwen3.5-4B-Q4_K_M.gguf --plan` rc=0 (no refusal in plan mode); quantize present; `apr run --help` has no --json-schema", + evidence: "the refusal on a real qwen35 finetune was NOT probed: running one is training work (§8 S-APR before 0.71). Verify command needs rewriting"} + - {id: G13, mark: U-pending-instrument, ticket: T16, ruling: "cop 2026-09-25: missing instrument, not a contradiction", value: null, evidence: "make teacher-logit-smoke absent; 27B teacher on gx10 needs infra#1132"} + - {id: G14, mark: V-host, value: "this host: cleanupPeriodDays = 36500 (default 30 overridden); oldest transcript 2026-08-13", + command: "jq .cleanupPeriodDays ~/.claude/settings.json; find ~/.claude/projects -name '*.jsonl' -printf '%T@\\n' | sort -n | head -1", + note: "the docs claim stays [X]; other hosts not audited (infra-27 T4 reports retention done 5/5, infra#1135)"} +summary: {V: [G3, G5, G8], V_partial: [G9, G14], A_ruling: [G6], STOP_S_BASE: [G1, G2, G4, G7, G10, G11, G12, G13]} diff --git a/scripts/transcript_retention_audit.sh b/scripts/transcript_retention_audit.sh new file mode 100755 index 0000000000..5ac5463988 --- /dev/null +++ b/scripts/transcript_retention_audit.sh @@ -0,0 +1,66 @@ +#!/usr/bin/env bash +# PRA-001 §1 G14 / T0: read-only transcript-retention audit, one row per fleet host. +# +# make transcript-retention-audit # the five fleet hosts +# HOSTS="gx10 mini" make transcript-retention-audit +# +# Row: host cleanupPeriodDays files_older_30d files_total oldest_mtime_utc +# cleanupPeriodDays: the value in ~/.claude/settings.json, `unset` (Claude Code's default of 30 applies) or +# `no-settings`. Files are regular files under ~/.claude/projects. A host with no ~/.claude/projects reports 0 0 -. +# +# Nothing is written on any host. Fails closed: a host that cannot be reached, or whose row does not parse, +# prints `UNREACHABLE` and makes the exit status 1, so a missing row can never read as a compliant one. +# Portable to macOS (mini): no `find -printf`, no jq, `stat` probed for GNU or BSD form. +set -euo pipefail + +HOSTS="${HOSTS:-lambda-labs intel gx10 mini yoga}" +SELF="$(uname -n)" + +# Runs on the audited host under sh. Prints exactly one line: cpd older total oldest_epoch +PROBE=' +p="$HOME/.claude/projects"; s="$HOME/.claude/settings.json" +if [ -f "$s" ]; then + cpd=$(tr -d "\n " < "$s" | sed -n "s/.*\"cleanupPeriodDays\":\([0-9][0-9]*\).*/\1/p") + [ -n "$cpd" ] || cpd=unset +else + cpd=no-settings +fi +if [ -d "$p" ]; then + older=$(find "$p" -type f -mtime +30 | wc -l | tr -d " ") + total=$(find "$p" -type f | wc -l | tr -d " ") + if stat -c %Y / >/dev/null 2>&1; then fmt="-c %Y"; else fmt="-f %m"; fi + oldest=$(find "$p" -type f -exec stat $fmt {} + 2>/dev/null | sort -n | head -1) + [ -n "$oldest" ] || oldest=- +else + older=0; total=0; oldest=- +fi +echo "ROW $cpd $older $total $oldest" +' + +to_utc() { + if [ "$1" = "-" ]; then + printf '%s' "-" + else + date -u -d "@$1" +%Y-%m-%dT%H:%MZ + fi +} + +rc=0 +printf '%s\t%s\t%s\t%s\t%s\n' host cleanupPeriodDays files_older_30d files_total oldest_mtime_utc +read -r -a hosts <<<"$HOSTS" +for h in "${hosts[@]}"; do + if [ "$h" = "$SELF" ]; then + out=$(sh -s <<<"$PROBE" 2>/dev/null) || out="" + else + out=$(timeout 60 ssh -o BatchMode=yes -o ConnectTimeout=10 "$h" sh -s <<<"$PROBE" 2>/dev/null) || out="" + fi + row=$(printf '%s\n' "$out" | sed -n 's/^ROW //p' | tail -1) + if [ -z "$row" ]; then + printf '%s\tUNREACHABLE\t-\t-\t-\n' "$h" + rc=1 + continue + fi + read -r cpd older total oldest <<<"$row" + printf '%s\t%s\t%s\t%s\t%s\n' "$h" "$cpd" "$older" "$total" "$(to_utc "$oldest")" +done +exit "$rc"