diff --git a/.claude/skills/apr-dogfood/SKILL.md b/.claude/skills/apr-dogfood/SKILL.md index 8c779c5b22..226a3bb07e 100644 --- a/.claude/skills/apr-dogfood/SKILL.md +++ b/.claude/skills/apr-dogfood/SKILL.md @@ -1591,7 +1591,7 @@ The nine clusters at zero, largest first: `http-orchestrate-banco` (95), `test-harness` (49), `rag-eval` (44), `qa-cgp` (37), `simulation` (18), `orchestrate-pacha-secrets` (17). -**Report both numbers or neither (T2).** "5 of 14 clusters gated (35.7%)" without "144 of 837 features gated (17.2%)" beside it is a proxy masquerading as coverage, and the gate refuses to emit it — *on this line too*. +**Report both numbers or neither (T2).** "5 of 14 clusters gated (35.7%)" without "144 of 838 features gated (17.2%)" beside it is a proxy masquerading as coverage, and the gate refuses to emit it — *on this line too*. The rule is about the NUMBER, not about a phrasing, and it is enforced on every surface that can emit one: the gate's own report, the receipt, the output of diff --git a/.github/workflows/book-contracts.yml b/.github/workflows/book-contracts.yml index 5a52062a7e..e1ca6e9c65 100644 --- a/.github/workflows/book-contracts.yml +++ b/.github/workflows/book-contracts.yml @@ -33,6 +33,11 @@ on: - "crates/aprender-core/src/**" - "scripts/book-gate.sh" - ".github/workflows/book-contracts.yml" + # #3676 rule 3 (scripts/check_workflow_path_filters.sh): a path filter is a + # claim that nothing outside these paths can break this gate; the nightly run + # re-checks the claim every day. Set ~5h early for the dispatch delay (#3292). + schedule: + - cron: '45 22 * * *' permissions: contents: read diff --git a/.github/workflows/book.yml b/.github/workflows/book.yml index 9f17e1c7e5..ba40bbd70b 100644 --- a/.github/workflows/book.yml +++ b/.github/workflows/book.yml @@ -49,6 +49,11 @@ on: - "crates/aprender-core/src/lib.rs" - "crates/apr-cli/src/extended_commands.rs" - "crates/apr-cli/src/tool_commands.rs" + # #3676 rule 3 (scripts/check_workflow_path_filters.sh): a path filter is a + # claim that nothing outside these paths can break this gate; the nightly run + # re-checks the claim every day. Set ~5h early for the dispatch delay (#3292). + schedule: + - cron: '30 22 * * *' permissions: contents: read diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f678438abc..6f5bfc10f7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -13,6 +13,10 @@ name: CI on: push: branches: [main, master] + # #3676: sovereign-ci's `coverage_on: tag` (below) runs coverage only on a push of a + # v* tag, and its gate REQUIRES it to succeed there. A push filtered to branches never + # fires for a tag, so this is the reusable's second required edit. + tags: ['v*'] pull_request: branches: [main, master] # Merge queue: GitHub dispatches a `merge_group` event for each batch it forms. @@ -63,7 +67,7 @@ jobs: # 23916c00d adds an OPTIONAL `runs_on` input whose default is the label set # this repo already gets, and 70e51ec04 swaps the digest on every job. No # input this repo passes changes meaning. - uses: paiml/.github/.github/workflows/sovereign-ci.yml@70e51ec045d95983e9a3d02342d13283bf5f9dc1 + uses: paiml/.github/.github/workflows/sovereign-ci.yml@f713290c86fcd70d6f26a0faab14e67e6713586f with: repo: ${{ github.event.repository.name }} # Phase 3 pilot (heavy workload) — build-performance.md §7 Phase 3. @@ -112,6 +116,20 @@ jobs: # MECHANISM is live fleet-wide via sovereign-ci (#37); a single-crate repo is # the natural first coverage pilot. aprender's blocking-quality pilot is the # diff-scoped mutation gate below. + # + # #3676 (operator 2026-09-21: "YES, coverage on tags release only"): so this + # job gated nothing AND measured nothing (0 tests on the facade root), at the + # cost of an instrumented rebuild on every PR, merge group and push (237 s + # on main run 35563537942). `coverage_on: tag` runs it only on a push of a v* + # tag (the trigger above) and on workflow_dispatch; everywhere else the job is + # skipped and sovereign-ci's gate prints "coverage: NOT MEASURED", never a + # green. On a v* tag the gate REQUIRES it. The first spelling, + # `skip_coverage: true`, was read by that gate as a mandatory job that did not + # succeed, so `ci / gate` went red on every PR (#3707). Every consumer of the + # number keeps a producer, asserted by scripts/check_coverage_has_producers.sh + # (R1-R5): the pre-publish dogfood's `make coverage-check`, coverage-nightly's + # schedule, and this job on each release tag. + coverage_on: tag secrets: inherit # APR-MONO: Workspace-wide test (all 75 crates) @@ -1164,8 +1182,14 @@ jobs: # unwiring the §4.9.1 signature arm drops it to 23 passed, 6 broken), so a # gate that silently stops discriminating now fails here rather than # certifying a release later. - - name: Perf gate can still discriminate (APR-PERF-GATE-001, case table) - run: bash scripts/perf_gate.sh --selftest + # + # #3676: PATH-SCOPED now. The table's verdict changes only when perf_gate.sh + # or a file it reads changes, and it cost 405 s on every PR, merge group and + # push (main run 35563537942). scripts/check_perf_gate_selftest_scoped.sh -- + # dispatched by guard_tree.sh --no-cargo above, no step here -- runs it when + # the diff vs origin/main touches its DERIVED scope (every $ROOT/ path it + # reads, followed through the scripts it calls) and says so on a SUMMARY + # line when it does not; guards-nightly.yml runs the table whole. # PERF-007, the HOST half of §4.9.1. `perf_gate.sh --selftest` already # runs this table, but check_guards_are_wired.sh cannot see one level of # indirection and its remedy for an unwired guard is to wire it, not to @@ -1365,12 +1389,12 @@ jobs: - name: "Receipt-wiring guard: every rule it states, mutated (15/15 kill)" run: bash scripts/mutate_pr_review_wiring_guard.sh # APR-RELEASE-001 §6 R-2, WIRED (#3200 shipped the guard; this is its - # caller). check_pr_closes_issue.sh is a check_*.sh, so guard_tree.sh - # auto-discovers it and runs it BARE -- and bare runs only its own - # self-test, "self-test OK: 9 case(s)", exit 0. Nine green rows about a - # regex, on every PR, judging NO PR body. The 0.67.0 reconcile found 48 - # merged PRs since the previous tag of which only 9 closed anything; that - # is what this guard exists to stop, and until now it had never read one. + # caller). This step is its ONLY runner: guard_tree.sh skips it as + # wired-with-args (it needs a body), so its case table ran in NO job until + # this step called `--self-test` beside `--body` (found by aprender-f0, + # 2026-09-21; the comment here used to claim guard_tree ran it bare). The + # 0.67.0 reconcile found 48 merged PRs since the previous tag of which only + # 9 closed anything; that is what this guard exists to stop. # # THE BODY ARRIVES THROUGH `env:`, NEVER INTERPOLATED INTO THE SCRIPT. # `${{ }}` is substituted textually BEFORE bash sees the line, so a body @@ -1394,6 +1418,7 @@ jobs: PR_BODY: ${{ github.event.pull_request.body }} run: | set -euo pipefail + bash scripts/check_pr_closes_issue.sh --self-test printf '%s' "$PR_BODY" > "$RUNNER_TEMP/pr-body.txt" bash scripts/check_pr_closes_issue.sh --body "$RUNNER_TEMP/pr-body.txt" # APR-RELEASE-001 §11.1, WIRED. The section shipped as prose in #3268 and @@ -2178,9 +2203,13 @@ jobs: # 3. the suites, judged against the measured ceiling DR_ENV=(-e "SATD_BASELINE=$n") dr "$GITHUB_WORKSPACE" cargo test -p aprender-core --features model-tests \ - --test falsification_spec_v10_tests \ - --test falsification_stress_tests \ - --test falsification_gpu_state_tests + --test falsification_spec_v10_tests + # #3676: the PR path keeps ONLY falsification_spec_v10_tests -- it holds the + # SATD ratchet (F-CHECKLIST-005) measured against origin/main above, the one + # property here a diff can change, and the reason this step was KEPT at the + # 67-E3 merge (on the nightly HEAD == origin/main, so there the ratchet is + # vacuous). falsification_stress_tests and falsification_gpu_state_tests + # judge the whole tree, not the diff; guards-nightly.yml runs all three. # 4. the satd class of the ratchet-semantics case table (rebuilds nothing) DR_ENV=() dr "$GITHUB_WORKSPACE" bash scripts/tests/ratchet_semantics_test.sh --class satd @@ -3094,11 +3123,22 @@ jobs: # main-branch pushes are not blocked by a job that intentionally # did not run. Only an explicit `failure` blocks. MUT="${{ needs.mutants.result }}" - if [ "$MUT" = "failure" ]; then - echo "mutants (diff-scoped mutation) failed: $MUT" - exit 1 - fi - echo "mutants result: $MUT (success/skipped both pass)" + EVT="${{ github.event_name }}" + # --- GATE-MUTANTS-RULE-BEGIN (#3676; scripts/check_ci_gate_mutants_rule.sh runs this block) --- + # `skipped` is a pass ONLY when the skip is the job's own filter: + # mutants declares `if: github.event_name == 'pull_request'`, so on every + # other event it is skipped by design. On a pull_request it must have RUN + # and succeeded -- a diff with no .rs still succeeds (its steps skip, the + # job passes), so a pull_request `skipped` or `cancelled` means the job + # never ran, and the old rule ("success/skipped both pass") passed that. + case "$EVT:$MUT" in + pull_request:success) echo "mutants result: success on a pull_request" ;; + pull_request:*) echo "mutants on a pull_request must succeed; got '$MUT' -- a skip here is not the job's filter"; exit 1 ;; + *:skipped) echo "mutants result: skipped on $EVT (its filter: pull_request only)" ;; + *:success) echo "mutants result: success on $EVT" ;; + *) echo "mutants (diff-scoped mutation) on $EVT: $MUT"; exit 1 ;; + esac + # --- GATE-MUTANTS-RULE-END --- # Receipt presence is NOT read here any more (PP-066 C0-5, PRQ-013, # #2982). This job runs under the PR head's definition of ci.yml, so a # receipt clause here was a check the PR under review could edit. The @@ -3158,6 +3198,7 @@ jobs: # Need history + base branch to compute the PR diff for --in-diff. fetch-depth: 0 - name: Compute PR diff for mutation scoping + id: prdiff # cargo-mutants --in-diff takes a unified diff and mutates ONLY the # lines it adds/changes. We diff the PR head against the merge-base with # the target branch so the scope is exactly "what this PR introduces". @@ -3180,7 +3221,19 @@ jobs: if [ ! -s pr.diff ]; then echo "Empty diff — no code to mutate" fi + # #3676: cargo-mutants mutates Rust. A diff with no .rs path has nothing to + # mutate, but used to pull the image and `cargo install cargo-mutants` + # anyway. --no-renames: a .rs file renamed away still counts. + git diff --no-renames --name-only "$MERGE_BASE"...HEAD > pr.files + if grep -qE '\.rs$' pr.files; then + echo "has_rs=true" >> "$GITHUB_OUTPUT" + echo "Rust paths in the diff: $(grep -cE '\.rs$' pr.files)" + else + echo "has_rs=false" >> "$GITHUB_OUTPUT" + echo "No .rs path in the diff ($(grep -c . pr.files || true) path(s)) -- nothing to mutate; the mutation steps skip and the job passes" + fi - name: Pull sovereign-ci image (with retry + local-cache fallback) + if: steps.prdiff.outputs.has_rs == 'true' # Same two-layer resilience as workspace-test — see that job for full context. run: | if docker image inspect "$IMAGE" > /dev/null 2>&1; then @@ -3203,6 +3256,7 @@ jobs: delay=$((delay + 6)) done - name: Install cargo-mutants + if: steps.prdiff.outputs.has_rs == 'true' run: | docker run --rm \ -e CI -e GITHUB_ACTIONS -e GITHUB_REF -e GITHUB_SHA -e GITHUB_REPOSITORY -e GITHUB_RUN_ID -e GITHUB_EVENT_NAME -e GITHUB_WORKFLOW \ @@ -3211,6 +3265,7 @@ jobs: "$IMAGE" \ cargo install cargo-mutants --locked - name: Run diff-scoped mutation testing (BLOCKING) + if: steps.prdiff.outputs.has_rs == 'true' # No continue-on-error: a missed mutant on the PR diff fails the job, # which fails `gate`, which blocks merge. --in-diff pr.diff restricts # mutation to PR-touched lines. Empty diff → 0 mutants → clean pass. @@ -3311,7 +3366,7 @@ jobs: exit 0 ' - name: Upload mutation results - if: always() + if: always() && steps.prdiff.outputs.has_rs == 'true' uses: actions/upload-artifact@v7 with: name: mutation-results diff --git a/.github/workflows/coverage-nightly.yml b/.github/workflows/coverage-nightly.yml index 19e49ea0e9..c1047285f0 100644 --- a/.github/workflows/coverage-nightly.yml +++ b/.github/workflows/coverage-nightly.yml @@ -33,13 +33,18 @@ on: # `gh workflow run` (#3292); until then, re-measure before trusting. - cron: '0 22 * * *' # 22:00 UTC -> lands ~03:00 UTC workflow_dispatch: {} + # #3676: no v* tag trigger here. The release tag's coverage is owned by ci.yml + # (`coverage_on: tag`, which sovereign-ci's gate requires on a tag push), so a tag + # is measured once. scripts/check_coverage_has_producers.sh R3 keeps it that way. permissions: contents: read issues: write # open/update the single coverage tracking issue concurrency: - group: coverage-nightly + # Per ref (#3676): a tag's run and the nightly on main must not cancel each + # other. A second nightly on the same ref still supersedes the first. + group: coverage-nightly-${{ github.ref }} cancel-in-progress: true jobs: diff --git a/.github/workflows/guards-nightly.yml b/.github/workflows/guards-nightly.yml index deef9af9d2..9ac9ed527b 100644 --- a/.github/workflows/guards-nightly.yml +++ b/.github/workflows/guards-nightly.yml @@ -170,6 +170,8 @@ jobs: run: bash scripts/check_hermetic_stdin_tests.sh - name: Publish safety (binary/oversize files in published packages) run: bash scripts/check_publish_safety.sh + - name: Perf gate can still discriminate (APR-PERF-GATE-001, case table) + run: bash scripts/perf_gate.sh --selftest - name: Book rust examples compile run: bash scripts/check_book_examples_compile.sh - name: aprender-profile lib tests (was gated by nothing) diff --git a/.github/workflows/install-script.yml b/.github/workflows/install-script.yml index 933806814a..df4b38dae6 100644 --- a/.github/workflows/install-script.yml +++ b/.github/workflows/install-script.yml @@ -31,6 +31,11 @@ on: - ".github/workflows/install-script.yml" - ".github/workflows/binary-release.yml" - ".github/workflows/nightly.yml" + # #3676 rule 3 (scripts/check_workflow_path_filters.sh): a path filter is a + # claim that nothing outside these paths can break this gate; the nightly run + # re-checks the claim every day. Set ~5h early for the dispatch delay (#3292). + schedule: + - cron: '0 23 * * *' permissions: contents: read diff --git a/Cargo.lock b/Cargo.lock index c8a90c9044..c5972f190f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1345,6 +1345,7 @@ dependencies = [ "hyper 1.10.1", "indicatif 0.17.11", "libc", + "llguidance", "lz4_flex 0.11.6", "memmap2", "mime", @@ -1370,6 +1371,7 @@ dependencies = [ "tower 0.5.3", "tower-http", "tracing", + "unicode-properties", "ureq 2.12.1", "uuid", "zstd", @@ -5145,6 +5147,21 @@ dependencies = [ "syn 2.0.118", ] +[[package]] +name = "derivre" +version = "0.3.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a892ec3953f4ca893e8d2086ed03d4b58e19c77ffb864a574b332d7d40c175f3" +dependencies = [ + "ahash 0.8.12", + "anyhow", + "bytemuck", + "bytemuck_derive", + "hashbrown 0.17.1", + "regex-syntax", + "strum 0.28.0", +] + [[package]] name = "dhat" version = "0.3.3" @@ -8005,6 +8022,22 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "11d3d7f243d5c5a8b9bb5d6dd2b1602c0cb0b9db1621bafc7ed66e35ff9fe092" +[[package]] +name = "llguidance" +version = "1.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "207e72ede15e79f1e7a7f0e2683387da031c1931f6de650020afb80c571efb67" +dependencies = [ + "anyhow", + "derivre", + "indexmap 2.14.0", + "rayon", + "regex-syntax", + "serde", + "serde_json", + "toktrie", +] + [[package]] name = "lock_api" version = "0.4.14" @@ -12119,6 +12152,15 @@ dependencies = [ "strum_macros 0.27.2", ] +[[package]] +name = "strum" +version = "0.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9628de9b8791db39ceda2b119bbe13134770b56c138ec1d3af810d045c04f9bd" +dependencies = [ + "strum_macros 0.28.0", +] + [[package]] name = "strum_macros" version = "0.24.3" @@ -12157,6 +12199,18 @@ dependencies = [ "syn 2.0.118", ] +[[package]] +name = "strum_macros" +version = "0.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab85eea0270ee17587ed4156089e10b9e6880ee688791d45a905f5b1ca36f664" +dependencies = [ + "heck 0.5.0", + "proc-macro2", + "quote", + "syn 2.0.118", +] + [[package]] name = "subtle" version = "2.6.1" @@ -12849,6 +12903,19 @@ dependencies = [ "tokio", ] +[[package]] +name = "toktrie" +version = "1.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16197572e4d4061591596b3629f587a1d7ac5858948c2097310cbb188da100b5" +dependencies = [ + "anyhow", + "bytemuck", + "bytemuck_derive", + "serde", + "serde_json", +] + [[package]] name = "toml" version = "0.8.23" diff --git a/Cargo.toml b/Cargo.toml index 7db1b1c0c1..319898b3a9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -216,6 +216,9 @@ trybuild = "1" # apr-cli; hoisted here for the normalized content hash (`nfc-trim-ws-v1`). # Hoisting does not change either consumer's resolution. unicode-normalization = "0.1" +# #3726: exact Unicode general categories (\p{L}/\p{N}/\p{M}) for the byte-level BPE +# pre-tokenizer. Already in the lock through rustybuzz; no new crate. +unicode-properties = { version = "0.1.4", default-features = false, features = ["general-category"] } # HuggingFace tokenizers (SetFit / MiniLM conformance, Phase 1 ENC-02). # `default-features = false` is load-bearing, not tidiness: the default set is @@ -632,6 +635,13 @@ gates = [ # runs this entry. The PR-time half, the comparison helpers' case table, is # scripts/check_float16_greedy_parity.sh, which guard_tree.sh dispatches. "scripts/float16_greedy_parity.sh", + # #3726: apr's token ids against the pinned llama-tokenize, for every GGUF on the + # release host and every file in evidence/tokenizer-parity/corpus/. A mismatch, a greedy + # fallback on a byte-level vocabulary, a failed round trip or a non-byte-level model is + # RED. `apr parity` cannot see a tokenizer defect: both of its sides share the tokenizer. + # Release time only, like the entry above; the PR-time half (the helpers' case table) is + # scripts/check_tokenizer_parity.sh, which guard_tree.sh dispatches. + "scripts/tokenizer_parity.sh", ] # The interfaces this release SHIPS, each paired with the e2e target that proves diff --git a/README.md b/README.md index 641c085126..9a73639841 100644 --- a/README.md +++ b/README.md @@ -41,7 +41,7 @@ publishing — all backed by YAML provable contracts that fail CI on drift. | Metric | Count | Source of truth | |-------:|------:|---| | Workspace crates | **79** workspace crates | `cargo metadata --no-deps` (NOT `ls crates/` — 4 are `exclude`d, 1 has no Cargo.toml) | -| Provable contracts | **1828** provable contracts | `contracts/census.json` `.n_files` — the set `pv lint` walks (`pv census`, ONT-001 ONT-1; regenerated by `make contracts`, written by `make readme-sync`, guarded by `scripts/check_readme_claims.sh`) | +| Provable contracts | **1829** provable contracts | `contracts/census.json` `.n_files` — the set `pv lint` walks (`pv census`, ONT-001 ONT-1; regenerated by `make contracts`, written by `make readme-sync`, guarded by `scripts/check_readme_claims.sh`) | | CLI commands | **110** CLI commands | `apr --help` | | Book CLI chapters | **112** chapters | `ls book/src/cli/*.md` | | Book lib chapters | **71** chapters | `ls book/src/lib/*.md` (parity with `pub mod`) | @@ -262,7 +262,7 @@ falsification_tests: prediction: apr validate bad-model.apr exits non-zero ``` -The tree carries 1828 contracts across inference, training, quantization, attention, FFN, +The tree carries 1829 contracts across inference, training, quantization, attention, FFN, tokenization, model formats, CLI safety — and this README itself. ## Migration from old crates diff --git a/ci/explicit-test-commands.d/450-aprender-contracts-cli-ont-release-readiness.cmd b/ci/explicit-test-commands.d/450-aprender-contracts-cli-ont-release-readiness.cmd new file mode 100644 index 0000000000..102d61ebf3 --- /dev/null +++ b/ci/explicit-test-commands.d/450-aprender-contracts-cli-ont-release-readiness.cmd @@ -0,0 +1 @@ +cargo test -p aprender-contracts-cli --test ont_release_readiness diff --git a/contracts/apr-cli-commands-v1.yaml b/contracts/apr-cli-commands-v1.yaml index 9c5006ed46..dfba96b2dc 100644 --- a/contracts/apr-cli-commands-v1.yaml +++ b/contracts/apr-cli-commands-v1.yaml @@ -299,7 +299,7 @@ commands: description: "Tokenizer training pipeline (plan/apply)" requires_model: false side_effects: [filesystem] - subcommands: [apply, encode-corpus, import-hf, plan, repair-manifest, train] + subcommands: [apply, encode, encode-corpus, import-hf, plan, repair-manifest, train] - name: tune category: training diff --git a/contracts/apr-dogfood-coverage-v1.yaml b/contracts/apr-dogfood-coverage-v1.yaml index 5124a5c5be..f4d1558b70 100644 --- a/contracts/apr-dogfood-coverage-v1.yaml +++ b/contracts/apr-dogfood-coverage-v1.yaml @@ -3,7 +3,7 @@ metadata: created: '2026-08-22' author: PAIML Engineering references: - - 'docs/audits/surface_audit.csv — 142 of 830 features gated, across 28 binaries and 14 clusters, measured 2026-08-22 @ 4bbfeb07f; 142 of 832 features gated after the PERF-046 partial re-audit (PR #2742); 143 of 833 features gated after the PP-LLAMA-001 re-audit (GET /v1/effective-config added and swept by the dogfood route enumeration, router.rs rows re-pointed); 144 of 837 features gated after PP-066 L0-1b added the three `apr parity` flag rows (--per-op, --out, --threshold), which the dogfood sweep does not reach -- the denominator grew and the numerator did not' + - 'docs/audits/surface_audit.csv — 142 of 830 features gated, across 28 binaries and 14 clusters, measured 2026-08-22 @ 4bbfeb07f; 142 of 832 features gated after the PERF-046 partial re-audit (PR #2742); 143 of 833 features gated after the PP-LLAMA-001 re-audit (GET /v1/effective-config added and swept by the dogfood route enumeration, router.rs rows re-pointed); 144 of 837 features gated after PP-066 L0-1b added the three `apr parity` flag rows (--per-op, --out, --threshold), which the dogfood sweep does not reach -- the denominator grew and the numerator did not; 144 of 838 features gated after #3726 added `apr tokenize encode` (the tokenizer-parity gate''s id source), which the dogfood sweep does not reach either' - 'docs/audits/surface_audit_elbow.png — k-selection evidence: inertia knee and silhouette over k in [2,45]' - 'scripts/dogfood_cluster.py — the k-sweep that produced the elbow plot (k-selection ONLY; it does not write the columns)' - 'scripts/check_no_cluster_id_keys.sh — T1 enforcement: nothing may key on `cluster_id`' @@ -66,9 +66,9 @@ metadata: overall: command: "python3 scripts/dogfood_baseline.py # section: totals" - rows: 837 + rows: 838 covered: 144 - ratio: 0.1720 # 144/837 -- 17.2%, NOT "about 17%" + ratio: 0.1718 # 144/838 -- 17.2%, NOT "about 17%" binaries: 28 binaries_at_zero_coverage: 27 clusters: 14 @@ -83,7 +83,7 @@ metadata: note: > covered may never DECREASE for any binary; total may never decrease without a reconciliation finding recording which direction moved. - apr: {total: 374, covered: 144} + apr: {total: 375, covered: 144} aprender-orchestrate: {total: 184, covered: 0} aprender-test-cli: {total: 39, covered: 0} pv: {total: 38, covered: 0} @@ -147,7 +147,7 @@ metadata: A PRIOR, NEVER EVIDENCE (T3). Cluster membership says where to look. It cannot assert that a feature works, and quality_1_10 is never derived from it -- severity comes from the 0.63.0 ledger. - apr-core-commands: {total: 110, covered: 39} # 35.5% of cluster, 26.8% of gate effort + apr-core-commands: {total: 111, covered: 39} # 35.1% of cluster, 27.1% of gate effort entrypoint-and-mcp: {total: 99, covered: 9} # 9.1% of cluster, 6.3% of gate effort http-orchestrate-banco: {total: 95, covered: 0} # ZERO -- one HTTP harness, the cheapest large win contracts-pv: {total: 76, covered: 0} # ZERO -- one contract-CLI harness, 76 pv subcommands @@ -172,7 +172,7 @@ metadata: the 4.3% coverage of that band is why. q1_2: {total: 80, covered: 58} # 72.5% q3_4: {total: 49, covered: 27} # 55.1% - q5_6: {total: 675, covered: 29} # 4.3% + q5_6: {total: 676, covered: 29} # 4.3% q7_8: {total: 12, covered: 10} # 83.3% q9_10: {total: 21, covered: 20} # 95.2% @@ -283,7 +283,7 @@ equations: codomain: bool invariants: - "T2. Cluster coverage is a PROXY: one gate in a 95-member cluster is 1%, not covered" - - "5/14 clusters (35.7%) and 144/837 features (17.2%) are reported together, always" + - "5/14 clusters (35.7%) and 144/838 features (17.2%) are reported together, always" - "Enforced mechanically by enforce_pairing(); deleting the feature fraction from the emitter turns the gate RED" - "This is the single most important trap: reporting the proxy alone is how a coverage gate becomes theatre while looking stricter" @@ -672,7 +672,7 @@ falsification_tests: Every emitted line stating a CLUSTER-level ratio also states the underlying FEATURE-level ratio, on EVERY channel one can be emitted through: this gate's report, the receipt, scripts/dogfood_baseline.py and - .claude/skills/apr-dogfood/SKILL.md. 5 of 14 clusters (35.7%) is a proxy and 144 of 837 features (17.2%) is the thing. + .claude/skills/apr-dogfood/SKILL.md. 5 of 14 clusters (35.7%) is a proxy and 144 of 838 features (17.2%) is the thing. One gate in a 95-member cluster is 1%. The rule keys on the NUMBER -- a denominator equal to the cluster count -- not on a phrase, because a phrase can be reworded and a denominator cannot; diff --git a/contracts/census.json b/contracts/census.json index 8162052f50..e7da1d0c6c 100644 --- a/contracts/census.json +++ b/contracts/census.json @@ -1,8 +1,8 @@ { "schema": "ont.paiml.dev/census/v1alpha1", "git_sha": null, - "n_files": 1828, - "n_parsed": 1828, + "n_files": 1829, + "n_parsed": 1829, "n_parse_errors": 0, "parse_errors": [], "quarantined_n": 0, @@ -12,7 +12,7 @@ "kernel": 387, "model-family": 28, "model-family-variant": 1, - "pattern": 90, + "pattern": 91, "pretraining-corpus": 2, "registry": 519, "schema": 766, @@ -23,14 +23,15 @@ "by_entity_type": { "gguf": 2, "json": 3, - "pv-contract": 1 + "pv-contract": 1, + "release-evidence": 1 }, "by_anchoring": { "unanchored": 1822, - "class": 4, + "class": 5, "instance": 2 }, - "id_set_sha256": "69d8018f1f9cc7c7f16fb61566d2a405c1c6ab652c90936cc55398a4549825fe", + "id_set_sha256": "c9188fe95552df5d97f5af193188dc40a8e5a80e4af875aaaf194d3f763e46cf", "declared_external": [ { "name": "provable-contracts", diff --git a/contracts/contracts.nt b/contracts/contracts.nt index 7646da81bd..a958ebaa1e 100644 --- a/contracts/contracts.nt +++ b/contracts/contracts.nt @@ -8911,6 +8911,18 @@ . "contracts/pacha/registry-integrity-v1.yaml"^^ . "registry-integrity-v1"^^ . + . + . + . + . + . + "release-evidence"^^ . + "contracts/release-readiness-v1.yaml"^^ . + "release-readiness-v1"^^ . + "pattern"^^ . + "release-readiness"^^ . + "active"^^ . + "1.0.0"^^ . . . "contracts/release-schedule-06x-v1.yaml"^^ . diff --git a/contracts/lint-baseline.json b/contracts/lint-baseline.json index 5230262959..3dc3abed58 100644 --- a/contracts/lint-baseline.json +++ b/contracts/lint-baseline.json @@ -4,14 +4,14 @@ "armed_shapes": ["ont-shapes-v1", "ladder-measured", "ladder-green"], "ont": { "consumer_present": true, - "contracts_total": 1881, - "entity_types_registered": 10, - "extractors_implemented": 7, - "contracts_anchored": 6, - "contracts_shaped": 5, + "contracts_total": 1882, + "entity_types_registered": 11, + "extractors_implemented": 8, + "contracts_anchored": 7, + "contracts_shaped": 6, "unanchored_but_bindable": 297, "formal_prose": 1464, "legacy_unresolved_depends_on": 8, - "shapes_unarmed": 6 + "shapes_unarmed": 15 } } diff --git a/contracts/model-capability-ladder-v1.yaml b/contracts/model-capability-ladder-v1.yaml index c141463952..8c6859e96e 100644 --- a/contracts/model-capability-ladder-v1.yaml +++ b/contracts/model-capability-ladder-v1.yaml @@ -44,10 +44,41 @@ entity: # for that arch; a claimed backend that falls back is RED. # sha256 pins the exact file: a rung measured on a different file is a different # measurement (Q5_K taught us three readers can each invent a layout). +# +# #3712 (operator 2026-09-21: "you must ensure all models Q4_K CUDA work; the end"; +# publishing with "most working" is a "p0 tire fire"): +# * NO Q4_K rung is optional. `required: false` on a Q4_K rung is refused by the +# gate, and so is a Q4_K rung whose `backends` does not claim cuda. +# * The universe is what each host HOLDS, not this list. `inventory` names where +# and what to look for; scripts/model_ladder.sh measures every match on the host +# (on `inventory.backends`) and writes the list into the receipt (schema v2). +# An inventory model missing from the run, or not green on CUDA, is a FAIL +# naming it. A rung is still how a model gets a pinned sha256 and a host list. ladder: hosts: - { id: lambda, isa: x86_64, gpu: "RTX 4090", cc: sm_89, required: true } - { id: gx10, isa: aarch64, gpu: "GB10", cc: sm_121, required: true } + inventory: + dirs: ["~/models", "~/.apr/models", "~/.cache/apr/models"] # depth 1, the fleet's model dirs + patterns: ["*q4_k*.gguf", "*q4k*.gguf", "*q4_k*.apr", "*q4k*.apr"] # case-insensitive + backends: [cuda] + # #3712 row B (operator 2026-09-21: "MORE than enough tokens for our needs (overdo it), and our verbs are + # dogfooded, i.e chat with and without thinking, ditto run, ditto code"). A cell is (model, host, verb, + # thinking, context rung). The RUNGS are not declared here: ids and token counts live in ONE file, + # evidence/release/context-rungs.json (apr-release-context-rungs/v1, #3715), which pv reads too; + # consumer-max is the max over its consumers[].max_prompt_tokens. Thinking modes are derived per model + # from its chat template (#3723). A cell fits a host iff weights + kv_bytes_per_token * tokens + workspace + # <= the host's TOTAL GPU memory (operator "a", #3710): a fitting cell must pass, a non-fitting one must be + # an honest pre-load refusal, and every owed cell must pass on >= 1 required host. Long rungs are owed by + # the families below plus ONE representative per other arch, named here by file (cop 2026-09-21). + cells: + verbs: [run, chat, serve, code] + long_rungs_for: + families: [qwen35] # every consumer's family: the full rung set, through declared + representatives: # one file per other arch the fleet holds + qwen2: qwen2.5-coder-7b-instruct-q4_k_m.gguf # lambda + gx10 + qwen3: Qwen3-8B-Q4_K_M.gguf # lambda + gx10 + qwen3moe: Qwen3-Coder-30B-A3B-Instruct-Q4_K_M.gguf # gx10 (128 GB unified); fits no 24 GB card long rungs: - id: qwen2-1.5b-q4km family: qwen2.5 @@ -71,8 +102,8 @@ ladder: gguf: Qwen3-8B-Q4_K_M.gguf sha256: d98cdcbd03e17ce47681435b5150e34c1417f50b5c0019dd560e4882c5745785 backends: [cpu, cuda] - required: false - note: "second dense-Qwen3 size; required once #3413 lands so the fix is proved at two widths" + required: false # REFUSED by check_model_ladder.sh (#3712): T-2 is RED while this key stands + note: "second dense-Qwen3 size. No Q4_K model is optional (#3712), so the release gate refuses this key and the release cannot ship on it. It is not flipped here because the ARMED ladder-green shape reads the committed 0.68.2 lambda receipt (golden_output: Empty output — the 0.69.0 hold) and would turn pv lint red on every PR; the flip to true lands with the Qwen3-8B CUDA fix and a committed green lambda receipt" - id: qwen35-0.8b-q4km family: qwen3.5 arch: qwen35 @@ -124,21 +155,24 @@ equations: - "a rung the host does not hold is absent, and absent is FAIL for a required rung: an unmeasured claim is not a passed claim" preconditions: - "binary pinned via scripts/apr_bin.sh (built from HEAD) or DOGFOOD_ALLOW_UNPINNED=1 for the published-crate mode" + - "every apr call runs through apr_locked: the fleet GPU lock (flock, bounded wait; a held lock is an ENV decline naming its holder) and choom -n 1000, so a measurement and never a CI job is the OOM victim (#3712; gx10 global OOM 2026-09-21 15:56:08Z killed CI containers)" host_receipt: - formula: "receipt(host) = {host, version, sha, gpu, cc, rungs: [{id, present, capability_match, golden_output, backends: {b: {ran, fallback}}}]}" + formula: "receipt(host) = {schema: apr-model-ladder-receipt/v2, host, version, sha, apr_sha, gpu, cc, inventory: [{file, sha256, bytes}], rungs: [{id, file, present, capability_match, golden_output, backends: {b: {ran, fallback, rc}}}]}" domain: "evidence/dogfood/models//.json, written only by scripts/model_ladder.sh" invariants: - "receipt.version equals the cargo root version of the tree it was measured on" - "receipt.sha equals the git HEAD the binary was built from" - "receipt.executed >= 1 — a receipt that measured nothing is a decline (exit 2), never a pass" + - "receipt.inventory is MEASURED on the host (every file under inventory.dirs matching inventory.patterns), never copied from the ladder; an empty inventory is FAIL" preconditions: - "python3 with json on the host (yaml is read by python3 too)" gate_verdict: - formula: "T2_models = ∀ host required: receipt(host) fresh(version) ∧ ∀ rung required: green(rung, host)" + formula: "T2_models = ∀ host required: receipt(host) fresh(version) ∧ ∀ rung required: green(rung, host) ∧ ∀ f ∈ receipt(host).inventory: green(f, host, inventory.backends)" domain: "scripts/check_model_ladder.sh, invoked by scripts/dogfood.sh through [package.metadata.dogfood]" invariants: - "the rung list at origin/main is the floor: a PR may add rungs, hosts or backends and may not remove any (same construction as check_multiplatform_dogfood.sh layer 2)" - "a missing receipt is FAIL, not DEFER: unlike the multiplatform gate this needs no published crate, a dev build measures it" + - "a Q4_K rung (file or id matches q4_?k) is required and claims cuda — the key `required: false` on one is refused, not tolerated (#3712)" preconditions: - "git can read origin/main:contracts/model-capability-ladder-v1.yaml, or the run is the bootstrap" @@ -168,6 +202,21 @@ proof_obligations: property: "The ladder can grow and cannot shrink in the same PR that reads it" formal: "rungs(origin/main) ⊆ rungs(HEAD) ∧ hosts(origin/main) ⊆ hosts(HEAD) ∧ ∀ rung: backends_main(rung) ⊆ backends_head(rung)" applies_to: gate_verdict + - id: MCL-INV-006 + type: invariant + property: "No Q4_K rung is optional, and every one claims CUDA" + formal: "∀ rung ∈ ladder.rungs: q4k(rung) ⟹ rung.required = true ∧ cuda ∈ rung.backends" + applies_to: gate_verdict + - id: MCL-INV-008 + type: invariant + property: "Every owed (model, verb, thinking, rung) cell is proved: pass where it fits, an honest refusal where it does not, and a pass on at least one required host" + formal: "∀ host, m ∈ inventory(host), v ∈ cells.verbs, t ∈ modes(template(m)), r ∈ rungs(m): |rows(host, m, v, t, r)| = 1 ∧ (fits(m, r, total(host)) ⟹ pass(row) ∧ backend = cuda ∧ ¬fallback ∧ prompt_tokens ≥ tokens(r) ∧ answer_chars > 0 ∧ (t = on ⟹ think_closed)) ∧ (¬fits ⟹ refused(row) ∧ required_bytes(row) > total(host)) ∧ ∃ host': pass(host', m, v, t, r)" + applies_to: gate_verdict + - id: MCL-INV-007 + type: invariant + property: "The universe is the host's measured inventory — every Q4_K model it holds is in the run and green on CUDA" + formal: "∀ host required, ∀ f ∈ receipt(host).inventory: ∃ row ∈ receipt(host).rungs: row.file = f ∧ row.present ∧ (f ∈ files(ladder) ∨ green(row, inventory.backends)) ∧ receipt(host).inventory ≠ ∅" + applies_to: host_receipt falsification_tests: - id: FALSIFY-MCL-001 @@ -220,6 +269,37 @@ falsification_tests: prediction: "narrowing a rung from every host at origin/main to hosts: [gx10] makes the gate exit 1 with 'hosts DROPPED'" test: "bash scripts/check_model_ladder.sh --self-test --case red-dropped-host-on-rung" if_fails: "a required (rung, host) pair could be dropped by adding a key instead of removing one" + - id: FALSIFY-MCL-011 + rule: "A Q4_K rung cannot be made optional or CPU-only" + prediction: "`required: false` on a Q4_K rung exits 1 naming the rung (case red-q4k-required-false); a Q4_K rung with backends [cpu] exits 1 (case red-q4k-rung-cpu-only)" + test: "bash scripts/check_model_ladder.sh --self-test --case red-q4k-required-false && bash scripts/check_model_ladder.sh --self-test --case red-q4k-rung-cpu-only" + if_fails: "the 0.69.0 hold: qwen3-8b-q4km was required: false, so an empty-output CUDA model was a note, not a failure" + - id: FALSIFY-MCL-012 + rule: "An inventory model missing from the run is named" + prediction: "a receipt whose inventory lists a file with no present row exits 1 with 'is MISSING from the run' naming the file (case red-inventory-model-missing)" + test: "bash scripts/check_model_ladder.sh --self-test --case red-inventory-model-missing" + if_fails: "the universe is the list again: a model the host holds is never measured" + - id: FALSIFY-MCL-013 + rule: "An inventory model beyond the ladder is judged on CUDA like a rung" + prediction: "an inventory-only row whose golden_output is skipped, or whose cuda fell back, exits 1 naming inv: (cases red-inventory-model-golden-skipped, red-inventory-model-fell-back); green on cuda exits 0 (case green-inventory-beyond-ladder)" + test: "bash scripts/check_model_ladder.sh --self-test --case red-inventory-model-golden-skipped && bash scripts/check_model_ladder.sh --self-test --case red-inventory-model-fell-back && bash scripts/check_model_ladder.sh --self-test --case green-inventory-beyond-ladder" + if_fails: "an unlisted model is recorded but never judged — present is not green" + - id: FALSIFY-MCL-014 + rule: "A receipt without a measured inventory is not evidence" + prediction: "a v1 receipt (no inventory) exits 1; an empty inventory exits 1; a ladder with no inventory block exits 1 (cases red-receipt-v1-no-inventory, red-inventory-empty, red-ladder-without-inventory)" + test: "bash scripts/check_model_ladder.sh --self-test --case red-receipt-v1-no-inventory && bash scripts/check_model_ladder.sh --self-test --case red-inventory-empty && bash scripts/check_model_ladder.sh --self-test --case red-ladder-without-inventory" + if_fails: "absence scored as conformance: a host that listed nothing proved nothing and passed" + - id: FALSIFY-MCL-015 + rule: "Every GPU apr call runs under the fleet lock, choom'd to 1000, with a bounded wait" + prediction: "check_model_ladder.sh --self-test: model_ladder.sh makes no \"$APR\" call outside apr_locked; a fake apr called through --lock-probe sees the lock held and oom_score_adj 1000; a lock held elsewhere declines with exit 2 naming the holder's pid; four producer mutants (raw call, no flock, no choom, no -w) are each killed. The real run is RED when the audit finds a raw call" + test: "bash scripts/check_model_ladder.sh --self-test" + if_fails: "a ladder run on gx10's unified memory OOM-kills the CI pool (15:56:08Z, 18 kills in 10 s) or hangs a release on a stuck lock" + + - id: FALSIFY-MCL-016 + rule: "Cells: verbs x thinking x context rung, derived and proved" + prediction: "check_model_ladder.sh --self-test: green-cells passes (20 owed on lambda, 12 pass, 8 honest refusals); a missing cell, a duplicate row, a fallback, a thinking block that never closed, a prompt under its rung, an empty answer, a co-tenant refusal, an OOM instead of a refusal, a rung that passes nowhere, thinking_modes that disagree with the template, a missing context_length (#3733), no cells, a consumer without a number, a missing context-rungs file, a dropped cells block or verb, a rung id or consumer dropped from context-rungs.json vs origin/main, an arch with no held representative, and a pass where the declared arithmetic says the cell cannot fit (the arithmetic decides what is owed, so it must be right) are each RED naming it; 11 module mutants are each killed by their case" + test: "bash scripts/check_model_ladder.sh --self-test" + if_fails: "a release dogfoods one verb at one short prompt with thinking off and calls the model family working" qa_gate: id: F-MCL-001 @@ -231,5 +311,7 @@ qa_gate: - required_rungs_present_and_green - claimed_backends_ran_without_fallback - ladder_not_shrunk_vs_origin_main - pass_criteria: "check_model_ladder.sh exits 0 with executed >= 1 on every required host" + - q4k_rungs_required_and_claim_cuda + - every_inventory_model_measured_and_green_on_cuda + pass_criteria: "check_model_ladder.sh exits 0 with executed >= 1 and a non-empty measured inventory on every required host" falsification: "plant a receipt with cuda.fallback=true → exit 1" diff --git a/contracts/ontology.yaml b/contracts/ontology.yaml index e026c513a5..ae7c0aa0c6 100644 --- a/contracts/ontology.yaml +++ b/contracts/ontology.yaml @@ -144,6 +144,9 @@ entity_types: # it moves under a shared `Receipt` class once quorum and dispatch receipts join, so the three families # inherit common shapes. NOT the throughput parity receipts — a different family with a different validator. - {name: parity-receipt, extractor: parity_receipt, implemented: true} + # aprender#3715: a release's receipts (per-host inventory + cells, kernel diffs, context rungs, the dogfood + # receipt R5 judged), extracted ONLY when a release subject is given — an ordinary PR has no release. + - {name: release-evidence, extractor: release_evidence, implemented: true} extractors: - {name: pv_contract, reader: ontology/extract/pv_contract.rs, implemented: true} @@ -156,6 +159,7 @@ extractors: - {name: gguf, reader: ontology/extract/gguf.rs, implemented: true} - {name: csv, reader: ontology/extract/csv.rs, implemented: false} - {name: parity_receipt, reader: ontology/extract/parity_receipt.rs, implemented: true} + - {name: release_evidence, reader: ontology/extract/release_evidence.rs, implemented: true} # What this ontology does NOT express, said out loud so nobody encodes it by accident. Each names the reader that # would have to change first. diff --git a/contracts/release-readiness-v1.yaml b/contracts/release-readiness-v1.yaml new file mode 100644 index 0000000000..bf8bc8d8db --- /dev/null +++ b/contracts/release-readiness-v1.yaml @@ -0,0 +1,262 @@ +# ───────────────────────────────────────────────────────────────────────────── +# release-readiness-v1 — the tag is REFUSED unless every cell of the release matrix has a Pass receipt +# (aprender#3715, EPIC #3710, measured side #3712, T-1/T-4 wiring #3708/#3717) +# +# Operator, 2026-09-21, verbatim: "you need to improve our dogfood process and pv SHACL so leaks don't exist +# pre-release. this is clown town." Every 0.69.0 leak was ABSENCE READ AS CONFORMANCE — a rung marked optional, a +# GPU golden error read as a skip (#3711), an inherited dogfood (#3708), models on a host that were not on the +# ladder (#3712), qwen3moe with no CUDA path and no ticket (#3714). So every obligation is a focus node, and a +# missing receipt is a missing edge that `minCount 1` rejects BY CONSTRUCTION. +# +# WHAT IT GOVERNS +# provable_contracts::ontology::extract::release_evidence — the release's receipts as a graph (Σ extractor +# `release_evidence`, entity type `release-evidence`), built ONLY when a release subject is given +# (`pv lint --gate shapes --shape release-readiness-v1 --release-version V --release-commit MC …`). +# provable_contracts::ontology::extract::release_inputs — the subject, the consumer-derived context rungs +# (evidence/release/context-rungs.json), the dogfood receipt R5 judged, the kernel-diff receipts. +# provable_contracts::ontology::receipts — apr-model-ladder-receipt v1 AND v2 (#3712's). +# +# THE UNIVERSE (computed, never listed). Hosts = every `ladder.hosts[]` entry with `required: true`. Per host, +# models = its measured `inventory[]` ∪ the ladder's cuda rungs listed for it. Per model: verbs {run, chat, serve, +# code} × thinking {on, off} (only off when the host MEASURED no thinking mode, from the template's markers) × +# context rungs: the SHORT ones {4k, 8k, 20k} for every model, the LONG ones {60k, consumer-max (computed = max +# over the consumer records), declared (the model's own GGUF context_length)} for the models +# `ladder.cells.long_rungs_for` names (every model when the ladder names none) — minus any rung larger than the +# model's measured length. Kernels = every (kernel, quant) on ANY required host's dispatch path, owed on EVERY +# required host (each GPU arch). +# +# THE FAMILY. `--shape release-readiness-v1` grades these nine shapes, armed whatever `armed_shapes` says, and +# reports them in this declaration order: release → host → context → model → coverage → tokenizer → kernel → +# refusal → cell, so a tokenizer that disagrees with llama.cpp (#3726) or a kernel that diverges across arches is +# named before any model cell it breaks (#3715 addenda). +# +# THE MEMORY RULE (#3710, operator decision "a"): a (model, host, rung) cell is owed on a host only if it FITS — +# weights + KV-per-token × rung + workspace ≤ the host's measured GPU memory, all carried in the host receipt. +# Where it does not fit, the owed cell is a `release:RefusalCell`: an honest pre-load refusal naming the +# arithmetic. Every rung a model declares must be owed on ≥ 1 required host. An unmeasured term leaves the cell +# owed in full (the stricter reading). +# +# NO WAIVERS. #3710 allows "a FAIL for the release unless the operator waives that cell by name", but every agent +# session posts through the operator's gh token, so a comment-based waiver is forgeable by any of them. Cop +# ruling (c): no waiver input exists; a red cell is released over only by the operator overriding the gate +# himself. The operator-only surface is #3722. +# +# KIND: pattern. Vocabulary, extractor and shapes over a graph; the proof is the gate's own case table. +# ───────────────────────────────────────────────────────────────────────────── +name: release-readiness +version: "1.0.0" +scope: > + Whether a release's evidence proves every (model, host, verb, thinking, context) cell, every (kernel, GPU + arch) cell and every model's tokenizer-parity cell, from receipts measured at the release commit. Out of scope: producing the receipts (#3712, + scripts/model_ladder.sh), calling the gate (#3708/#3717, autopilot T-1 and check_publish_preflight T-4), + waivers (#3722), the intel-CPU column (#3710, aprender-a8). +status: active + +metadata: + version: "1.0.0" + kind: pattern + created: '2026-09-21' + last_modified: '2026-09-21' + author: PAIML Engineering + description: > + The release matrix as focus nodes: every obligation of a release is a node derived from measured + inventory, consumer records and dispatch paths, and each needs exactly one fresh Pass receipt; absence, + a skip, a fallback, a stale sha, an unclosed think block, an empty answer or an unmet context size is a + violation naming the cell. + references: + - 'aprender#3715 (this row), #3710 (the 0.69.1 bar, issuecomment-5763489958), #3712 (the receipt format, issuecomment-5763332078 / -5763412455 / -5763540875), #3716 (consumer records), #3722 (waivers)' + - 'contracts/ont-model-receipts-v1.yaml (the ladder join this generalises), contracts/model-capability-ladder-v1.yaml (the required hosts)' + - 'crates/aprender-contracts/src/ontology/extract/{release_evidence.rs, release_inputs.rs}, ontology/receipts.rs, lint/shapes_gate.rs' + +entity: + type: release-evidence + +relations: + depends_on: [ont-shapes-v1, ont-model-receipts-v1, model-capability-ladder-v1] + +shapes: + - id: release-readiness-v1.release + targetClass: release:Release + properties: + - {path: release:requiredHost, minCount: 1, nodeKind: IRI, class: release:Host} + - {path: release:contextRung, minCount: 1, nodeKind: IRI, class: release:ContextRung} + # the dogfood receipt R5 judged: GO, at THIS commit and version — an inherited GO is stale (#3708) + - {path: release:dogfoodReceipt, minCount: 1, maxCount: 1, nodeKind: IRI, + node: {properties: [{path: release:verdict, minCount: 1, maxCount: 1, in: [GO]}, + {path: release:fresh, minCount: 1, maxCount: 1, in: [true]}]}} + - id: release-readiness-v1.host + targetClass: release:Host + properties: + - {path: release:hostReceipt, minCount: 1, maxCount: 1, nodeKind: IRI, + node: {properties: [{path: release:schema, minCount: 1, in: ["apr-model-ladder-receipt/v2"]}, + {path: release:fresh, minCount: 1, maxCount: 1, in: [true]}]}} + # a host that holds nothing proved nothing + - {path: release:holds, minCount: 1} + - {path: release:unmeasuredModel, maxCount: 0} + - {path: release:unkeyedRow, maxCount: 0} + - {path: release:kernelReceipt, minCount: 1, nodeKind: IRI, + node: {properties: [{path: release:fresh, minCount: 1, maxCount: 1, in: [true]}]}} + - {path: release:dispatchKernel, minCount: 1} + - {path: release:modelWithoutDispatch, maxCount: 0} + - id: release-readiness-v1.context + targetClass: release:ContextRung + properties: + - {path: release:tokens, minCount: 1, maxCount: 1} + - {path: release:derivedFrom, minCount: 1} + - {path: release:unmeasuredConsumer, maxCount: 0} + - id: release-readiness-v1.model + targetClass: release:Model + properties: + # the declared rung's size is read from the GGUF on the host, never picked + - {path: release:contextLength, minCount: 1, maxCount: 1} + # `long_rungs_for` keys on general.architecture (#3712 amendment 2) + - {path: release:arch, minCount: 1, maxCount: 1} + # thinking must be what its template evidence says, and the producer's long-rung echo what pv derives + - {path: release:thinkingContradiction, maxCount: 0} + - {path: release:longRungsMismatch, maxCount: 0} + # the memory rule (#3710, operator "a"): every rung a model declares is owed — and so must Pass — on ≥ 1 + # required host; a rung that fits NOWHERE by the declared arithmetic is named here + - id: release-readiness-v1.coverage + targetClass: release:RungCoverage + properties: + - {path: release:owedOn, minCount: 1, nodeKind: IRI, class: release:Host} + # tokenizer parity (#3726): apr's ids against the pinned llama.cpp, one cell per model — the blind spot of every + # apr-vs-apr gate. Zero tokens compared is no measurement, never parity; the round trip must hold too. + - id: release-readiness-v1.tokenizer + targetClass: release:TokenizerCell + properties: + - {path: release:row, minCount: 1, nodeKind: IRI, + node: {properties: [{path: release:verdict, minCount: 1, maxCount: 1, in: [pass]}, + {path: release:parityExact, minCount: 1, maxCount: 1, in: [true]}, + {path: release:roundTrip, minCount: 1, maxCount: 1, in: [true]}, + {path: release:corpusSha, minCount: 1}, + {path: release:comparatorSha, minCount: 1}, + {path: release:fresh, minCount: 1, maxCount: 1, in: [true]}]}} + - id: release-readiness-v1.kernel + targetClass: release:KernelCell + properties: + - {path: release:row, minCount: 1, nodeKind: IRI, + node: {properties: [{path: release:verdict, minCount: 1, maxCount: 1, in: [pass]}, + {path: release:withinBound, minCount: 1, maxCount: 1, in: [true]}, + {path: release:boundSource, minCount: 1}, + {path: release:fresh, minCount: 1, maxCount: 1, in: [true]}]}} + # a cell that does not FIT the host is owed as an HONEST PRE-LOAD REFUSAL naming its arithmetic — never an OOM + # mid-run, never a silent truncation, never a pass (#3710 rule) + - id: release-readiness-v1.refusal + targetClass: release:RefusalCell + properties: + - {path: release:row, minCount: 1, maxCount: 1, nodeKind: IRI, + node: {properties: [{path: release:verdict, minCount: 1, maxCount: 1, in: [refused]}, + {path: release:arithmeticNamed, minCount: 1, maxCount: 1, in: [true]}, + {path: release:fresh, minCount: 1, maxCount: 1, in: [true]}]}} + - id: release-readiness-v1 + targetClass: release:Cell + properties: + - {path: release:row, minCount: 1, maxCount: 1, nodeKind: IRI, + node: {properties: [{path: release:verdict, minCount: 1, maxCount: 1, in: [pass]}, + {path: release:backend, minCount: 1, maxCount: 1, in: [cuda]}, + {path: release:fallback, minCount: 1, maxCount: 1, in: [false]}, + {path: release:fresh, minCount: 1, maxCount: 1, in: [true]}, + {path: release:contextMet, minCount: 1, maxCount: 1, in: [true]}, + {path: release:thinkOk, minCount: 1, maxCount: 1, in: [true]}, + {path: release:answered, minCount: 1, maxCount: 1, in: [true]}]}} + +equations: + cell_universe: + formula: "Cells = ⋃_{h ∈ H_req} ⋃_{m ∈ inv(h) ∪ rungs_cuda(h)} VERBS × modes(m) × ({r ∈ R : (¬long(r) ∨ owes_long(m)) ∧ (tokens(r) ≤ len(m) ∨ len(m) = ⊥)} ∪ {declared | owes_long(m)})" + domain: "H_req = ladder.hosts with required: true; inv(h) = the host receipt's inventory[] with a sha256; R = the declared context rungs" + codomain: "one release:Cell node per element, each with 0..n release:row edges" + invariants: + - "an inventory row without sha256 is a named unmeasuredModel, never dropped" + - "modes(m) = {off} iff the host measured thinking = false; otherwise {on, off}" + - "owes_long(m) ⇔ long_rungs_for absent ∨ arch(m) ∈ families ∨ representatives[arch(m)] = file(m)" + preconditions: + - "a release subject was given: version and a full 40-hex commit" + postconditions: + - "two extractions of the same inputs are byte-identical (R-15)" + lean_theorem: none — L4 not declared + cell_pass: + formula: "Pass(c) ⇔ |rows(c)| = 1 ∧ verdict = pass ∧ backend = cuda ∧ fallback = false ∧ fresh ∧ contextMet ∧ thinkOk ∧ answered" + domain: "every release:Cell" + codomain: "a PV-ONT-011 violation naming the cell, or none" + invariants: + - "fresh ⇔ receipt.version = V ∧ receipt.apr_sha = (receipts_commit ∨ release_commit)" + - "thinkOk ⇔ thinking = off ∨ think_closed = true" + - "answered ⇔ answer_chars > 0" + - "contextMet ⇔ prompt_tokens ≥ tokens(rung), or for declared prompt_tokens + max_tokens ≥ context_length" + preconditions: + - "an absent field is never the passing value" + postconditions: + - "the gate is Pass only if every cell of every required host passes" + lean_theorem: none — L4 not declared + kernel_pass: + formula: "Pass(k, h) ⇔ |rows(k, h)| ≥ 1 ∧ ∀ row: verdict = pass ∧ max_err ≤ bound ∧ bound_source ≠ ⊥ ∧ fresh" + domain: "K × H_req, K = ⋃_h dispatch kernels(h)" + codomain: "a violation naming (kernel, host) — the arch that diverges" + invariants: + - "a kernel measured on one arch only is a missing row on the other" + preconditions: + - "every required host has a kernel-diff receipt listing its dispatch path" + postconditions: + - "kernel violations are reported before model-cell violations" + lean_theorem: none — L4 not declared + +invariants: + - id: RR-INV-001 + property: a missing cell receipt fails the gate naming the cell + formal: '|rows(c)| = 0 ⇒ Fail(release-readiness-v1, c)' + prose: false + - id: RR-INV-002 + property: a skipped, errored, fallback or stale receipt fails the gate naming the cell + formal: 'verdict(row) ≠ pass ∨ fallback(row) ∨ ¬fresh(row) ⇒ Fail(release-readiness-v1, c)' + prose: false + - id: RR-INV-003 + property: a thinking-ON row with an unclosed block or an empty answer fails + formal: 'thinking(c) = on ∧ (¬think_closed(row) ∨ answer_chars(row) = 0) ⇒ Fail(release-readiness-v1, c)' + prose: false + - id: RR-INV-004 + property: a kernel that passes on one arch and fails on another fails on the failing arch + formal: 'Pass(k, gx10) ∧ ¬Pass(k, lambda) ⇒ Fail(release-readiness-v1.kernel, (k, lambda))' + prose: false + - id: RR-INV-005 + property: a required host with no receipt, or an empty inventory, fails naming the host + formal: '|hostReceipt(h)| = 0 ∨ |holds(h)| = 0 ⇒ Fail(release-readiness-v1.host, h)' + prose: false + - id: RR-INV-007 + property: a model whose tokenizer disagrees with the pinned llama.cpp on one id fails + formal: 'mismatches(model) ≥ 1 ∨ tokens_compared(model) = 0 ⇒ Fail(release-readiness-v1.tokenizer, model)' + prose: false + - id: RR-INV-006 + property: no release subject, no verdict + formal: 'subject = ⊥ ⇒ verdict(--shape release-readiness-v1) = Unknown{NoFocus}' + prose: false + +falsification_tests: + - id: FALSIFY-RR-001 + rule: the universe and the join + prediction: inventory ∪ cuda rungs per host, both thinking modes unless measured otherwise, rungs up to the declared length, rows keyed by (sha256, verb, thinking, rung); a foreign schema is refused by name + test: cargo test -p aprender-contracts --lib ontology::extract::release_evidence + if_fails: a model, mode or rung silently drops out of the universe, or a row keys onto the wrong cell + - id: FALSIFY-RR-002 + rule: v1 and v2 receipts + prediction: a v2 receipt joins its rungs like v1 and carries inventory, apr_sha and cells; a v3 is refused by name + test: cargo test -p aprender-contracts --lib ontology::receipts + if_fails: the first committed v2 receipt turns every PR's shapes gate into exit 3 + - id: FALSIFY-RR-003 + rule: every single-cell failure turns the gate RED + prediction: "the all-green fixture passes with every cell named; a missing, skipped, fallback, stale-sha, unclosed-think, empty-answer, short-context cell each fail naming that cell; a kernel green on sm_121 and red on sm_89 fails naming lambda; a missing host receipt, an inherited dogfood and an absent declared length each fail; no subject declines" + test: cargo test -p aprender-contracts-cli --test ont_release_readiness + if_fails: a leak class from 0.69.0 reaches the tag again + +qa_gate: + id: F-RR-001 + name: release-readiness + description: 'Every release cell is a focus node with exactly one fresh Pass receipt (aprender#3715)' + checks: + - universe_derived_not_listed + - absence_is_a_violation + - single_cell_failures_turn_red + - kernel_arch_divergence_named_first + - no_subject_declines + pass_criteria: All 3 falsification tests pass; `pv lint contracts --gate shapes --shape release-readiness-v1 --release-version V --release-commit MC --receipts DIR --dogfood-receipt FILE` exits 0 only when every cell passes + falsification: 'flip one cell row of tests/fixtures/ont/release-green to verdict skip → the gate exits 1 naming that cell' diff --git a/contracts/shapes.ttl b/contracts/shapes.ttl index ebfcdfafc0..667b5d4245 100644 --- a/contracts/shapes.ttl +++ b/contracts/shapes.ttl @@ -248,3 +248,318 @@ ] ; . + a sh:NodeShape ; + sh:targetClass ; + sh:property [ + sh:path ; + sh:minCount 1 ; + sh:class ; + sh:nodeKind sh:IRI ; + ] ; + sh:property [ + sh:path ; + sh:minCount 1 ; + sh:class ; + sh:nodeKind sh:IRI ; + ] ; + sh:property [ + sh:path ; + sh:minCount 1 ; + sh:maxCount 1 ; + sh:nodeKind sh:IRI ; + sh:node ; + ] ; +. + + a sh:NodeShape ; + sh:property [ + sh:path ; + sh:minCount 1 ; + sh:maxCount 1 ; + sh:in ( "GO" ) ; + ] ; + sh:property [ + sh:path ; + sh:minCount 1 ; + sh:maxCount 1 ; + sh:in ( "true"^^ ) ; + ] ; +. + + a sh:NodeShape ; + sh:targetClass ; + sh:property [ + sh:path ; + sh:minCount 1 ; + sh:maxCount 1 ; + sh:nodeKind sh:IRI ; + sh:node ; + ] ; + sh:property [ + sh:path ; + sh:minCount 1 ; + ] ; + sh:property [ + sh:path ; + sh:maxCount 0 ; + ] ; + sh:property [ + sh:path ; + sh:maxCount 0 ; + ] ; + sh:property [ + sh:path ; + sh:minCount 1 ; + sh:nodeKind sh:IRI ; + sh:node ; + ] ; + sh:property [ + sh:path ; + sh:minCount 1 ; + ] ; + sh:property [ + sh:path ; + sh:maxCount 0 ; + ] ; +. + + a sh:NodeShape ; + sh:property [ + sh:path ; + sh:minCount 1 ; + sh:in ( "apr-model-ladder-receipt/v2" ) ; + ] ; + sh:property [ + sh:path ; + sh:minCount 1 ; + sh:maxCount 1 ; + sh:in ( "true"^^ ) ; + ] ; +. + + a sh:NodeShape ; + sh:property [ + sh:path ; + sh:minCount 1 ; + sh:maxCount 1 ; + sh:in ( "true"^^ ) ; + ] ; +. + + a sh:NodeShape ; + sh:targetClass ; + sh:property [ + sh:path ; + sh:minCount 1 ; + sh:maxCount 1 ; + ] ; + sh:property [ + sh:path ; + sh:minCount 1 ; + ] ; + sh:property [ + sh:path ; + sh:maxCount 0 ; + ] ; +. + + a sh:NodeShape ; + sh:targetClass ; + sh:property [ + sh:path ; + sh:minCount 1 ; + sh:maxCount 1 ; + ] ; + sh:property [ + sh:path ; + sh:minCount 1 ; + sh:maxCount 1 ; + ] ; + sh:property [ + sh:path ; + sh:maxCount 0 ; + ] ; + sh:property [ + sh:path ; + sh:maxCount 0 ; + ] ; +. + + a sh:NodeShape ; + sh:targetClass ; + sh:property [ + sh:path ; + sh:minCount 1 ; + sh:class ; + sh:nodeKind sh:IRI ; + ] ; +. + + a sh:NodeShape ; + sh:targetClass ; + sh:property [ + sh:path ; + sh:minCount 1 ; + sh:nodeKind sh:IRI ; + sh:node ; + ] ; +. + + a sh:NodeShape ; + sh:property [ + sh:path ; + sh:minCount 1 ; + sh:maxCount 1 ; + sh:in ( "pass" ) ; + ] ; + sh:property [ + sh:path ; + sh:minCount 1 ; + sh:maxCount 1 ; + sh:in ( "true"^^ ) ; + ] ; + sh:property [ + sh:path ; + sh:minCount 1 ; + sh:maxCount 1 ; + sh:in ( "true"^^ ) ; + ] ; + sh:property [ + sh:path ; + sh:minCount 1 ; + ] ; + sh:property [ + sh:path ; + sh:minCount 1 ; + ] ; + sh:property [ + sh:path ; + sh:minCount 1 ; + sh:maxCount 1 ; + sh:in ( "true"^^ ) ; + ] ; +. + + a sh:NodeShape ; + sh:targetClass ; + sh:property [ + sh:path ; + sh:minCount 1 ; + sh:nodeKind sh:IRI ; + sh:node ; + ] ; +. + + a sh:NodeShape ; + sh:property [ + sh:path ; + sh:minCount 1 ; + sh:maxCount 1 ; + sh:in ( "pass" ) ; + ] ; + sh:property [ + sh:path ; + sh:minCount 1 ; + sh:maxCount 1 ; + sh:in ( "true"^^ ) ; + ] ; + sh:property [ + sh:path ; + sh:minCount 1 ; + ] ; + sh:property [ + sh:path ; + sh:minCount 1 ; + sh:maxCount 1 ; + sh:in ( "true"^^ ) ; + ] ; +. + + a sh:NodeShape ; + sh:targetClass ; + sh:property [ + sh:path ; + sh:minCount 1 ; + sh:maxCount 1 ; + sh:nodeKind sh:IRI ; + sh:node ; + ] ; +. + + a sh:NodeShape ; + sh:property [ + sh:path ; + sh:minCount 1 ; + sh:maxCount 1 ; + sh:in ( "refused" ) ; + ] ; + sh:property [ + sh:path ; + sh:minCount 1 ; + sh:maxCount 1 ; + sh:in ( "true"^^ ) ; + ] ; + sh:property [ + sh:path ; + sh:minCount 1 ; + sh:maxCount 1 ; + sh:in ( "true"^^ ) ; + ] ; +. + + a sh:NodeShape ; + sh:targetClass ; + sh:property [ + sh:path ; + sh:minCount 1 ; + sh:maxCount 1 ; + sh:nodeKind sh:IRI ; + sh:node ; + ] ; +. + + a sh:NodeShape ; + sh:property [ + sh:path ; + sh:minCount 1 ; + sh:maxCount 1 ; + sh:in ( "pass" ) ; + ] ; + sh:property [ + sh:path ; + sh:minCount 1 ; + sh:maxCount 1 ; + sh:in ( "cuda" ) ; + ] ; + sh:property [ + sh:path ; + sh:minCount 1 ; + sh:maxCount 1 ; + sh:in ( "false"^^ ) ; + ] ; + sh:property [ + sh:path ; + sh:minCount 1 ; + sh:maxCount 1 ; + sh:in ( "true"^^ ) ; + ] ; + sh:property [ + sh:path ; + sh:minCount 1 ; + sh:maxCount 1 ; + sh:in ( "true"^^ ) ; + ] ; + sh:property [ + sh:path ; + sh:minCount 1 ; + sh:maxCount 1 ; + sh:in ( "true"^^ ) ; + ] ; + sh:property [ + sh:path ; + sh:minCount 1 ; + sh:maxCount 1 ; + sh:in ( "true"^^ ) ; + ] ; +. + diff --git a/crates/apr-cli/Cargo.toml b/crates/apr-cli/Cargo.toml index a0a0f43dd7..9a48234091 100644 --- a/crates/apr-cli/Cargo.toml +++ b/crates/apr-cli/Cargo.toml @@ -85,7 +85,7 @@ xet = ["hf-hub", "aprender/xet"] code = [] dev = [] # Enable `apr mono` subcommands (publish, shims, audit, archive) safetensors-compare = ["aprender/safetensors-compare"] -inference = ["realizar", "trueno", "tokio", "axum", "futures-util"] +inference = ["realizar", "realizar/structured-output", "trueno", "tokio", "axum", "futures-util"] # #3545: `trueno/cuda` is NOT optional here, and leaving it out was the defect. The inference # stack got CUDA through `realizar/cuda` and WORKED, while `trueno`'s backend registry — the thing # `apr devices` reads — was built without its cuda feature, so `default_factories()` omitted diff --git a/crates/apr-cli/src/commands/golden_output.rs b/crates/apr-cli/src/commands/golden_output.rs index 50292e766e..c0e338c755 100644 --- a/crates/apr-cli/src/commands/golden_output.rs +++ b/crates/apr-cli/src/commands/golden_output.rs @@ -159,9 +159,94 @@ fn generate_golden_for_format( } } +/// Is there a CUDA device this build can use? Both golden rungs ask (#3711). +#[cfg(feature = "cuda")] +fn cuda_device_present() -> bool { + use realizar::cuda::CudaExecutor; + CudaExecutor::is_available() && CudaExecutor::num_devices() > 0 +} + +/// Without the cuda feature there is no device this build can use. +#[cfg(not(feature = "cuda"))] +fn cuda_device_present() -> bool { + false +} + +/// One golden case, judged (#3711). +#[cfg(feature = "inference")] +enum GoldenCaseOutcome { + /// The CPU answer is right, and so is the GPU's when its leg ran; the leg says which. + Passed(GpuGoldenLeg), + /// The gate's verdict for this case: a failure, or the tokenizer skip. + Verdict(GateResult), +} + +/// The GPU half of one golden case (#3711): run and judged when a cuda build has a device +/// and the model is a GGUF, else `NotRun` naming which of those it lacks. +#[cfg(all(feature = "inference", feature = "cuda"))] +fn gpu_golden_leg( + prompt: &str, + expected_patterns: &[&str], + config: &QaConfig, + format: realizar::format::ModelFormat, + mapped: Option<&realizar::gguf::MappedGGUFModel>, + gguf_model: Option<&realizar::gguf::GGUFModel>, + cuda_available: bool, + golden_max_tokens: usize, +) -> Result { + use realizar::format::ModelFormat; + use realizar::gguf::QuantizedGenerateConfig; + + if let Some(why) = gpu_golden_not_run(true, cuda_available, format == ModelFormat::Gguf) { + return Ok(GpuGoldenLeg::NotRun(why)); + } + // Safe: format==Gguf guarantees these are Some + let gguf_ref = gguf_model.expect("GGUF model required for GPU golden output"); + let mapped_ref = mapped.expect("GGUF mapped model required for GPU golden output"); + let specials = aprender::demo::SpecialTokens::qwen2(); + let prompt_tokens = gguf_ref + .encode(prompt) + .unwrap_or_else(|| vec![specials.bos_id, 9707]); + // #1864: GPU path mirrors the CPU gate's fix above — set stop_tokens + // to EOS so generation terminates at end-of-turn rather than running + // the full 512-token budget and drifting into `<|im_start|>` repeats. + let gen_config = QuantizedGenerateConfig { + max_tokens: golden_max_tokens, // GH-279-4: match CPU budget + temperature: 0.0, + top_k: 1, + stop_tokens: vec![specials.eos_id], + ..Default::default() + }; + validate_gpu_golden_output( + mapped_ref, + &prompt_tokens, + &gen_config, + gguf_ref, + expected_patterns, + config, + ) +} + +/// Without the cuda feature the GPU leg never starts, and says so. +#[cfg(all(feature = "inference", not(feature = "cuda")))] +fn gpu_golden_leg( + prompt: &str, + expected_patterns: &[&str], + config: &QaConfig, + format: realizar::format::ModelFormat, + mapped: Option<&realizar::gguf::MappedGGUFModel>, + gguf_model: Option<&realizar::gguf::GGUFModel>, + cuda_available: bool, + golden_max_tokens: usize, +) -> Result { + let _ = (prompt, expected_patterns, config, mapped, gguf_model, golden_max_tokens); + let gguf = format == realizar::format::ModelFormat::Gguf; + Ok(GpuGoldenLeg::NotRun( + gpu_golden_not_run(false, cuda_available, gguf).unwrap_or("this build has no cuda feature"), + )) +} + /// Validate a single golden test case: generate output, check GPU parity, verify patterns. -/// -/// Returns `Ok(None)` on success, `Ok(Some(GateResult))` on failure/skip. #[cfg(feature = "inference")] fn validate_golden_test_case( path: &Path, @@ -173,9 +258,7 @@ fn validate_golden_test_case( gguf_model: Option<&realizar::gguf::GGUFModel>, cuda_available: bool, start: Instant, -) -> Result> { - use realizar::format::ModelFormat; - +) -> Result { // GH-279-4: Thinking models (Qwen3) need extra tokens for ... // chain-of-thought before the answer. 32 tokens is not enough — the model // exhausts the budget on reasoning and never emits the answer. Qwen3's @@ -186,51 +269,32 @@ fn validate_golden_test_case( let Some((_, output_text)) = generate_golden_for_format(path, prompt, golden_max_tokens, format, mapped, gguf_model)? else { - return Ok(Some(GateResult::skipped( + return Ok(GoldenCaseOutcome::Verdict(GateResult::skipped( "golden_output", "SafeTensors: tokenizer.json not found", ))); }; - #[cfg(feature = "cuda")] - if cuda_available && format == ModelFormat::Gguf { - use realizar::gguf::QuantizedGenerateConfig; - // Safe: format==Gguf guarantees these are Some - let gguf_ref = gguf_model.expect("GGUF model required for GPU golden output"); - let mapped_ref = mapped.expect("GGUF mapped model required for GPU golden output"); - let specials = aprender::demo::SpecialTokens::qwen2(); - let prompt_tokens = gguf_ref - .encode(prompt) - .unwrap_or_else(|| vec![specials.bos_id, 9707]); - // #1864: GPU path mirrors the CPU gate's fix above — set stop_tokens - // to EOS so generation terminates at end-of-turn rather than running - // the full 512-token budget and drifting into `<|im_start|>` repeats. - let gen_config = QuantizedGenerateConfig { - max_tokens: golden_max_tokens, // GH-279-4: match CPU budget - temperature: 0.0, - top_k: 1, - stop_tokens: vec![specials.eos_id], - ..Default::default() - }; - if let Some(failure) = validate_gpu_golden_output( - mapped_ref, - &prompt_tokens, - &gen_config, - gguf_ref, - expected_patterns, - config, - )? { - return Ok(Some(GateResult::failed( - "golden_output", - &failure, - None, - None, - start.elapsed(), - ))); - } + // #3711: a GPU leg that errored is a FAIL naming the error, never a skip. + let gpu_leg = gpu_golden_leg( + prompt, + expected_patterns, + config, + format, + mapped, + gguf_model, + cuda_available, + golden_max_tokens, + )?; + if let Some(failure) = gpu_leg.failure() { + return Ok(GoldenCaseOutcome::Verdict(GateResult::failed( + "golden_output", + &failure, + None, + None, + start.elapsed(), + ))); } - #[cfg(not(feature = "cuda"))] - let _ = cuda_available; // GH-279-4: generate_with_cache returns prompt + generated tokens. // Strip the prompt echo so we verify only the model's generated output. @@ -241,7 +305,7 @@ fn validate_golden_test_case( if let OutputVerification::Fail { reason } = verify_output(&answer_text, "golden_output", expected_patterns) { - return Ok(Some(GateResult::failed( + return Ok(GoldenCaseOutcome::Verdict(GateResult::failed( "golden_output", &reason, None, @@ -250,7 +314,7 @@ fn validate_golden_test_case( ))); } - Ok(None) + Ok(GoldenCaseOutcome::Passed(gpu_leg)) } fn run_golden_output_gate(path: &Path, config: &QaConfig) -> Result { @@ -267,13 +331,7 @@ fn run_golden_output_gate(path: &Path, config: &QaConfig) -> Result use realizar::format::{detect_format, ModelFormat}; use realizar::gguf::{GGUFModel, MappedGGUFModel}; - #[cfg(feature = "cuda")] - let cuda_available = { - use realizar::cuda::CudaExecutor; - CudaExecutor::is_available() && CudaExecutor::num_devices() > 0 - }; - #[cfg(not(feature = "cuda"))] - let cuda_available = false; + let cuda_available = cuda_device_present(); let model_bytes = std::fs::read(path) .map_err(|e| CliError::ValidationFailed(format!("Failed to read model: {e}")))?; let format = detect_format(&model_bytes[..8.min(model_bytes.len())]) @@ -290,8 +348,10 @@ fn run_golden_output_gate(path: &Path, config: &QaConfig) -> Result (None, None) }; + // #3711: the pass message names the GPU leg, so a skipped leg says which skip it was + let mut gpu_leg = GpuGoldenLeg::NotRun("no golden case ran"); for (prompt, expected_patterns) in &test_cases { - if let Some(result) = validate_golden_test_case( + match validate_golden_test_case( path, prompt, expected_patterns, @@ -302,13 +362,18 @@ fn run_golden_output_gate(path: &Path, config: &QaConfig) -> Result cuda_available, start, )? { - return Ok(result); + GoldenCaseOutcome::Verdict(result) => return Ok(result), + GoldenCaseOutcome::Passed(leg) => gpu_leg = leg, } } Ok(GateResult::passed( "golden_output", - &format!("{} golden test cases passed", test_cases.len()), + &format!( + "{} golden test cases passed ({})", + test_cases.len(), + gpu_leg.describe() + ), Some(test_cases.len() as f64), Some(test_cases.len() as f64), start.elapsed(), @@ -595,6 +660,119 @@ mod golden_output_tests { ); } } + + // ========================================================================= + // #3711: the GPU leg's verdicts. An ERROR is a FAIL naming the error, never + // a skip; a skip is only a leg that never started, and it names which. + // ========================================================================= + + const TWO_PLUS_TWO: &[&str] = &["4"]; + + #[test] + fn gpu_generation_error_fails_the_gate_naming_the_error() { + let leg = GpuGoldenLeg::judge( + Err("GPU generation: CUDA_ERROR_ILLEGAL_ADDRESS".to_string()), + TWO_PLUS_TWO, + ); + assert_eq!( + leg, + GpuGoldenLeg::Errored("GPU generation: CUDA_ERROR_ILLEGAL_ADDRESS".to_string()) + ); + let failure = leg.failure().expect("a GPU generation error must FAIL the gate"); + assert!(failure.contains("CUDA_ERROR_ILLEGAL_ADDRESS"), "{failure}"); + assert!(failure.contains("not a skip"), "{failure}"); + } + + #[test] + fn cuda_init_error_on_a_host_with_a_device_fails_the_gate() { + let leg = GpuGoldenLeg::judge( + Err("CUDA init on device 0: CUDA_ERROR_OUT_OF_MEMORY".to_string()), + TWO_PLUS_TWO, + ); + let failure = leg.failure().expect("a CUDA init error must FAIL the gate"); + assert!(failure.contains("CUDA_ERROR_OUT_OF_MEMORY"), "{failure}"); + } + + #[test] + fn gpu_wrong_answer_fails_the_gate() { + let leg = GpuGoldenLeg::judge(Ok("2 + 2 = 5".to_string()), TWO_PLUS_TWO); + assert!(matches!(leg, GpuGoldenLeg::WrongAnswer(_)), "{leg:?}"); + let failure = leg.failure().expect("a wrong GPU answer must FAIL the gate"); + assert!(failure.starts_with("GPU output failed (CPU passed)"), "{failure}"); + } + + #[test] + fn gpu_right_answer_passes_and_says_it_was_judged() { + for text in ["2 + 2 = 4", "two and twoThe answer is 4."] { + let leg = GpuGoldenLeg::judge(Ok(text.to_string()), TWO_PLUS_TWO); + assert_eq!(leg, GpuGoldenLeg::Passed, "{text:?}"); + assert_eq!(leg.failure(), None); + assert_eq!(leg.describe(), "GPU leg judged on CUDA device 0"); + } + } + + #[test] + fn no_device_is_the_skip_and_names_it() { + let why = gpu_golden_not_run(true, false, true).expect("no device: the leg never starts"); + assert_eq!(why, "no CUDA device on this host"); + let leg = GpuGoldenLeg::NotRun(why); + assert_eq!(leg.failure(), None, "a leg that never started is not a failure"); + assert_eq!(leg.describe(), "GPU leg SKIPPED: no CUDA device on this host"); + } + + #[test] + fn each_skip_names_which_and_a_cuda_host_runs_the_leg() { + assert_eq!( + gpu_golden_not_run(false, true, true), + Some("this build has no cuda feature") + ); + assert_eq!( + gpu_golden_not_run(false, false, false), + Some("this build has no cuda feature") + ); + assert_eq!( + gpu_golden_not_run(true, true, false), + Some("the dense GPU leg judges GGUF only") + ); + // a cuda build with a device and a GGUF: the leg STARTS, so no skip can be reported + assert_eq!(gpu_golden_not_run(true, true, true), None); + } + + // The runtime rung (the hybrid, and architectures the GPU declines) names the + // backend the dispatch REPORTED. It never names the one the build could have used. + + #[test] + fn runtime_cuda_build_with_no_device_is_labelled_cpu_never_gpu() { + // the old message said "GPU hybrid forward" on every cuda build + let not_run = gpu_golden_not_run(true, false, true); + let label = runtime_golden_backend(false, not_run).expect("CPU was expected: a pass"); + assert!(label.starts_with("CPU: the dispatch reported used_gpu=false"), "{label}"); + assert!(label.contains("no CUDA device on this host"), "{label}"); + assert!(!label.contains("GPU:"), "{label}"); + } + + #[test] + fn runtime_gpu_fallback_on_a_cuda_host_fails_the_gate() { + // a cuda build, a device, an architecture the GPU runs, and the dispatch reported CPU + let failure = runtime_golden_backend(false, gpu_golden_not_run(true, true, true)) + .expect_err("a GPU that fell back to the CPU must FAIL the gate"); + assert!(failure.contains("fell back"), "{failure}"); + assert!(failure.contains("not a pass"), "{failure}"); + } + + #[test] + fn runtime_gpu_that_served_is_labelled_gpu_from_the_dispatch() { + let label = runtime_golden_backend(true, gpu_golden_not_run(true, true, true)) + .expect("the GPU served: a pass"); + assert_eq!(label, "GPU: the dispatch reported used_gpu=true"); + } + + #[test] + fn runtime_architecture_the_gpu_declines_is_a_cpu_pass_that_says_why() { + let label = runtime_golden_backend(false, Some("the GPU backend declines this architecture")) + .expect("CPU was expected: a pass"); + assert!(label.contains("the GPU backend declines this architecture"), "{label}"); + } } include!("throughput.rs"); diff --git a/crates/apr-cli/src/commands/hex.rs b/crates/apr-cli/src/commands/hex.rs index 015dd0775d..f12c79e440 100644 --- a/crates/apr-cli/src/commands/hex.rs +++ b/crates/apr-cli/src/commands/hex.rs @@ -124,11 +124,18 @@ pub(crate) fn detected_model_format(path: &Path) -> &'static str { let mut head = Vec::with_capacity(9); let read = std::fs::File::open(path).and_then(|f| f.take(9).read_to_end(&mut head)); match read { - Ok(_) => detect_format(&head).map_or("model", format_display_name), + Ok(_) => detected_format_of(&head), Err(_) => "model", } } +/// [`detected_model_format`] for bytes already in memory (#3691). `run` reads +/// the whole file before it dispatches, so the SafeTensors header reader names +/// the format from the bytes it holds instead of assuming it. +pub(crate) fn detected_format_of(bytes: &[u8]) -> &'static str { + detect_format(bytes).map_or("model", format_display_name) +} + // ============================================================================ // Main entry point // ============================================================================ @@ -320,7 +327,7 @@ fn parse_gguf(path: &Path) -> Result { use aprender::format::gguf::GgufReader; let reader = GgufReader::from_file(path) - .map_err(|e| CliError::InvalidFormat(format!("Failed to parse GGUF: {e}")))?; + .map_err(|e| CliError::invalid_model_file(path, "Failed to parse GGUF", &e))?; let tensors = reader .tensors @@ -347,7 +354,7 @@ fn get_gguf_tensor_f32(path: &Path, name: &str) -> Result<(Vec, Vec) use aprender::format::gguf::GgufReader; let reader = GgufReader::from_file(path) - .map_err(|e| CliError::InvalidFormat(format!("Failed to parse GGUF: {e}")))?; + .map_err(|e| CliError::invalid_model_file(path, "Failed to parse GGUF", &e))?; reader .get_tensor_f32(name) .map_err(|e| CliError::InvalidFormat(format!("Failed to read tensor '{name}': {e}"))) @@ -455,3 +462,9 @@ include!("valid.rs"); include!("sliding_window_entropy.rs"); include!("hex_print.rs"); include!("hex_06.rs"); + +// #3691: every hex and trace GGUF/SafeTensors parse failure names the format. +// A child of hex so it can reach the header reader behind `run`'s dispatch. +#[cfg(test)] +#[path = "model_file_error_tests_3691.rs"] +mod model_file_error_tests_3691; diff --git a/crates/apr-cli/src/commands/inference_output.rs b/crates/apr-cli/src/commands/inference_output.rs index fd84665dd6..4f4db44080 100644 --- a/crates/apr-cli/src/commands/inference_output.rs +++ b/crates/apr-cli/src/commands/inference_output.rs @@ -184,6 +184,8 @@ struct InferenceOutput { /// Populated only when `--stream` asked for it (see /// [`decode_token_pieces`]) — every other mode renders the whole `text`. token_texts: Option>, + /// Prompt and completion counts plus the finish reason (#3718). + usage: RunUsage, } /// Execute inference on model @@ -231,6 +233,7 @@ fn execute_inference( used_gpu: None, generated_tokens: None, token_texts: None, + usage: RunUsage::default(), }) } } @@ -299,7 +302,8 @@ fn execute_with_realizar( options: &RunOptions, _use_mmap: bool, ) -> Result { - use realizar::{run_inference, InferenceConfig}; + use realizar::infer::run_inference_report; + use realizar::InferenceConfig; // Get prompt from options or input file let prompt = if let Some(ref p) = options.prompt { @@ -344,7 +348,10 @@ fn execute_with_realizar( } // Run inference via realizar - let result = run_inference(&config).map_err(inference_error)?; + // #3718: the report carries what only the decode path knows (finish reason, + // context window); the prompt count is `input_token_count`, taken after the + // chat template, so it is the number the model actually read. + let (result, report) = run_inference_report(&config).map_err(inference_error)?; // Report performance if benchmarking if options.benchmark { @@ -382,6 +389,12 @@ fn execute_with_realizar( used_gpu: Some(result.used_gpu), generated_tokens, token_texts, + usage: RunUsage { + prompt_tokens: Some(result.input_token_count), + completion_tokens: Some(result.generated_token_count), + finish_reason: report.finish_reason.map(|r| r.as_str()), + context_length: report.context_length, + }, }) } diff --git a/crates/apr-cli/src/commands/inspect.rs b/crates/apr-cli/src/commands/inspect.rs index 6fc74333ac..60296fbd93 100644 --- a/crates/apr-cli/src/commands/inspect.rs +++ b/crates/apr-cli/src/commands/inspect.rs @@ -215,6 +215,20 @@ pub(crate) fn run( /// GGUF/SafeTensors inspect via RosettaStone /// Print rosetta inspection report as JSON. fn output_rosetta_json(path: &Path, report: &aprender::format::rosetta::InspectionReport) { + if let Ok(json) = serde_json::to_string_pretty(&rosetta_json(path, report)) { + println!("{json}"); + } +} + +/// The `apr inspect --json` document for a GGUF/SafeTensors file. +/// +/// `metadata` is every key in the report, unfiltered: #3733's gate reads a +/// model's declared context and memory arithmetic from it, and a second GGUF +/// reader is refused (`check_no_hand_rolled_parsers.sh`). +fn rosetta_json( + path: &Path, + report: &aprender::format::rosetta::InspectionReport, +) -> serde_json::Value { let mut json_map = serde_json::Map::new(); json_map.insert( "file".to_string(), @@ -262,10 +276,7 @@ fn output_rosetta_json(path: &Path, report: &aprender::format::rosetta::Inspecti .collect::>() .into(); json_map.insert("metadata".to_string(), metadata); - - if let Ok(json) = serde_json::to_string_pretty(&json_map) { - println!("{json}"); - } + serde_json::Value::Object(json_map) } /// Print rosetta inspection report as rich text. diff --git a/crates/apr-cli/src/commands/inspect_tests.rs b/crates/apr-cli/src/commands/inspect_tests.rs index 8ad8204569..a2757cf464 100644 --- a/crates/apr-cli/src/commands/inspect_tests.rs +++ b/crates/apr-cli/src/commands/inspect_tests.rs @@ -5,6 +5,38 @@ mod inspect_tests { use super::*; use std::path::Path; + // ======================================================================== + // rosetta_json (#3733): --json shows every metadata key, unfiltered + // ======================================================================== + + #[test] + fn rosetta_json_emits_every_metadata_key() { + use aprender::format::rosetta::{FormatType, InspectionReport}; + let keys = [ + "general.architecture", + "qwen35.context_length", + "qwen35.full_attention_interval", + "quantize.imatrix.file", + "zzz_never_heard_of.depth", + "tokenizer.ggml.model", + ]; + let report = InspectionReport { + format: FormatType::Gguf, + file_size: 1, + metadata: keys.iter().map(|k| ((*k).to_string(), "v".to_string())).collect(), + tensors: vec![], + total_params: 0, + quantization: None, + architecture: Some("qwen35".to_string()), + }; + let v = rosetta_json(Path::new("m.gguf"), &report); + let shown = v["metadata"].as_object().expect("metadata object"); + assert_eq!(shown.len(), keys.len()); + for k in keys { + assert!(shown.contains_key(k), "--json dropped {k}"); + } + } + // ======================================================================== // validate_path // ======================================================================== diff --git a/crates/apr-cli/src/commands/mod.rs b/crates/apr-cli/src/commands/mod.rs index 0c2be4309d..70fa582a57 100644 --- a/crates/apr-cli/src/commands/mod.rs +++ b/crates/apr-cli/src/commands/mod.rs @@ -199,6 +199,9 @@ pub(crate) mod stop_op; pub(crate) mod tensors; pub(crate) mod token_redactor; pub(crate) mod tokenize; +/// #3726: `apr tokenize encode` — a GGUF's own token ids for a text, and the path that made them. +#[cfg(feature = "inference")] +pub(crate) mod tokenize_encode; pub(crate) mod tokenize_parquet; pub(crate) mod tool_use_classifier; pub(crate) mod tool_use_lint; diff --git a/crates/apr-cli/src/commands/model_file_error_tests_3661.rs b/crates/apr-cli/src/commands/model_file_error_tests_3661.rs index d19581fc11..469d98c78e 100644 --- a/crates/apr-cli/src/commands/model_file_error_tests_3661.rs +++ b/crates/apr-cli/src/commands/model_file_error_tests_3661.rs @@ -13,7 +13,7 @@ use std::path::{Path, PathBuf}; /// A GGUF written by aprender's own exporter, cut off inside its metadata. /// That is the issue's input: a real GGUF truncated mid-header. -fn truncated_gguf(dir: &Path) -> PathBuf { +pub(super) fn truncated_gguf(dir: &Path) -> PathBuf { use aprender::format::gguf::{export_tensors_to_gguf, GgmlType, GgufTensor, GgufValue}; let tensors = vec![GgufTensor { name: "token_embd.weight".to_string(), @@ -43,7 +43,7 @@ fn truncated_gguf(dir: &Path) -> PathBuf { /// 4 KiB of noise named `.safetensors`, opening with the issue's own magic bytes /// (`8e44f12f`). Its first u64 is far past any real header length, so no format matches. -fn random_safetensors(dir: &Path) -> PathBuf { +pub(super) fn random_safetensors(dir: &Path) -> PathBuf { let mut state: u32 = 0x3661; let mut bytes: Vec = vec![0x8e, 0x44, 0xf1, 0x2f, 0x9d, 0x27, 0x6b, 0xb3]; while bytes.len() < 4096 { @@ -56,7 +56,7 @@ fn random_safetensors(dir: &Path) -> PathBuf { } /// A real APR v2 file cut to 32 bytes, less than its 64-byte header. -fn truncated_apr(dir: &Path) -> PathBuf { +pub(super) fn truncated_apr(dir: &Path) -> PathBuf { use aprender::format::v2::{AprV2Metadata, AprV2Writer}; let mut writer = AprV2Writer::new(AprV2Metadata::new("truncated-3661")); writer.add_f32_tensor("w", vec![2, 2], &[0.0, 1.0, 2.0, 3.0]); diff --git a/crates/apr-cli/src/commands/model_file_error_tests_3691.rs b/crates/apr-cli/src/commands/model_file_error_tests_3691.rs new file mode 100644 index 0000000000..5f006619e8 --- /dev/null +++ b/crates/apr-cli/src/commands/model_file_error_tests_3691.rs @@ -0,0 +1,195 @@ +//! #3691: `apr hex` and `apr trace` name a GGUF or SafeTensors file by its format. +//! +//! #3661 fixed `apr inspect` and `apr tensors`. Ten more refusals in hex, its +//! SafeTensors header reader, and trace still wrapped their message in +//! `CliError::InvalidFormat`, so a truncated GGUF printed +//! "error: Invalid APR format: Failed to parse GGUF: …". One row per site, +//! rendered exactly as `apr` prints it (`error: {e}`, main.rs), checked the same +//! four ways as #3661's table: the prefix, no "Invalid APR" for a non-APR file, +//! no core "Invalid model format:" leaking under the CLI's own, and exit code 4. +//! +//! This module is a child of `hex`, so it can call the header reader directly. +//! Three sites (the reader's size check, and the "not a JSON object" check in +//! each of its two callers) sit behind `run`'s `detect_format`, which only picks +//! SafeTensors for 9+ bytes with `{` at byte 8. No file reaches them through +//! `run`, so their rows call the private function with the bytes that would. + +use super::{parse_safetensors_header, run, run_safetensors, slice_safetensors, HexOptions}; +use crate::commands::model_file_error_tests_3661::{ + random_safetensors, truncated_apr, truncated_gguf, +}; +use crate::commands::trace; +use crate::error::CliError; +use std::path::{Path, PathBuf}; + +/// A SafeTensors file: `header_len` as u64 LE, then `header`, then `tail`. +fn safetensors_bytes(header_len: u64, header: &[u8], tail: &[u8]) -> Vec { + let mut bytes = header_len.to_le_bytes().to_vec(); + bytes.extend_from_slice(header); + bytes.extend_from_slice(tail); + bytes +} + +fn write(dir: &Path, name: &str, bytes: &[u8]) -> PathBuf { + let path = dir.join(name); + std::fs::write(&path, bytes).expect("write fixture"); + path +} + +/// The header claims 1000 bytes; the file holds 16. +fn st_header_overrun() -> Vec { + safetensors_bytes(1000, b"{\"a\":1}", b"x") +} + +/// A 4-byte header that opens like JSON and is not UTF-8. +fn st_header_not_utf8() -> Vec { + safetensors_bytes(4, b"{\xff\xfe}", b"") +} + +/// A 5-byte header, `{"a":`, cut off mid-object. It opens with `{"`, so +/// rosetta's magic check (the one trace uses) takes it for SafeTensors too. +fn st_header_not_json() -> Vec { + safetensors_bytes(5, b"{\"a\":", b"") +} + +/// A header that parses as JSON but is an array, not an object. +fn st_header_array() -> Vec { + safetensors_bytes(3, b"[1]", b"") +} + +fn hex_err(path: &Path) -> CliError { + let opts = HexOptions { + file: path.to_path_buf(), + ..HexOptions::default() + }; + run(&opts).expect_err("hex must refuse") +} + +/// `apr hex FILE --slice 0:1 --tensor w` +fn hex_slice_err(path: &Path) -> CliError { + let opts = HexOptions { + file: path.to_path_buf(), + tensor: Some("w".to_string()), + slice: Some("0:1".to_string()), + ..HexOptions::default() + }; + run(&opts).expect_err("hex --slice must refuse") +} + +fn trace_err(path: &Path) -> CliError { + trace::run(path, None, None, false, false, false, false, false).expect_err("trace must refuse") +} + +#[test] +fn every_hex_and_trace_parse_failure_names_the_format_not_apr() { + let dir = tempfile::tempdir().expect("tempdir"); + let d = dir.path(); + let gguf = truncated_gguf(d); + let apr = truncated_apr(d); + let overrun = write(d, "overrun.safetensors", &st_header_overrun()); + let not_utf8 = write(d, "not_utf8.safetensors", &st_header_not_utf8()); + let not_json = write(d, "not_json.safetensors", &st_header_not_json()); + // #3661's noise file: its first u64 is far past any header length, so no + // magic matches and trace reaches the SafeTensors reader by extension alone. + let noise = random_safetensors(d); + let opts = HexOptions::default(); + let array = st_header_array(); + + // (site, the error, the prefix the rendered line must start with) + let rows: Vec<(&str, CliError, &str)> = vec![ + ( + "hex.rs parse_gguf: hex on a truncated GGUF", + hex_err(&gguf), + "error: Invalid GGUF file: Failed to parse GGUF: ", + ), + ( + "hex.rs get_gguf_tensor_f32: hex --slice on a truncated GGUF", + hex_slice_err(&gguf), + "error: Invalid GGUF file: Failed to parse GGUF: ", + ), + ( + "safe_tensors_header.rs size check: 4 bytes, direct", + parse_safetensors_header(&[0u8; 4]) + .err() + .expect("4 bytes must refuse"), + "error: Invalid model file: SafeTensors file too small", + ), + ( + "safe_tensors_header.rs length check: hex on an overrunning header", + hex_err(&overrun), + "error: Invalid SafeTensors file: SafeTensors header length exceeds file size", + ), + ( + "safe_tensors_header.rs UTF-8: hex on a non-UTF-8 header", + hex_err(¬_utf8), + "error: Invalid SafeTensors file: Invalid SafeTensors header UTF-8: ", + ), + ( + "safe_tensors_header.rs JSON: hex on a cut-off header", + hex_err(¬_json), + "error: Invalid SafeTensors file: Invalid SafeTensors JSON: ", + ), + ( + "safe_tensors_header.rs JSON, the reader's second caller: hex --slice", + hex_slice_err(¬_json), + "error: Invalid SafeTensors file: Invalid SafeTensors JSON: ", + ), + ( + "safe_tensors_header.rs run_safetensors: an array header, direct", + run_safetensors(&opts, &array).expect_err("an array header must refuse"), + "error: Invalid model file: SafeTensors header is not a JSON object", + ), + ( + "safe_tensors_header.rs slice_safetensors: an array header, direct", + slice_safetensors(&opts, &array, "w", 0, 1).expect_err("an array header must refuse"), + "error: Invalid model file: SafeTensors header is not a JSON object", + ), + ( + "trace trace_gguf: trace on a truncated GGUF", + trace_err(&gguf), + "error: Invalid GGUF file: Failed to parse GGUF: ", + ), + ( + "trace trace_safetensors: trace on a cut-off header", + trace_err(¬_json), + "error: Invalid SafeTensors file: Failed to inspect SafeTensors: ", + ), + ( + "trace trace_safetensors: trace on noise named .safetensors", + trace_err(&noise), + "error: Invalid model file: Failed to inspect SafeTensors: ", + ), + // Control: hex on a truncated .apr still says APR (run_apr, unchanged), + // so the fix is not "delete the word". + ( + "control, hex run_apr: hex on a truncated APR", + hex_err(&apr), + "error: Invalid APR format: Failed to read APR: ", + ), + ]; + + let mut failures = Vec::new(); + for (row, err, want_prefix) in &rows { + let rendered = format!("error: {err}"); + let is_apr = want_prefix.starts_with("error: Invalid APR"); + if !rendered.starts_with(want_prefix) { + failures.push(format!("{row}: wanted `{want_prefix}…`, got `{rendered}`")); + } + if !is_apr && rendered.contains("Invalid APR") { + failures.push(format!("{row}: calls a non-APR file APR: `{rendered}`")); + } + if rendered.contains("Invalid model format") { + failures.push(format!( + "{row}: the core prefix leaked under the CLI's: `{rendered}`" + )); + } + if err.exit_code_value() != 4 { + failures.push(format!("{row}: exit {} (wanted 4)", err.exit_code_value())); + } + } + assert!( + failures.is_empty(), + "#3691 rows failed:\n{}", + failures.join("\n") + ); +} diff --git a/crates/apr-cli/src/commands/output_verification.rs b/crates/apr-cli/src/commands/output_verification.rs index fe5fe17a0c..adf367f376 100644 --- a/crates/apr-cli/src/commands/output_verification.rs +++ b/crates/apr-cli/src/commands/output_verification.rs @@ -468,10 +468,117 @@ pub fn strip_thinking_blocks(output: &str) -> String { result.trim().to_string() } +/// #3711: what the GPU half of one golden case came to. +/// +/// An ERROR is a verdict about the GPU, never a skip. It used to be a skip: CUDA +/// init or generation failing went to `note_gpu_golden_skip`, the case went on to +/// judge only the CPU answer, and the gate said "N golden test cases passed". So +/// on a host where CUDA generation was broken, `golden_output` read GREEN. That +/// was absence scored as conformance, on the one gate that has to prove every +/// Q4_K model works on CUDA. Only a leg that never STARTED is not run, and it +/// says why. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) enum GpuGoldenLeg { + /// Generated on the device, and the answer matches the golden patterns. + Passed, + /// Generated on the device, and the answer is wrong. + WrongAnswer(String), + /// CUDA init or generation ERRORED on a cuda build with a device. + Errored(String), + /// Never started, and why: no cuda feature, no device, not a GGUF, or judged + /// by the runtime rung. + NotRun(&'static str), +} + +impl GpuGoldenLeg { + /// Judge one GPU generation, given its decoded text or the error that stopped it. + pub(crate) fn judge( + generated: std::result::Result, + expected_patterns: &[&str], + ) -> Self { + match generated { + Err(e) => Self::Errored(e), + Ok(text) => match verify_output( + &strip_thinking_blocks(&text), // GH-279-4 + "golden_output_gpu", + expected_patterns, + ) { + OutputVerification::Pass => Self::Passed, + OutputVerification::Fail { reason } => Self::WrongAnswer(reason), + }, + } + } + + /// `Some(reason)` fails the golden gate: a wrong answer, or an error. + pub(crate) fn failure(&self) -> Option { + match self { + Self::Passed | Self::NotRun(_) => None, + Self::WrongAnswer(reason) => Some(format!("GPU output failed (CPU passed): {reason}")), + Self::Errored(e) => Some(format!( + "GPU golden generation ERRORED on a cuda build with a CUDA device: a broken \ + GPU, not a skip (#3711): {e}" + )), + } + } + + /// What the gate's pass message says about the GPU leg. + pub(crate) fn describe(&self) -> String { + match self { + Self::NotRun(why) => format!("GPU leg SKIPPED: {why}"), + _ => "GPU leg judged on CUDA device 0".to_string(), + } + } +} + +/// Why the dense GPU leg will not start, or `None` when it will (#3711). These are +/// the only skips, and each names which. +pub(crate) fn gpu_golden_not_run( + cuda_feature: bool, + cuda_device: bool, + gguf: bool, +) -> Option<&'static str> { + if !cuda_feature { + Some("this build has no cuda feature") + } else if !cuda_device { + Some("no CUDA device on this host") + } else if !gguf { + Some("the dense GPU leg judges GGUF only") + } else { + None + } +} + +/// #3711: the backend the runtime rung names is the one the dispatch REPORTED +/// (`InferenceResult::used_gpu`), never the one the build could have used. The +/// pass message used to say "GPU hybrid forward" on every cuda build, including +/// one on a host with no device. Worse, the hybrid's GPU forward falls back to the +/// CPU on any GPU failure, so a broken GPU passed on the CPU's answer. +/// +/// `gpu_not_run` is `None` when the GPU should serve the model: a cuda build, a +/// device, and an architecture the GPU runs. `Err` then means the dispatch +/// reported CPU, and it fails the gate. +pub(crate) fn runtime_golden_backend( + used_gpu: bool, + gpu_not_run: Option<&'static str>, +) -> std::result::Result { + match (used_gpu, gpu_not_run) { + (true, _) => Ok("GPU: the dispatch reported used_gpu=true".to_string()), + (false, Some(why)) => Ok(format!( + "CPU: the dispatch reported used_gpu=false, and the GPU was not expected ({why})" + )), + (false, None) => Err("the GPU should have served this model (a cuda build, a CUDA \ + device, an architecture the GPU runs) and the dispatch reported CPU: the GPU \ + forward failed and fell back, and its reason is on stderr. A broken GPU, not a \ + pass (#3711)" + .to_string()), + } +} + /// JIDOKA: Validate GPU golden output matches expected patterns (PMAT-232 lesson). /// /// Without this, GPU correctness was NEVER tested — `apr qa` golden output only ran CPU. -/// Returns `Some(failure_reason)` if GPU output fails, `None` if pass or skipped. +/// Called only when `gpu_golden_not_run` said the leg starts (a cuda build, a device, +/// a GGUF), so an init or generation error here is `Errored`, a gate FAIL (#3711). #[cfg_attr(coverage_nightly, coverage(off))] #[cfg(all(feature = "inference", feature = "cuda"))] fn validate_gpu_golden_output( @@ -481,7 +588,7 @@ fn validate_gpu_golden_output( gguf: &realizar::gguf::GGUFModel, expected_patterns: &[&str], config: &QaConfig, -) -> Result> { +) -> Result { use realizar::gguf::{OwnedQuantizedModel, OwnedQuantizedModelCuda}; // #3432 / #3477: the Qwen3.5 hybrid never reaches this dense-loader gate — // `golden_gate_for` routes it to `run_golden_output_gate_runtime`, which goes @@ -491,31 +598,21 @@ fn validate_gpu_golden_output( // gate, so the guard stays as a fail-safe and says where the GPU output IS // judged rather than claiming it is not. if realizar::gguf::hybrid_forward_handles(mapped.model.architecture().unwrap_or_default()) { - note_gpu_golden_skip( - config, - "GPU golden output for the Gated DeltaNet hybrid is judged by the runtime rung \ - (run_inference → Qwen35CudaModel, #3090), not by the dense loader", - ); - return Ok(None); + const JUDGED_BY_RUNTIME: &str = "the Gated DeltaNet hybrid's GPU output is judged by \ + the runtime rung (run_inference → Qwen35CudaModel, #3090), not by the dense loader"; + note_gpu_golden_skip(config, JUDGED_BY_RUNTIME); + return Ok(GpuGoldenLeg::NotRun(JUDGED_BY_RUNTIME)); } let model = OwnedQuantizedModel::from_mapped(mapped) .map_err(|e| CliError::ValidationFailed(format!("Model failed: {e}")))?; - match OwnedQuantizedModelCuda::new(model, 0) { - Ok(mut cuda_model) => match cuda_model.generate_gpu_resident(prompt_tokens, gen_config) { - Ok(gpu_tokens) => { - let gpu_text = gguf.decode(&gpu_tokens); - let gpu_answer = strip_thinking_blocks(&gpu_text); // GH-279-4 - if let OutputVerification::Fail { reason } = - verify_output(&gpu_answer, "golden_output_gpu", expected_patterns) - { - return Ok(Some(format!("GPU output failed (CPU passed): {reason}"))); - } - } - Err(e) => note_gpu_golden_skip(config, &format!("GPU golden output skipped: {e}")), - }, - Err(e) => note_gpu_golden_skip(config, &format!("CUDA init skipped: {e}")), - } - Ok(None) + let generated = match OwnedQuantizedModelCuda::new(model, 0) { + Ok(mut cuda_model) => cuda_model + .generate_gpu_resident(prompt_tokens, gen_config) + .map(|gpu_tokens| gguf.decode(&gpu_tokens)) + .map_err(|e| format!("GPU generation: {e}")), + Err(e) => Err(format!("CUDA init on device 0: {e}")), + }; + Ok(GpuGoldenLeg::judge(generated, expected_patterns)) } /// Note, in a verbose human-readable run, that the GPU half of the golden gate @@ -553,8 +650,10 @@ fn note_gpu_golden_skip(config: &QaConfig, message: &str) { /// via `with_input_tokens` to bypass `prepare_tokens`' chat-template auto-wrap /// (the same reason `golden_output_apr` does it). Stop tokens come from the /// model's own EOS, which `run_gguf_inference` merges in. +/// +/// Returns the text and the dispatch's own `used_gpu` (#3711). #[cfg(feature = "inference")] -fn golden_output_runtime(path: &Path, prompt: &str, max_tokens: usize) -> Result { +fn golden_output_runtime(path: &Path, prompt: &str, max_tokens: usize) -> Result<(String, bool)> { use realizar::gguf::MappedGGUFModel; use realizar::{run_inference, InferenceConfig}; @@ -575,7 +674,7 @@ fn golden_output_runtime(path: &Path, prompt: &str, max_tokens: usize) -> Result .with_top_k(1); let result = run_inference(&infer_config) .map_err(|e| CliError::ValidationFailed(format!("Generation failed: {e}")))?; - Ok(result.text) + Ok((result.text, result.used_gpu)) } /// Gate 1 for an architecture the dense loader refuses: the same golden cases, @@ -583,8 +682,16 @@ fn golden_output_runtime(path: &Path, prompt: &str, max_tokens: usize) -> Result /// /// This gate is the point of #3477: `apr qa` must be able to say PASS about the /// backend that runs the model — and since #3090 that backend is the GPU. +/// +/// `cpu_only`: the GPU backend declines this architecture, so the CPU is the +/// expected backend. Otherwise a cuda build with a device must be served by the +/// GPU, and a CPU fallback fails the gate (#3711). #[cfg(feature = "inference")] -fn run_golden_output_gate_runtime(path: &Path, config: &QaConfig) -> Result { +fn run_golden_output_gate_runtime( + path: &Path, + config: &QaConfig, + cpu_only: bool, +) -> Result { let start = Instant::now(); if !config.json && config.verbose { @@ -598,9 +705,28 @@ fn run_golden_output_gate_runtime(path: &Path, config: &QaConfig) -> Result... + the answer. let golden_max_tokens = config.max_tokens.max(512); + let gpu_not_run = if cpu_only { + Some("the GPU backend declines this architecture") + } else { + gpu_golden_not_run(cfg!(feature = "cuda"), cuda_device_present(), true) + }; + let mut served_by = String::new(); for (prompt, expected_patterns) in &test_cases { - let output_text = golden_output_runtime(path, prompt, golden_max_tokens)?; + let (output_text, used_gpu) = golden_output_runtime(path, prompt, golden_max_tokens)?; + // #3711: the backend first — a GPU that fell back is a FAIL even when the CPU's answer is right + match runtime_golden_backend(used_gpu, gpu_not_run) { + Ok(label) => served_by = label, + Err(failure) => { + return Ok(GateResult::failed( + "golden_output", + &failure, + None, + None, + start.elapsed(), + )) + } + } let answer_text = strip_thinking_blocks(&output_text); if let OutputVerification::Fail { reason } = verify_output(&answer_text, "golden_output_runtime", expected_patterns) @@ -618,13 +744,8 @@ fn run_golden_output_gate_runtime(path: &Path, config: &QaConfig) -> Result Result Result { - let _ = (path, config); +fn run_golden_output_gate_runtime( + path: &Path, + config: &QaConfig, + cpu_only: bool, +) -> Result { + let _ = (path, config, cpu_only); Ok(GateResult::skipped( "golden_output", "Requires 'inference' feature", diff --git a/crates/apr-cli/src/commands/qa_gguf.rs b/crates/apr-cli/src/commands/qa_gguf.rs index a9e727b00f..3e8e80778a 100644 --- a/crates/apr-cli/src/commands/qa_gguf.rs +++ b/crates/apr-cli/src/commands/qa_gguf.rs @@ -97,7 +97,7 @@ fn golden_gate_for( config: &QaConfig, ) -> Result { if cpu_only || hybrid { - run_golden_output_gate_runtime(path, config) + run_golden_output_gate_runtime(path, config, cpu_only) } else { run_golden_output_gate(path, config) } diff --git a/crates/apr-cli/src/commands/run.rs b/crates/apr-cli/src/commands/run.rs index f6d82433b0..9c43843e48 100644 --- a/crates/apr-cli/src/commands/run.rs +++ b/crates/apr-cli/src/commands/run.rs @@ -211,6 +211,25 @@ impl Default for RunOptions { } } +/// What `apr run --json` reports about the prompt and how generation ended (#3718). +/// +/// RAH (#3716) rebuilt Qwen3.5's tokenizer from the GGUF just to learn how many +/// prompt tokens the model saw, and could not tell a reply cut at `--max-tokens` +/// from one that finished. Every field is `None` when the inference path did not +/// report it, which is `null` in the JSON and never a zero that reads as measured. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub(crate) struct RunUsage { + /// Tokens actually fed to the model: after the chat template, including BOS. + pub prompt_tokens: Option, + /// Tokens the engine generated. Unlike `RunResult::tokens_generated`, never a + /// word-count stand-in. + pub completion_tokens: Option, + /// `"stop"` (a stop token ended it) or `"length"` (the budget did). + pub finish_reason: Option<&'static str>, + /// The model's context window, from its metadata. + pub context_length: Option, +} + /// Run result #[derive(Debug, Clone)] pub(crate) struct RunResult { @@ -234,6 +253,8 @@ pub(crate) struct RunResult { /// could be resolved. `--stream` used to emit `"text":""` for every token /// because nothing ever decoded the ids one at a time. pub token_texts: Option>, + /// Prompt and completion counts plus the finish reason (#3718). + pub usage: RunUsage, } /// Resolve a user-supplied model argument into a [`ModelSource`]. @@ -320,6 +341,7 @@ pub(crate) fn run_model(source: &str, options: &RunOptions) -> Result used_gpu: output.used_gpu, generated_tokens: output.generated_tokens, token_texts: output.token_texts, + usage: output.usage, }) } diff --git a/crates/apr-cli/src/commands/run_07.rs b/crates/apr-cli/src/commands/run_07.rs index 607f57955c..2f280e89a5 100644 --- a/crates/apr-cli/src/commands/run_07.rs +++ b/crates/apr-cli/src/commands/run_07.rs @@ -11,4 +11,5 @@ include!("run_tests_chrome_trace.rs"); include!("run_tests_layer_trace.rs"); include!("run_tests_stream_output.rs"); include!("run_tests_accel_reconcile.rs"); +include!("run_tests_usage_3718.rs"); } diff --git a/crates/apr-cli/src/commands/run_entry.rs b/crates/apr-cli/src/commands/run_entry.rs index 2ed50c2cb7..842cc0c968 100644 --- a/crates/apr-cli/src/commands/run_entry.rs +++ b/crates/apr-cli/src/commands/run_entry.rs @@ -530,6 +530,15 @@ fn build_final_json( "inference_time_ms": (result.duration_secs * 1000.0 * 100.0).round() / 100.0, "used_gpu": result.used_gpu.unwrap_or(false), "cached": result.cached, + // #3718: what the model read and why it stopped writing. `prompt_tokens` is + // the post-chat-template count (BOS included) fed to the model, so a + // consumer no longer rebuilds the tokenizer to learn it. `finish_reason` + // is "length" when the budget ended the reply, so a cut answer is never + // mistaken for a finished one. Each is null when the path did not report it. + "prompt_tokens": result.usage.prompt_tokens, + "completion_tokens": result.usage.completion_tokens, + "finish_reason": result.usage.finish_reason, + "context_length": result.usage.context_length, // #3602: `used_gpu: false` alone collapses two different outcomes — "no // accelerator was asked for" and "one was asked for, attempted, and // REFUSED at runtime". A consumer cannot tell a CPU run from a rejected diff --git a/crates/apr-cli/src/commands/run_tests_accel_reconcile.rs b/crates/apr-cli/src/commands/run_tests_accel_reconcile.rs index de7f96ad61..d0c5da3588 100644 --- a/crates/apr-cli/src/commands/run_tests_accel_reconcile.rs +++ b/crates/apr-cli/src/commands/run_tests_accel_reconcile.rs @@ -62,6 +62,7 @@ fn a_forced_accelerator_that_ran_on_cpu_is_refused() { used_gpu: Some(false), generated_tokens: Some(vec![9707]), token_texts: None, + usage: Default::default(), }; let err = reconcile_accelerator(true, &result) @@ -148,6 +149,7 @@ fn gpu_result(used_gpu: Option) -> RunResult { used_gpu, generated_tokens: Some(vec![9707]), token_texts: None, + usage: Default::default(), } } diff --git a/crates/apr-cli/src/commands/run_tests_chrome_trace.rs b/crates/apr-cli/src/commands/run_tests_chrome_trace.rs index 5eb2df77bc..55acf6e4f6 100644 --- a/crates/apr-cli/src/commands/run_tests_chrome_trace.rs +++ b/crates/apr-cli/src/commands/run_tests_chrome_trace.rs @@ -24,6 +24,7 @@ used_gpu: Some(false), generated_tokens: None, token_texts: None, + usage: Default::default(), }; let json = build_chrome_trace_events(&result, "model.gguf", 16, true); @@ -54,6 +55,7 @@ used_gpu: None, generated_tokens: None, token_texts: None, + usage: Default::default(), }; let json = build_chrome_trace_events(&result, "empty.gguf", 0, false); @@ -73,6 +75,7 @@ used_gpu: None, generated_tokens: None, token_texts: None, + usage: Default::default(), }; let json = build_chrome_trace_events(&result, "model.gguf", 10, false); @@ -92,6 +95,7 @@ used_gpu: Some(true), generated_tokens: None, token_texts: None, + usage: Default::default(), }; let json = build_chrome_trace_events(&result, "my-model.gguf", 64, true); @@ -111,6 +115,7 @@ used_gpu: Some(false), generated_tokens: None, token_texts: None, + usage: Default::default(), }; let json = build_chrome_trace_events(&result, "model.gguf", 10, false); @@ -140,6 +145,7 @@ used_gpu: Some(false), generated_tokens: None, token_texts: None, + usage: Default::default(), }; let json = build_chrome_trace_events(&result, "model.gguf", 10, false); @@ -159,6 +165,7 @@ used_gpu: None, generated_tokens: None, token_texts: None, + usage: Default::default(), }; let json = build_chrome_trace_events(&result, "m.gguf", 1, false); assert_eq!(json["displayTimeUnit"], "ms"); @@ -178,6 +185,7 @@ used_gpu: Some(false), generated_tokens: None, token_texts: None, + usage: Default::default(), }; // Just ensure no panic; file creation is best-effort print_chrome_trace(&result, "test-model.gguf", 32, false, None); @@ -210,6 +218,7 @@ used_gpu: Some(false), generated_tokens: None, token_texts: None, + usage: Default::default(), }; print_chrome_trace(&result, "test-model.gguf", 32, false, Some(&target)); @@ -239,6 +248,7 @@ used_gpu: Some(false), generated_tokens: None, token_texts: None, + usage: Default::default(), }; print_benchmark_results(&result, "model.gguf", "text", 100); } @@ -254,6 +264,7 @@ used_gpu: Some(false), generated_tokens: None, token_texts: None, + usage: Default::default(), }; print_benchmark_results(&result, "model.gguf", "json", 50); } @@ -269,6 +280,7 @@ used_gpu: None, generated_tokens: None, token_texts: None, + usage: Default::default(), }; print_benchmark_results(&result, "model.gguf", "text", 10); } diff --git a/crates/apr-cli/src/commands/run_tests_inference_output.rs b/crates/apr-cli/src/commands/run_tests_inference_output.rs index 10caee8089..0174ecadf6 100644 --- a/crates/apr-cli/src/commands/run_tests_inference_output.rs +++ b/crates/apr-cli/src/commands/run_tests_inference_output.rs @@ -15,6 +15,7 @@ used_gpu: Some(false), generated_tokens: Some(vec![1, 2, 3, 4, 5]), token_texts: None, + usage: Default::default(), }; assert_eq!(output.text, "hello"); assert_eq!(output.tokens_generated, Some(5)); @@ -32,6 +33,7 @@ used_gpu: None, generated_tokens: None, token_texts: None, + usage: Default::default(), }; assert!(output.tokens_generated.is_none()); assert!(output.inference_ms.is_none()); diff --git a/crates/apr-cli/src/commands/run_tests_layer_trace.rs b/crates/apr-cli/src/commands/run_tests_layer_trace.rs index e71ea0609d..6bf19f17ab 100644 --- a/crates/apr-cli/src/commands/run_tests_layer_trace.rs +++ b/crates/apr-cli/src/commands/run_tests_layer_trace.rs @@ -20,6 +20,7 @@ used_gpu: Some(false), generated_tokens: None, token_texts: None, + usage: Default::default(), } } @@ -52,8 +53,11 @@ } /// The run total and the decode rate must be labelled, or the table - /// contradicts the profiler block printed a few lines above it (1.0 tok/s - /// vs 19.2 tok/s for the identical run, neither labelled). + /// contradicts the profiler block printed a few lines above it: the same + /// run reports an end-to-end rate and a decode rate that differ by more + /// than an order of magnitude, and neither one says which it is. The + /// assertions below are the check; no rate literal belongs in this + /// comment, because a number here is a claim no measurement resolves. #[test] fn layer_trace_labels_wall_clock_and_end_to_end_rate() { let out = render_layer_trace(&layer_trace_result(2.0, 4), 4); diff --git a/crates/apr-cli/src/commands/run_tests_model_source.rs b/crates/apr-cli/src/commands/run_tests_model_source.rs index 3779970053..cb8bdd5322 100644 --- a/crates/apr-cli/src/commands/run_tests_model_source.rs +++ b/crates/apr-cli/src/commands/run_tests_model_source.rs @@ -39,6 +39,7 @@ used_gpu: None, generated_tokens: None, token_texts: None, + usage: Default::default(), }; let _debug = format!("{:?}", result); assert_eq!(result.tokens_generated, Some(5)); @@ -191,6 +192,7 @@ used_gpu: None, generated_tokens: None, token_texts: None, + usage: Default::default(), }; let cloned = result.clone(); assert_eq!(result.text, cloned.text); diff --git a/crates/apr-cli/src/commands/run_tests_parse_token_ids.rs b/crates/apr-cli/src/commands/run_tests_parse_token_ids.rs index 32e2045b48..7977032b32 100644 --- a/crates/apr-cli/src/commands/run_tests_parse_token_ids.rs +++ b/crates/apr-cli/src/commands/run_tests_parse_token_ids.rs @@ -232,6 +232,7 @@ used_gpu: None, generated_tokens: None, token_texts: None, + usage: Default::default(), }; let result_zero = RunResult { text: String::new(), @@ -242,6 +243,7 @@ used_gpu: None, generated_tokens: None, token_texts: None, + usage: Default::default(), }; assert_ne!( result_none.tokens_generated, result_zero.tokens_generated, @@ -262,6 +264,7 @@ used_gpu: Some(true), generated_tokens: Some(vec![10, 20, 30]), token_texts: None, + usage: Default::default(), }; assert_eq!(result.text, "output"); assert!((result.duration_secs - 1.234).abs() < f64::EPSILON); diff --git a/crates/apr-cli/src/commands/run_tests_stream_output.rs b/crates/apr-cli/src/commands/run_tests_stream_output.rs index e6062f9a66..20bc6952b3 100644 --- a/crates/apr-cli/src/commands/run_tests_stream_output.rs +++ b/crates/apr-cli/src/commands/run_tests_stream_output.rs @@ -16,6 +16,7 @@ fn stream_output_emits_n_plus_one_json_lines() { " world".to_string(), "!".to_string(), ]), + usage: Default::default(), }; let mut buf: Vec = Vec::new(); @@ -85,6 +86,7 @@ fn stream_token_events_carry_their_own_decoded_text() { " here".to_string(), " to help".to_string(), ]), + usage: Default::default(), }; let mut buf: Vec = Vec::new(); @@ -125,6 +127,7 @@ fn stream_token_events_degrade_to_empty_text_without_a_tokenizer() { used_gpu: Some(false), generated_tokens: Some(vec![7, 9]), token_texts: None, + usage: Default::default(), }; let mut buf: Vec = Vec::new(); @@ -150,6 +153,7 @@ fn stream_output_no_tokens_emits_only_final() { used_gpu: Some(false), generated_tokens: Some(Vec::new()), token_texts: None, + usage: Default::default(), }; let mut buf: Vec = Vec::new(); @@ -175,6 +179,7 @@ fn stream_output_none_tokens_emits_only_final() { used_gpu: None, generated_tokens: None, token_texts: None, + usage: Default::default(), }; let mut buf: Vec = Vec::new(); @@ -199,6 +204,7 @@ fn build_final_json_matches_legacy_json_shape() { used_gpu: Some(true), generated_tokens: Some(vec![1, 2, 3]), token_texts: None, + usage: Default::default(), }; let v = build_final_json(&result, "src.apr", 100, false); assert_eq!(v["model"], "src.apr"); diff --git a/crates/apr-cli/src/commands/run_tests_usage_3718.rs b/crates/apr-cli/src/commands/run_tests_usage_3718.rs new file mode 100644 index 0000000000..d3ed93b341 --- /dev/null +++ b/crates/apr-cli/src/commands/run_tests_usage_3718.rs @@ -0,0 +1,64 @@ +// #3718: `apr run --json` says how many tokens the model read and why it stopped. + +fn result_with(usage: RunUsage) -> RunResult { + RunResult { + text: "4".to_string(), + duration_secs: 1.0, + cached: true, + // The word-count stand-in `run_model` substitutes when the engine + // reports nothing. `completion_tokens` must never echo it. + tokens_generated: Some(1), + tok_per_sec: None, + used_gpu: Some(false), + generated_tokens: None, + token_texts: None, + usage, + } +} + +#[test] +fn json_carries_the_engine_counts_and_the_finish() { + let usage = RunUsage { + prompt_tokens: Some(79), + completion_tokens: Some(32), + finish_reason: Some("length"), + context_length: Some(262_144), + }; + let v = build_final_json(&result_with(usage), "m.gguf", 32, false); + assert_eq!(v["prompt_tokens"], 79); + assert_eq!(v["completion_tokens"], 32); + assert_eq!(v["finish_reason"], "length"); + assert_eq!(v["context_length"], 262_144); +} + +/// Not reported is `null`, never `0` or `"stop"`: a zero reads as measured and +/// a default "stop" is the silent cut this row removes. +#[test] +fn unreported_fields_are_null_not_zero() { + let v = build_final_json(&result_with(RunUsage::default()), "m.apr", 32, false); + for key in ["prompt_tokens", "completion_tokens", "finish_reason", "context_length"] { + assert!(v.get(key).is_some(), "{key} must be present so consumers can key on it"); + assert!(v[key].is_null(), "{key} must be null when unreported, got {}", v[key]); + } + assert_eq!(v["tokens_generated"], 1, "the legacy field keeps its stand-in"); +} + +/// `--stream`'s terminal event is built by the same function, so it carries +/// the same fields. +#[test] +fn stream_final_event_carries_them_too() { + let usage = RunUsage { + prompt_tokens: Some(12), + completion_tokens: Some(3), + finish_reason: Some("stop"), + context_length: Some(4096), + }; + let mut buf: Vec = Vec::new(); + write_stream_output(&mut buf, &result_with(usage), "m.gguf", 8, false).expect("write"); + let text = String::from_utf8(buf).expect("utf-8"); + let last = text.lines().last().expect("final line"); + let v: serde_json::Value = serde_json::from_str(last).expect("json"); + assert_eq!(v["event"], "final"); + assert_eq!(v["prompt_tokens"], 12); + assert_eq!(v["finish_reason"], "stop"); +} diff --git a/crates/apr-cli/src/commands/safe_tensors_header.rs b/crates/apr-cli/src/commands/safe_tensors_header.rs index c0ad96abd3..8294b78866 100644 --- a/crates/apr-cli/src/commands/safe_tensors_header.rs +++ b/crates/apr-cli/src/commands/safe_tensors_header.rs @@ -163,23 +163,26 @@ struct SafeTensorsHeader { } fn parse_safetensors_header(bytes: &[u8]) -> Result { + // #3691: every refusal here names the format the magic bytes identify. + // It used to be `InvalidFormat`, which printed "Invalid APR format: …". if bytes.len() < 9 { - return Err(CliError::InvalidFormat( - "SafeTensors file too small".to_string(), - )); + return Err(CliError::invalid_model_bytes(bytes, "SafeTensors file too small")); } let header_len = u64::from_le_bytes([ bytes[0], bytes[1], bytes[2], bytes[3], bytes[4], bytes[5], bytes[6], bytes[7], ]) as usize; if 8 + header_len > bytes.len() { - return Err(CliError::InvalidFormat( - "SafeTensors header length exceeds file size".to_string(), + return Err(CliError::invalid_model_bytes( + bytes, + "SafeTensors header length exceeds file size", )); } - let header_json = std::str::from_utf8(&bytes[8..8 + header_len]) - .map_err(|e| CliError::InvalidFormat(format!("Invalid SafeTensors header UTF-8: {e}")))?; - let header: serde_json::Value = serde_json::from_str(header_json) - .map_err(|e| CliError::InvalidFormat(format!("Invalid SafeTensors JSON: {e}")))?; + let header_json = std::str::from_utf8(&bytes[8..8 + header_len]).map_err(|e| { + CliError::invalid_model_bytes(bytes, format!("Invalid SafeTensors header UTF-8: {e}")) + })?; + let header: serde_json::Value = serde_json::from_str(header_json).map_err(|e| { + CliError::invalid_model_bytes(bytes, format!("Invalid SafeTensors JSON: {e}")) + })?; Ok(SafeTensorsHeader { header_len, header }) } @@ -189,7 +192,7 @@ fn run_safetensors(opts: &HexOptions, bytes: &[u8]) -> Result<(), CliError> { let header_len = parsed.header_len; let tensor_map = parsed.header.as_object().ok_or_else(|| { - CliError::InvalidFormat("SafeTensors header is not a JSON object".to_string()) + CliError::invalid_model_bytes(bytes, "SafeTensors header is not a JSON object") })?; let tensor_names: Vec<&String> = tensor_map.keys().filter(|k| *k != "__metadata__").collect(); @@ -415,7 +418,7 @@ fn slice_safetensors( ) -> Result<(), CliError> { let parsed = parse_safetensors_header(bytes)?; let tensor_map = parsed.header.as_object().ok_or_else(|| { - CliError::InvalidFormat("SafeTensors header is not a JSON object".to_string()) + CliError::invalid_model_bytes(bytes, "SafeTensors header is not a JSON object") })?; let info = tensor_map.get(tensor_name).ok_or_else(|| { diff --git a/crates/apr-cli/src/commands/serve/handler_apr_cpu_completion.rs b/crates/apr-cli/src/commands/serve/handler_apr_cpu_completion.rs index e4651ca2f8..0cc52d93d0 100644 --- a/crates/apr-cli/src/commands/serve/handler_apr_cpu_completion.rs +++ b/crates/apr-cli/src/commands/serve/handler_apr_cpu_completion.rs @@ -376,7 +376,7 @@ async fn handle_apr_cpu_chat_completion( let latency_ms = start.elapsed().as_millis() as u64; let mut response = serde_json::json!({ "id": request_id, "object": "chat.completion", "created": created, "model": &s.model_name, - "choices": [{"index": 0, "message": {"role": "assistant", "content": out.text}, "finish_reason": "stop"}], + "choices": [{"index": 0, "message": {"role": "assistant", "content": out.text}, "finish_reason": out.finish_reason.as_str()}], "usage": {"prompt_tokens": out.input_token_count, "completion_tokens": out.tokens_generated, "total_tokens": out.input_token_count + out.tokens_generated}, "_apr_metrics": {"latency_ms": latency_ms, "tok_per_sec": tok_per_sec} }); diff --git a/crates/apr-cli/src/commands/serve/handler_gpu_completion.rs b/crates/apr-cli/src/commands/serve/handler_gpu_completion.rs index 07e390a622..2769e6999b 100644 --- a/crates/apr-cli/src/commands/serve/handler_gpu_completion.rs +++ b/crates/apr-cli/src/commands/serve/handler_gpu_completion.rs @@ -387,25 +387,7 @@ fn start_gguf_server(model_path: &Path, config: &ServerConfig) -> Result<()> { ); println!("{}", "Building quantized inference model...".dimmed()); - // #3571: the Qwen3.5 hybrid (Gated Delta Net) has no dense layers, so `from_mapped` refuses - // it by name and `apr serve` could not load the architecture the last release shipped — while - // `apr run` loaded it fine, because `run_gguf_inference` has carried exactly this branch since - // #3091. The routing gap was in ONE verb, and it was invisible because every gate we own is - // single-stream `apr run`: no ladder rung, parity record or dogfood row has ever asked - // `apr serve` to load a model (#3555). - let is_qwen35 = mapped_model.model.architecture() == Some("qwen35"); - let quantized_model = if is_qwen35 { - realizar::gguf::forward_qwen35::Qwen35Model::create_base_model( - &mapped_model.model, - mapped_model.data(), - ) - .map_err(|e| { - CliError::ModelLoadFailed(format!("Failed to build the Qwen3.5 base model: {e}")) - })? - } else { - OwnedQuantizedModel::from_mapped(&mapped_model) - .map_err(|e| CliError::ModelLoadFailed(format!("Failed to build quantized model: {e}")))? - }; + let quantized_model = build_serve_model(&mapped_model)?; println!( "{}", @@ -463,6 +445,68 @@ fn start_gguf_server(model_path: &Path, config: &ServerConfig) -> Result<()> { run_cpu_server(quantized_model, vocab, Some(mapped_model), config, Some(offload)) } +/// Build the model `apr serve` hands its routes, or refuse it (#3571). +/// +/// Every GGUF serve route — `--gpu`, `--gpu --batch`, the CPU server — takes the model this +/// returns, so the zero-layer refusal here is the one no route can walk around. +fn build_serve_model( + mapped_model: &realizar::gguf::MappedGGUFModel, +) -> Result { + use realizar::gguf::OwnedQuantizedModel; + // #3571: the Qwen3.5 hybrid (Gated Delta Net) has no dense layers, so `from_mapped` refuses + // it by name and `apr serve` could not load the architecture the last release shipped — while + // `apr run` loaded it fine, because `run_gguf_inference` has carried exactly this branch since + // #3091. The routing gap was in ONE verb, and it was invisible because every gate we own is + // single-stream `apr run`: no ladder rung, parity record or dogfood row has ever asked + // `apr serve` to load a model (#3555). + let is_qwen35 = mapped_model.model.architecture() == Some("qwen35"); + let quantized_model = if is_qwen35 { + realizar::gguf::forward_qwen35::Qwen35Model::create_base_model( + &mapped_model.model, + mapped_model.data(), + ) + .map_err(|e| { + CliError::ModelLoadFailed(format!("Failed to build the Qwen3.5 base model: {e}")) + })? + } else { + OwnedQuantizedModel::from_mapped(mapped_model) + .map_err(|e| CliError::ModelLoadFailed(format!("Failed to build quantized model: {e}")))? + }; + // #3571: before ANY route is chosen — --gpu, --gpu --batch or the CPU server — so no + // request handler can be handed a stack with nothing in it. + if let Some(refusal) = zero_layer_refusal( + &quantized_model.config().architecture, + quantized_model.layers().len(), + ) { + return Err(refusal); + } + Ok(quantized_model) +} + +/// #3571: a model whose decoder stack resolved to zero layers is refused at load, whatever +/// its architecture, on every serve route. +/// +/// Measured on #3571: the Qwen3.5 hybrid reached the servers as its BASE — embeddings, final +/// norm and `lm_head`, no layers, because its Gated-DeltaNet and attention layers live in +/// `Qwen35Model`, which no serve handler calls. With the `` refusal removed (#3609) the +/// CPU route answered HTTP 200 with 1024 tokens of `"\n"` in 6 s and the GPU route HTTP 500. +/// A stack with nothing in it has no answer to give, so the server does not start: an error +/// at load is the one a user can act on, and a 200 of nothing is the one they cannot see. +pub(super) fn zero_layer_refusal(architecture: &str, layers: usize) -> Option { + (layers == 0).then(|| { + let hybrid = if architecture == "qwen35" { + " The Qwen3.5 hybrid's layers are served by `apr run` and `apr chat` today." + } else { + "" + }; + CliError::ModelLoadFailed(format!( + "'{architecture}' resolved to 0 transformer layers, so no serve route can answer \ + from it — it would decode through the embeddings and lm_head alone. Refused at \ + load (#3571).{hybrid}" + )) + }) +} + /// Extract the vocabulary from a GGUF model, or refuse to serve it (#3609). /// /// GH-226 once filled a missing `tokenizer.ggml.tokens` with `token0..tokenN` and put @@ -649,9 +693,8 @@ fn start_gguf_server_cuda( "{}", format!("CUDA init failed, falling back to CPU: {e}").yellow() ); - let quantized_model = OwnedQuantizedModel::from_mapped(&mapped_model).map_err(|e| { - CliError::ModelLoadFailed(format!("Failed to rebuild quantized model: {e}")) - })?; + // #3571: the rebuild goes through the one GGUF serve loader and its refusal. + let quantized_model = build_serve_model(&mapped_model)?; let vocab = extract_gguf_vocab(&mapped_model)?; // CUDA init failed and this process fell back to CPU. The offload // report travels with it UNCHANGED, so `/v1/effective-config` shows @@ -730,3 +773,53 @@ mod ctx_length_2762_tests { assert_eq!(resolve_serve_max_seq_len(Some("0"), None), 2048); } } + +// The zero-layer refusal tests sit at the END, after every shipping item: the context-length +// guard (serve/mod.rs, accelerator_guard_tests) scans this file up to its FIRST `#[cfg(test)]`, +// and a test module above the REALIZR_CONTEXT_LENGTH read hid the read from it (batch-1 fold). +#[cfg(test)] +mod zero_layer_refusal_tests { + use super::*; + + /// The case table: zero layers is refused naming the architecture and #3571; any layer + /// at all is admitted. + #[test] + fn zero_layers_is_refused_by_name_and_any_layer_is_admitted() { + let cases: [(&str, usize, bool); 5] = [ + ("qwen35", 0, true), + ("llama", 0, true), + ("qwen2", 1, false), + ("qwen3", 28, false), + ("qwen35", 24, false), + ]; + for (arch, layers, refused) in cases { + let got = zero_layer_refusal(arch, layers); + assert_eq!(got.is_some(), refused, "({arch}, {layers})"); + if let Some(CliError::ModelLoadFailed(msg)) = got { + assert!(msg.contains(&format!("'{arch}' resolved to 0 transformer layers")), "{msg}"); + assert!(msg.contains("#3571"), "{msg}"); + assert_eq!(msg.contains("apr chat"), arch == "qwen35", "the hint is the hybrid's: {msg}"); + } + } + } + + /// The load path itself refuses, before any route is chosen. Delete the check in + /// `build_serve_model` and this goes RED: the zero-layer base loads. Needs the real + /// Qwen3.5 file, whose base IS a zero-layer stack. + #[test] + fn serve_refuses_the_qwen35_base_at_load() { + const MODEL: &str = "/home/noah/models/Qwen3.5-0.8B-Q4_K_M.gguf"; + if !Path::new(MODEL).exists() { + eprintln!("SKIP: {MODEL} is absent"); + return; + } + let mapped = realizar::gguf::MappedGGUFModel::from_path(MODEL).expect("map the GGUF"); + match build_serve_model(&mapped) { + Err(CliError::ModelLoadFailed(msg)) => { + assert!(msg.contains("'qwen35' resolved to 0 transformer layers"), "{msg}"); + } + Err(other) => panic!("refused for the wrong reason: {other}"), + Ok(model) => panic!("a {}-layer stack reached the serve routes", model.layers().len()), + } + } +} diff --git a/crates/apr-cli/src/commands/serve/handlers.rs b/crates/apr-cli/src/commands/serve/handlers.rs index ded665a5b4..5e696eaeef 100644 --- a/crates/apr-cli/src/commands/serve/handlers.rs +++ b/crates/apr-cli/src/commands/serve/handlers.rs @@ -927,11 +927,11 @@ fn try_start_wgpu_backend(model_path: &Path, config: &ServerConfig) -> Result input_tokens.len() { - &output_tokens[input_tokens.len()..] - } else { - &output_tokens[..] - }; + // Extract new tokens. `generate_with_cache` returns prompt + generated, so + // nothing past the prompt means nothing was generated. The old fallback + // (`&output_tokens[..]`) handed the PROMPT back as the completion, decoded it + // as the reply, and counted it as `completion_tokens` (#3718). + let new_tokens = output_tokens.get(input_tokens.len()..).unwrap_or(&[]); // Decode: embedded APR tokenizer → sibling tokenizer.json → character-level fallback let text = if let Some(ref tok) = state.embedded_tokenizer { @@ -1139,11 +1140,19 @@ fn run_apr_cpu_inference( .collect() }; + // #3718: the chat handler hardcoded "stop", so a reply cut at `max_tokens` + // (which the handler caps at 4096) read as finished. The loop's stop set is + // `gen_config.stop_tokens` (empty here) plus token 0, which it pushes before + // breaking (`is_eos_token`, apr_transformer/generation.rs). + let finish_reason = + realizar::infer::run_report::FinishReason::from_decode(new_tokens, &[0], max_tokens); + Ok(AprInferenceOutput { text, tokens_generated: new_tokens.len(), gen_duration, input_token_count, + finish_reason, }) } @@ -1224,6 +1233,12 @@ fn load_apr_model_state(model_path: &Path, config: &ServerConfig) -> Result { + // #3571: a stack with no layers has no answer to give — refused at load. + if let Some(refusal) = + zero_layer_refusal(&t.config.architecture, t.config.num_layers) + { + return Err(refusal); + } println!( "{}", format!( @@ -1424,6 +1439,12 @@ fn try_apr_quantized_cpu(model_path: &Path, config: &ServerConfig) -> Result<()> let quantized = OwnedQuantizedModel::from_apr(&mapped) .map_err(|e| CliError::InferenceFailed(format!("Failed to create quantized model: {e}")))?; + // #3571: a stack with no layers has no answer to give — refused at load, as on every route. + if let Some(refusal) = + zero_layer_refusal(&quantized.config().architecture, quantized.layers().len()) + { + return Err(refusal); + } println!( "{}", diff --git a/crates/apr-cli/src/commands/serve/handlers_include_01.rs b/crates/apr-cli/src/commands/serve/handlers_include_01.rs index c03979450b..095c0aa348 100644 --- a/crates/apr-cli/src/commands/serve/handlers_include_01.rs +++ b/crates/apr-cli/src/commands/serve/handlers_include_01.rs @@ -38,6 +38,13 @@ fn start_apr_server_gpu( let quantized = OwnedQuantizedModel::from_apr(&mapped) .map_err(|e| CliError::InferenceFailed(format!("Failed to create quantized model: {e}")))?; + // #3571: a stack with no layers has no answer to give — refused at load, as on every route. + if let Some(refusal) = + zero_layer_refusal(&quantized.config().architecture, quantized.layers().len()) + { + return Err(refusal); + } + println!( "{}", format!( @@ -130,6 +137,19 @@ fn start_apr_q4k_server_gpu( println!(" EOS token ID: {eos}"); } + // #3571: the pool path reads its layer count on its own thread, after the upload has + // begun. Read the count the file declares first, so a zero-layer APR is refused at load + // like every other route. A file that declares none is left to the pool path, which + // refuses a missing `num_layers` by name (`parse_apr_q4k_config`). + if let Ok(apr) = AprV2Model::load(model_path) { + let meta = apr.metadata(); + if let Some(refusal) = meta.num_layers.and_then(|layers| { + zero_layer_refusal(meta.architecture.as_deref().unwrap_or("apr"), layers) + }) { + return Err(refusal); + } + } + // Spawn Q4K inference thread (loads model, uploads weights to GPU via pool allocator) let q4k_tx = apr_q4k_scheduler::spawn_apr_q4k_inference_thread(&model_str) .map_err(|e| CliError::InferenceFailed(format!("Q4K inference thread failed: {e}")))?; @@ -179,6 +199,13 @@ fn start_safetensors_server_gpu( let quantized = OwnedQuantizedModel::from_apr(&mapped) .map_err(|e| CliError::InferenceFailed(format!("Failed to create quantized model: {e}")))?; + // #3571: a stack with no layers has no answer to give — refused at load, as on every route. + if let Some(refusal) = + zero_layer_refusal(&quantized.config().architecture, quantized.layers().len()) + { + return Err(refusal); + } + println!( "{}", format!( @@ -255,6 +282,13 @@ fn start_safetensors_server_cpu_quantized( let quantized = OwnedQuantizedModel::from_apr(&mapped) .map_err(|e| CliError::InferenceFailed(format!("Failed to create quantized model: {e}")))?; + // #3571: a stack with no layers has no answer to give — refused at load, as on every route. + if let Some(refusal) = + zero_layer_refusal(&quantized.config().architecture, quantized.layers().len()) + { + return Err(refusal); + } + println!( "{}", format!( diff --git a/crates/apr-cli/src/commands/serve/safetensors.rs b/crates/apr-cli/src/commands/serve/safetensors.rs index 51d5a05e4f..60bcde2c78 100644 --- a/crates/apr-cli/src/commands/serve/safetensors.rs +++ b/crates/apr-cli/src/commands/serve/safetensors.rs @@ -80,6 +80,12 @@ pub(crate) fn start_safetensors_server(model_path: &Path, config: &ServerConfig) // Try to convert to AprTransformer for inference (PAR-301) let transformer = match SafetensorsToAprConverter::convert(model_path) { Ok(t) => { + // #3571: a stack with no layers has no answer to give — refused at load. + if let Some(refusal) = + super::handlers::zero_layer_refusal(&t.config.architecture, t.config.num_layers) + { + return Err(refusal); + } println!( "{}", format!( @@ -275,6 +281,12 @@ pub(crate) fn start_sharded_safetensors_server( // Convert to AprTransformer let transformer = match SafetensorsToAprConverter::convert_sharded(&sharded, &st_config) { Ok(t) => { + // #3571: a stack with no layers has no answer to give — refused at load. + if let Some(refusal) = + super::handlers::zero_layer_refusal(&t.config.architecture, t.config.num_layers) + { + return Err(refusal); + } println!( "{}", format!( diff --git a/crates/apr-cli/src/commands/tokenize_encode.rs b/crates/apr-cli/src/commands/tokenize_encode.rs new file mode 100644 index 0000000000..79391c81ad --- /dev/null +++ b/crates/apr-cli/src/commands/tokenize_encode.rs @@ -0,0 +1,143 @@ +//! `apr tokenize encode` (#3726): the token ids a GGUF model's own tokenizer gives a text. +//! +//! The ids come from `GGUFModel::encode`, the function `apr run`, `apr serve` and +//! `apr parity` tokenize prompts with, so a regression in how that function routes a +//! vocabulary shows up here too. The output names the path that produced the ids: the +//! canonical byte-level BPE (pre-tokenizer plus ranked merges, identical to llama.cpp), or the +//! greedy longest-match fallback and why it was taken. `scripts/tokenizer_parity.sh` compares +//! these ids with the pinned llama.cpp's and refuses a fallback. +//! +//! Only the file's metadata is read (the GGUF is memory-mapped); no weights are loaded. + +use std::path::Path; + +use crate::error::CliError; + +type Result = std::result::Result; + +/// Run `apr tokenize encode MODEL (-p TEXT | -f FILE)`. +pub(crate) fn run_encode( + model: &Path, + prompt: Option<&str>, + file: Option<&Path>, + json_output: bool, +) -> Result<()> { + let text = match (prompt, file) { + (Some(p), None) => p.to_string(), + (None, Some(f)) => std::fs::read_to_string(f) + .map_err(|e| CliError::ValidationFailed(format!("cannot read {}: {e}", f.display())))?, + _ => { + return Err(CliError::ValidationFailed( + "give exactly one of -p/--prompt TEXT or -f/--file FILE".to_string(), + )) + } + }; + if !model.exists() { + return Err(CliError::FileNotFound(model.to_path_buf())); + } + let mapped = realizar::gguf::MappedGGUFModel::from_path(model).map_err(|e| { + CliError::ValidationFailed(format!("{} is not a readable GGUF: {e}", model.display())) + })?; + let meta_str = |key: &str| match mapped.model.metadata.get(key) { + Some(realizar::gguf::GGUFValue::String(s)) => Some(s.clone()), + _ => None, + }; + let tokenizer_model = meta_str("tokenizer.ggml.model"); + let tokenizer_pre = meta_str("tokenizer.ggml.pre"); + let vocab = mapped.model.vocabulary().ok_or_else(|| { + CliError::ValidationFailed(format!( + "{} has no tokenizer.ggml.tokens: there is no tokenizer to run", + model.display() + )) + })?; + + // Which path GGUFModel::encode takes for this file. It is asked of the same function + // encode consults, so the label cannot disagree with the ids. + let byte_level = matches!(tokenizer_model.as_deref(), Some("gpt2" | "bpe")); + let path = if byte_level { + match realizar::gguf::byte_level_bpe::ByteLevelBpe::from_gguf( + &mapped.model.metadata, + &vocab, + ) { + Ok(_) => "canonical".to_string(), + Err(refusal) => format!("greedy-fallback: {refusal}"), + } + } else { + format!( + "greedy-fallback: tokenizer.ggml.model is {:?}, not a byte-level vocabulary", + tokenizer_model.as_deref().unwrap_or("absent") + ) + }; + + let ids = mapped.model.encode(&text).ok_or_else(|| { + CliError::ValidationFailed(format!( + "{}: the tokenizer returned nothing", + model.display() + )) + })?; + + // decode(encode(x)) == x: the byte-level map loses nothing (#3726 done_when 1). + let roundtrip = mapped.model.decode(&ids) == text; + + if json_output { + let out = serde_json::json!({ + "model": model.display().to_string(), + "tokenizer_model": tokenizer_model, + "tokenizer_pre": tokenizer_pre, + "path": path, + "count": ids.len(), + "roundtrip": roundtrip, + "ids": ids, + }); + println!("{out}"); + } else { + let joined: Vec = ids.iter().map(u32::to_string).collect(); + println!("{}", joined.join(" ")); + eprintln!("{} ids; path: {path}; roundtrip: {roundtrip}", ids.len()); + } + Ok(()) +} + +#[cfg(test)] +mod tests { + /// The first quorum round on #3726 found `Encode` spliced into the middle of + /// `EncodeCorpus`'s doc comment: `apr tokenize --help` described `encode` as "Encode a + /// JSONL corpus into .bin shards" and `encode-corpus` lost its summary. Each subcommand + /// must carry its own. (The clap tree is deep; it is built on a 16 MiB stack, as the + /// other `Cli::command()` tests do.) + #[test] + fn tokenize_help_gives_encode_and_encode_corpus_their_own_summaries() { + let (encode, corpus) = std::thread::Builder::new() + .stack_size(16 * 1024 * 1024) + .spawn(|| { + use clap::CommandFactory; + let cli = crate::Cli::command(); + let tokenize = cli + .find_subcommand("tokenize") + .expect("apr tokenize exists"); + let about = |name: &str| { + tokenize + .find_subcommand(name) + .and_then(|c| c.get_about()) + .map(ToString::to_string) + }; + (about("encode"), about("encode-corpus")) + }) + .expect("spawn") + .join() + .expect("clap tree builds"); + let encode = encode.expect("apr tokenize encode exists"); + assert!(encode.contains("token ids"), "encode's summary: {encode:?}"); + assert!( + !encode.contains("JSONL"), + "encode took encode-corpus's summary: {encode:?}" + ); + // encode-corpus exists only with the `training` feature. + if let Some(corpus) = corpus { + assert!( + corpus.starts_with("Encode a JSONL corpus"), + "encode-corpus's summary: {corpus:?}" + ); + } + } +} diff --git a/crates/apr-cli/src/commands/trace_likely_has_repeated.rs b/crates/apr-cli/src/commands/trace_likely_has_repeated.rs index 1bac63db5e..c0f9caac06 100644 --- a/crates/apr-cli/src/commands/trace_likely_has_repeated.rs +++ b/crates/apr-cli/src/commands/trace_likely_has_repeated.rs @@ -264,7 +264,7 @@ fn trace_gguf(path: &Path, layer_filter: Option<&str>) -> Result) -> Result = report.tensors.iter().map(|t| t.name.as_str()).collect(); diff --git a/crates/apr-cli/src/dispatch_analysis.rs b/crates/apr-cli/src/dispatch_analysis.rs index 5acab7943c..7f776f9588 100644 --- a/crates/apr-cli/src/dispatch_analysis.rs +++ b/crates/apr-cli/src/dispatch_analysis.rs @@ -1132,6 +1132,17 @@ fn dispatch_tokenize_command( output, include_added_tokens, } => tokenize::run_import_hf(input, output, *include_added_tokens, cli.json), + #[cfg(feature = "inference")] + TokenizeCommands::Encode { + model, + prompt, + file, + } => commands::tokenize_encode::run_encode( + model, + prompt.as_deref(), + file.as_deref(), + cli.json, + ), #[cfg(feature = "training")] TokenizeCommands::EncodeCorpus { corpus, diff --git a/crates/apr-cli/src/error.rs b/crates/apr-cli/src/error.rs index 586eb6bbad..39317023e7 100644 --- a/crates/apr-cli/src/error.rs +++ b/crates/apr-cli/src/error.rs @@ -54,7 +54,8 @@ pub enum CliError { /// /// #3661: `apr inspect` and `apr tensors` wrapped every parse failure in /// [`CliError::InvalidFormat`], so a truncated GGUF was reported as - /// "Invalid APR format". Build it with [`CliError::invalid_model_file`]. + /// "Invalid APR format". Build it with [`CliError::invalid_model_file`], + /// or [`CliError::invalid_model_bytes`] when the bytes are already in memory. /// Shares exit code 4, the same failure class. #[error("Invalid {format} file: {message}")] InvalidModelFile { @@ -190,6 +191,18 @@ impl CliError { message: format!("{context}: {inner}"), } } + + /// A parse failure in model-file bytes already in memory (#3691). + /// + /// Same rule as [`CliError::invalid_model_file`]: the format is read from + /// the bytes' magic, never assumed. `apr hex` reads the whole file before + /// dispatching, so its SafeTensors header reader passes what it holds. + pub(crate) fn invalid_model_bytes(bytes: &[u8], message: impl Into) -> Self { + Self::InvalidModelFile { + format: crate::commands::hex::detected_format_of(bytes), + message: message.into(), + } + } } impl From for CliError { diff --git a/crates/apr-cli/src/tokenize_commands.rs b/crates/apr-cli/src/tokenize_commands.rs index 94409dc439..08cf198b51 100644 --- a/crates/apr-cli/src/tokenize_commands.rs +++ b/crates/apr-cli/src/tokenize_commands.rs @@ -115,6 +115,24 @@ pub enum TokenizeCommands { include_added_tokens: bool, }, + /// Print the token ids a GGUF model's own tokenizer gives a text (#3726). + /// + /// The ids come from the same encoder `apr run`, `apr serve` and `apr parity` use. The + /// output names the path that produced them: `canonical` (pre-tokenizer plus ranked + /// merges, identical to llama.cpp) or `greedy-fallback: `. Only the file's metadata + /// is read; no weights are loaded. + #[cfg(feature = "inference")] + Encode { + /// GGUF model file whose tokenizer to run + #[arg(value_name = "MODEL")] + model: PathBuf, + /// Text to encode + #[arg(short = 'p', long, value_name = "TEXT", conflicts_with = "file")] + prompt: Option, + /// File whose contents to encode (read as UTF-8, verbatim) + #[arg(short = 'f', long, value_name = "FILE")] + file: Option, + }, /// Encode a JSONL corpus into `.bin` shards per contracts/pretokenize-bin-v1.yaml. /// /// Loads a trained BPE tokenizer (vocab.json + merges.txt) from `--tokenizer`, diff --git a/crates/aprender-contracts-cli/src/cli.rs b/crates/aprender-contracts-cli/src/cli.rs index 538dc5e94f..c790ff16f8 100644 --- a/crates/aprender-contracts-cli/src/cli.rs +++ b/crates/aprender-contracts-cli/src/cli.rs @@ -105,6 +105,12 @@ pub enum Commands { /// Write nothing; exit 1 if the tracked files differ from a fresh extraction (what CI runs) #[arg(long)] check: bool, + /// With `--release-*`: write the corpus graph PLUS the release evidence to this N-Triples file, and leave + /// the tracked contracts.nt / shapes.ttl untouched (aprender#3715) + #[arg(long)] + out: Option, + #[command(flatten)] + release: Box, }, /// Show cross-contract obligation coverage report Coverage { @@ -284,6 +290,12 @@ pub enum Commands { /// Run ONE named gate and report only it (ONT-001 section 5 ONT-2b): `--gate sigma`. #[arg(long)] gate: Option, + /// With `--gate shapes`: grade only this shape family (the shape and every `.*` shape), armed + /// whatever `armed_shapes` says (aprender#3715: `--shape release-readiness-v1`). + #[arg(long)] + shape: Option, + #[command(flatten)] + release: Box, }, /// Score contracts or a codebase directory Score { @@ -469,3 +481,73 @@ pub enum CensusFormat { /// The bytes `contracts/census.json` carries. Json, } + +/// aprender#3715 — the release subject `extract:release-evidence` reads, for `pv lint --gate shapes` and +/// `pv extract`. All absent → no release graph. `--release-version` and `--release-commit` come together; the +/// rest need them. +#[derive(clap::Args, Debug, Default, Clone)] +pub struct ReleaseArgs { + /// The release version whose evidence `release-readiness-v1` grades + #[arg(long)] + pub release_version: Option, + /// The release commit (MC), full 40-hex: the dogfood receipt's commit, and the receipts' unless --receipts-commit + #[arg(long)] + pub release_commit: Option, + /// T-4 only: the sha the committed receipts were measured at, after R7 proved the tree equal modulo evidence/ + #[arg(long)] + pub receipts_commit: Option, + /// The per-host model receipts (default: evidence/dogfood/models//) + #[arg(long)] + pub receipts: Option, + /// The per-host kernel-diff receipts (default: evidence/dogfood/kernels//) + #[arg(long)] + pub kernel_receipts: Option, + /// The dogfood receipt R5 judged (without it the release has no dogfood receipt, which is a violation) + #[arg(long)] + pub dogfood_receipt: Option, + /// The tokenizer-parity receipts, apr vs the pinned llama.cpp (default: evidence/dogfood/tokenizer//) + #[arg(long)] + pub tokenizer_receipts: Option, +} + +impl ReleaseArgs { + /// Did the caller pass any release flag at all? + #[must_use] + pub fn any(&self) -> bool { + self.release_version.is_some() + || self.release_commit.is_some() + || self.receipts_commit.is_some() + || self.receipts.is_some() + || self.kernel_receipts.is_some() + || self.dogfood_receipt.is_some() + || self.tokenizer_receipts.is_some() + } + + /// The subject, or `None` when no flag was passed. A partial set is refused, never completed by a default. + pub fn subject( + &self, + ) -> Result, String> + { + use provable_contracts::ontology::extract::release_inputs::Subject; + if !self.any() { + return Ok(None); + } + let (Some(v), Some(c)) = (&self.release_version, &self.release_commit) else { + return Err( + "--release-version and --release-commit are both required with any --release-*, \ + --receipts*, --kernel-receipts or --dogfood-receipt flag" + .into(), + ); + }; + let mut s = Subject::new(v, c).map_err(|e| e.to_string())?; + if let Some(rc) = &self.receipts_commit { + s = s.with_receipts_commit(rc).map_err(|e| e.to_string())?; + } + s.receipts_dir.clone_from(&self.receipts); + s.kernel_receipts_dir.clone_from(&self.kernel_receipts); + s.dogfood_receipt.clone_from(&self.dogfood_receipt); + s.tokenizer_receipts_dir + .clone_from(&self.tokenizer_receipts); + Ok(Some(s)) + } +} diff --git a/crates/aprender-contracts-cli/src/commands/extract_rdf.rs b/crates/aprender-contracts-cli/src/commands/extract_rdf.rs index 2b5d5f2998..a962e180df 100644 --- a/crates/aprender-contracts-cli/src/commands/extract_rdf.rs +++ b/crates/aprender-contracts-cli/src/commands/extract_rdf.rs @@ -10,6 +10,9 @@ use std::path::Path; use provable_contracts::lint::shapes_gate::collect_shapes; use provable_contracts::ontology::extract; +use provable_contracts::ontology::extract::release_inputs::Subject; + +use crate::contract_walk::ReleaseArgsRefused; use provable_contracts::ontology::shapes::to_turtle; use sha2::{Digest, Sha256}; @@ -22,8 +25,24 @@ struct ExtractReport<'a> { check: Option>, } -/// `check == true` → compare and report drift, write nothing. -pub fn run(contract_dir: &Path, check: bool) -> Result<(), Box> { +/// `check == true` → compare and report drift, write nothing. With a release `subject` (aprender#3715) the +/// release evidence joins the graph, and the result goes ONLY to `out`: the tracked `contracts.nt` is the corpus, +/// and a release's receipts written into it would be a release baked into every later PR's baseline. +pub fn run( + contract_dir: &Path, + check: bool, + subject: Option<&Subject>, + out: Option<&Path>, +) -> Result<(), Box> { + if let Some(subject) = subject { + return run_release(contract_dir, check, subject, out); + } + if out.is_some() { + return Err(ReleaseArgsRefused( + "--out writes the release evidence graph; it needs --release-version and --release-commit".into(), + ) + .into()); + } let extraction = match extract::all(contract_dir) { Ok(x) => x, Err(e) => { @@ -90,3 +109,54 @@ pub fn run(contract_dir: &Path, check: bool) -> Result<(), Box { + triples: usize, + sha256: String, + out: &'a str, + release: provable_contracts::ontology::extract::release_evidence::ReleaseStats, +} + +/// `pv extract --release-version V --release-commit MC … --out FILE`: the corpus graph plus the release evidence, +/// written to FILE (never to the tracked files), with its content address and what was derived. +fn run_release( + contract_dir: &Path, + check: bool, + subject: &Subject, + out: Option<&Path>, +) -> Result<(), Box> { + if check { + return Err(ReleaseArgsRefused( + "--check compares the TRACKED corpus files; a release graph is never tracked, so --check with \ + --release-* has nothing to compare" + .into(), + ) + .into()); + } + let Some(out) = out else { + return Err(ReleaseArgsRefused( + "--release-* needs --out FILE: the release graph is never written into the tracked contracts.nt".into(), + ) + .into()); + }; + let extraction = match extract::all_with(contract_dir, Some(subject)) { + Ok(x) => x, + Err(e) => { + eprintln!("error: {e}"); + std::process::exit(3); + } + }; + let nt = extraction.graph.to_ntriples(); + let mut hasher = Sha256::new(); + hasher.update(nt.as_bytes()); + std::fs::write(out, &nt)?; + let report = ReleaseExtractReport { + triples: extraction.graph.len(), + sha256: format!("{:x}", hasher.finalize()), + out: out.to_str().unwrap_or("?"), + release: extraction.release.unwrap_or_default(), + }; + println!("{}", serde_json::to_string_pretty(&report)?); + Ok(()) +} diff --git a/crates/aprender-contracts-cli/src/commands/lint.rs b/crates/aprender-contracts-cli/src/commands/lint.rs index c63ee905fa..8c9c62febc 100644 --- a/crates/aprender-contracts-cli/src/commands/lint.rs +++ b/crates/aprender-contracts-cli/src/commands/lint.rs @@ -2,6 +2,7 @@ use std::path::Path; use provable_contracts::lint::config::{find_config, load_config}; use provable_contracts::lint::rules::RuleSeverity; +use provable_contracts::lint::shapes_gate::ShapesOptions; use provable_contracts::lint::trend; use provable_contracts::lint::{run_lint, GateDetail, LintConfig, LintReport}; use provable_contracts::ontology::verdict::Verdict; @@ -48,6 +49,7 @@ pub fn run( strict_test_binding: bool, armed_baseline_ref: Option<&str>, gate: Option<&str>, + shapes_opts: ShapesOptions, ) -> Result<(), Box> { refuse_missing_corpus(contract_dir)?; // Both preconditions before any dispatch: a single file under @@ -56,7 +58,7 @@ pub fn run( // missing on exactly the input the refusal exists for. refuse_single_file_strict_binding(contract_dir, strict_test_binding)?; if let Some(name) = gate { - return run_single_gate(contract_dir, name); + return run_single_gate(contract_dir, name, &shapes_opts); } if watch { return run_watch( @@ -160,6 +162,28 @@ fn report_coverage( } } +/// aprender#3715: `--shape` and `--release-*` as the shapes gate's options. They mean something ONLY under +/// `--gate shapes`; anywhere else they are refused (exit 3), because a release flag that was silently ignored is +/// a release gate that silently did not run. +pub fn shapes_options( + gate: Option<&str>, + shape: Option, + release: &crate::cli::ReleaseArgs, +) -> Result { + use crate::contract_walk::ReleaseArgsRefused; + if (shape.is_some() || release.any()) && gate != Some("shapes") { + return Err(ReleaseArgsRefused( + "--shape and --release-* / --receipts* / --kernel-receipts / --dogfood-receipt apply only to \ + `--gate shapes`" + .into(), + )); + } + Ok(ShapesOptions { + only: shape, + release: release.subject().map_err(ReleaseArgsRefused)?, + }) +} + /// One gate's report. NOT a `LintReport`: `--gate` answers about one gate, and a reader must be able to tell the /// two apart without counting keys. #[derive(serde::Serialize)] @@ -187,8 +211,12 @@ struct SingleGateFinding<'a> { /// ONT-2b: `--gate ` runs ONE gate and reports only it, mapping its verdict through ONT-6's lattice — /// Pass 0 · Fail 1 `reject:` · no Σ 2 `decline:` (R-2, zero is a decline) · malformed Σ 3 `error:`. -fn run_single_gate(contract_dir: &Path, name: &str) -> Result<(), Box> { - let (result, findings) = decide_named_gate(contract_dir, name)?; +fn run_single_gate( + contract_dir: &Path, + name: &str, + shapes_opts: &ShapesOptions, +) -> Result<(), Box> { + let (result, findings) = decide_named_gate(contract_dir, name, shapes_opts)?; let report = SingleGateReport { gate: &result.name, @@ -235,12 +263,13 @@ type NamedGateAnswer = ( fn decide_named_gate( contract_dir: &Path, name: &str, + shapes_opts: &ShapesOptions, ) -> Result> { use provable_contracts::lint::{ relations_gate::RelationsOutcome, sigma_gate::SigmaOutcome, NamedGateOutcome, NAMED_GATES, }; - match provable_contracts::lint::run_named_gate(contract_dir, name) { + match provable_contracts::lint::run_named_gate_with(contract_dir, name, shapes_opts) { NamedGateOutcome::UnknownGate => Err(crate::contract_walk::UnknownGate { asked: name.to_string(), known: NAMED_GATES.iter().map(|g| (*g).to_string()).collect(), diff --git a/crates/aprender-contracts-cli/src/contract_walk.rs b/crates/aprender-contracts-cli/src/contract_walk.rs index d04c0299e3..45dff87517 100644 --- a/crates/aprender-contracts-cli/src/contract_walk.rs +++ b/crates/aprender-contracts-cli/src/contract_walk.rs @@ -226,6 +226,19 @@ impl fmt::Display for UnknownGate { impl std::error::Error for UnknownGate {} +/// aprender#3715: a `--shape` / `--release-*` flag set that cannot name a release (a partial subject, a short +/// sha, a flag outside `--gate shapes`). The CALLER's error, exit 3 — never a verdict about the evidence. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ReleaseArgsRefused(pub String); + +impl fmt::Display for ReleaseArgsRefused { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{}", self.0) + } +} + +impl std::error::Error for ReleaseArgsRefused {} + /// Exit status of an `armed_gates` list that dropped a gate its comparand armed (ONT-001 §3.9). pub const ARMED_GATES_SHRANK_EXIT: i32 = 3; @@ -239,6 +252,7 @@ pub fn exit_code_for(err: &(dyn std::error::Error + 'static)) -> i32 { } else if err.downcast_ref::().is_some() || err.downcast_ref::().is_some() || err.downcast_ref::().is_some() + || err.downcast_ref::().is_some() { ARMED_GATES_SHRANK_EXIT } else { diff --git a/crates/aprender-contracts-cli/src/lib.rs b/crates/aprender-contracts-cli/src/lib.rs index 9dc9adee4f..a1050b0411 100644 --- a/crates/aprender-contracts-cli/src/lib.rs +++ b/crates/aprender-contracts-cli/src/lib.rs @@ -125,7 +125,14 @@ pub fn dispatch(command: Commands) -> Result<(), Box> { Commands::Extract { contract_dir, check, - } => commands::extract_rdf::run(&contract_dir, check), + out, + release, + } => { + let subject = release + .subject() + .map_err(crate::contract_walk::ReleaseArgsRefused)?; + commands::extract_rdf::run(&contract_dir, check, subject.as_ref(), out.as_deref()) + } Commands::Coverage { contract_dir, binding, @@ -205,6 +212,8 @@ pub fn dispatch(command: Commands) -> Result<(), Box> { strict_test_binding, armed_baseline_ref, gate, + shape, + release, .. } => { if let Some(ref rule_id) = explain { @@ -236,6 +245,7 @@ pub fn dispatch(command: Commands) -> Result<(), Box> { strict_test_binding, armed_baseline_ref.as_deref(), gate.as_deref(), + commands::lint::shapes_options(gate.as_deref(), shape, &release)?, ) } Commands::Score { diff --git a/crates/aprender-contracts-cli/tests/ont4b_shapes_gate.rs b/crates/aprender-contracts-cli/tests/ont4b_shapes_gate.rs index 4b2f19820f..1057520206 100644 --- a/crates/aprender-contracts-cli/tests/ont4b_shapes_gate.rs +++ b/crates/aprender-contracts-cli/tests/ont4b_shapes_gate.rs @@ -228,11 +228,13 @@ fn the_tracked_repo_graph_is_fresh() { // is shared across branches, so a sibling PR that also adds a shape // (#3600's parity-receipt-v2) will need the number raised again at merge — // that is the ratchet working, not a conflict to route around. It did: the - // 0.69 batch folded #3600 in and the count went 6 -> 9 with its three shapes. + // 0.69 batch folded #3600 in and the count went 6 -> 9 with its three shapes. #3715 added the nine-shape + // `release-readiness-v1` family (shapes_n=18, triples=15863, measured on its branch); it contributes no focus + // node to a PR's graph — the release evidence is extracted only under `--release-*`. assert_eq!( v["shapes_n"], - 9, - "ont-shapes-v1 + ladder-measured + ladder-green (ONT-4c1) + bound-symbols-resolve + lean-statements-grounded (ONT-4b2) + refusal-receipt-v1 (#3605) + parity-receipt-complete + parity-comparator-self + parity-comparator-oracle (parity-receipt-v2, #3600)\n{}", + 18, + "ont-shapes-v1 + ladder-measured + ladder-green (ONT-4c1) + bound-symbols-resolve + lean-statements-grounded (ONT-4b2) + refusal-receipt-v1 (#3605) + parity-receipt-complete + parity-comparator-self + parity-comparator-oracle (parity-receipt-v2, #3600) + release-readiness-v1{{,.release,.host,.context,.model,.coverage,.tokenizer,.kernel,.refusal}} (#3715)\n{}", show(&r) ); } diff --git a/crates/aprender-contracts-cli/tests/ont4c3_parity_receipts.rs b/crates/aprender-contracts-cli/tests/ont4c3_parity_receipts.rs index e6a055ef77..c2452e7955 100644 --- a/crates/aprender-contracts-cli/tests/ont4c3_parity_receipts.rs +++ b/crates/aprender-contracts-cli/tests/ont4c3_parity_receipts.rs @@ -231,19 +231,9 @@ fn the_committed_tree_agrees_with_its_own_denominator() { .current_dir(repo_root()) .output() .expect("failed to spawn the denominator predicate"); - // #3695: exit 3 is fleet state, not a verdict. This runner has no python3 (the fleet is - // python-free for automation, infra#708), so the predicate could not classify anything. - // It is accepted ONLY with the UNMEASURED line naming the interpreter; exits 1 and 2 still - // fail. The T-2 ledger requires the MEASURED "PASS 7 receipt(s)" at the release commit, and - // #3694 removes this path once classify() needs no python. - let stderr = String::from_utf8_lossy(&out.stderr); - if out.status.code() == Some(3) - && stderr.contains("UNMEASURED runner=") - && stderr.contains("reason=no-interpreter interpreter=") - { - eprintln!("{}", stderr.trim()); - return; - } + // #3694: the classifier is awk, so a python-free runner (infra#708) measures the count like any + // other. The UNMEASURED exit 3 that #3695 accepted here as a stop-gap is gone: anything but + // success is RED. assert!( out.status.success(), "exit {:?}\n{}{}", diff --git a/crates/aprender-contracts-cli/tests/ont_release_readiness.rs b/crates/aprender-contracts-cli/tests/ont_release_readiness.rs new file mode 100644 index 0000000000..7f60586a5b --- /dev/null +++ b/crates/aprender-contracts-cli/tests/ont_release_readiness.rs @@ -0,0 +1,812 @@ +//! aprender#3715 — `release-readiness-v1` on the CLI: the red-turning case table. +//! +//! Every case copies `tests/fixtures/ont/release-green/` AND the real `contracts/release-readiness-v1.yaml` (so the +//! fixture cannot drift from the shapes it proves) into a scratch repo, changes exactly ONE thing, and runs +//! `pv lint contracts --gate shapes --shape release-readiness-v1 --release-version 0.69.1 --release-commit MC …`. +//! +//! DISCRIMINATION: the green base must PASS with every cell named (48/48, 4 kernel cells) — so a build that +//! rejects everything fails this file — and each single-cell change must FAIL naming that cell — so a build that +//! accepts everything fails it too. The ONT-001 rule: a shape that has never gone red is not armed. + +use std::path::{Path, PathBuf}; +use std::process::Command; + +use serde_json::Value; + +const MC: &str = "1111111111111111111111111111111111111111"; +const PARENT: &str = "3333333333333333333333333333333333333333"; +const BUMP: &str = "2222222222222222222222222222222222222222"; +const MODEL: &str = "tiny-qwen35.gguf"; + +struct Run { + code: i32, + stdout: String, + stderr: String, +} + +fn show(r: &Run) -> String { + format!( + "exit {}\n--- stdout\n{}\n--- stderr\n{}", + r.code, r.stdout, r.stderr + ) +} + +fn repo_root() -> PathBuf { + Path::new(env!("CARGO_MANIFEST_DIR")).join("../..") +} + +fn copy_dir(from: &Path, to: &Path) { + std::fs::create_dir_all(to).expect("mkdir"); + for e in std::fs::read_dir(from).expect("read fixture dir").flatten() { + let p = e.path(); + let dst = to.join(e.file_name()); + if p.is_dir() { + copy_dir(&p, &dst); + } else { + std::fs::copy(&p, &dst).expect("copy fixture file"); + } + } +} + +/// The green base: the fixture plus the REAL shape contract. +fn green() -> tempfile::TempDir { + let t = tempfile::tempdir().expect("tempdir"); + copy_dir( + &repo_root().join("tests/fixtures/ont/release-green"), + t.path(), + ); + std::fs::copy( + repo_root().join("contracts/release-readiness-v1.yaml"), + t.path().join("contracts/release-readiness-v1.yaml"), + ) + .expect("copy the real contract"); + t +} + +fn pv(cwd: &Path, args: &[&str]) -> Run { + let out = Command::new(env!("CARGO_BIN_EXE_pv")) + .current_dir(cwd) + .args(args) + .output() + .expect("spawn pv"); + Run { + code: out.status.code().unwrap_or(-1), + stdout: String::from_utf8_lossy(&out.stdout).into_owned(), + stderr: String::from_utf8_lossy(&out.stderr).into_owned(), + } +} + +fn s(p: &Path) -> String { + p.to_str().expect("utf-8 path").to_string() +} + +/// The release gate as T-1 calls it, plus `extra` flags. +fn gate(t: &Path, extra: &[&str]) -> Run { + let contracts = s(&t.join("contracts")); + let dogfood = s(&t.join("dogfood-receipt.json")); + let mut args = vec![ + "lint", + &contracts, + "--gate", + "shapes", + "--shape", + "release-readiness-v1", + "--release-version", + "0.69.1", + "--release-commit", + MC, + "--dogfood-receipt", + &dogfood, + ]; + args.extend_from_slice(extra); + pv(t, &args) +} + +fn json_of(r: &Run) -> Value { + serde_json::from_str(&r.stdout).unwrap_or_else(|e| panic!("stdout is JSON: {e}\n{}", show(r))) +} + +fn edit(path: &Path, f: impl FnOnce(&mut Value)) { + let mut v: Value = + serde_json::from_str(&std::fs::read_to_string(path).expect("read")).expect("json"); + f(&mut v); + std::fs::write(path, serde_json::to_string_pretty(&v).expect("ser")).expect("write"); +} + +fn models(t: &Path, host: &str) -> PathBuf { + t.join(format!("evidence/dogfood/models/0.69.1/{host}.json")) +} + +fn kernels(t: &Path, host: &str) -> PathBuf { + t.join(format!("evidence/dogfood/kernels/0.69.1/{host}.json")) +} + +/// Index of the one row for (verb, thinking, rung) in a model receipt. +fn row_index(v: &Value, verb: &str, thinking: &str, rung: &str) -> usize { + v["cells"] + .as_array() + .expect("cells") + .iter() + .position(|c| c["verb"] == verb && c["thinking"] == thinking && c["context"] == rung) + .unwrap_or_else(|| panic!("no row {verb}/{thinking}/{rung}")) +} + +fn row<'a>(v: &'a mut Value, verb: &str, thinking: &str, rung: &str) -> &'a mut Value { + let i = row_index(v, verb, thinking, rung); + &mut v["cells"][i] +} + +fn cell(host: &str, verb: &str, thinking: &str, rung: &str) -> String { + format!("release-cell/0.69.1/{host}/{MODEL}/{verb}/think-{thinking}/{rung}") +} + +/// RED, and the findings name `needle`. +fn assert_red_naming(r: &Run, needle: &str) -> Value { + assert_eq!(r.code, 1, "{}", show(r)); + let v = json_of(r); + assert_eq!(v["verdict"], "Fail", "{}", show(r)); + let msgs: Vec<&str> = v["findings"] + .as_array() + .expect("findings") + .iter() + .filter_map(|f| f["message"].as_str()) + .collect(); + assert!( + msgs.iter().any(|m| m.contains(needle)), + "no finding names {needle}:\n{}", + msgs.join("\n") + ); + v +} + +#[test] +fn the_green_release_passes_with_every_cell_named() { + let t = green(); + let r = gate(t.path(), &[]); + assert_eq!(r.code, 0, "{}", show(&r)); + let v = json_of(&r); + assert_eq!(v["verdict"], "Pass"); + assert_eq!( + v["pc_shape"], "fired", + "the plant fired from the named family" + ); + assert_eq!( + v["pc_extract"]["release-evidence"], "fired", + "the extractor's own control (PMAT-3704)" + ); + let rel = &v["release"]; + assert_eq!( + rel["cells"], 48, + "2 hosts × 4 verbs × 2 thinking × (4k, consumer-max, declared)" + ); + assert_eq!(rel["cells_with_row"], 48); + assert_eq!(rel["cell_names"].as_array().map(Vec::len), Some(48)); + assert_eq!( + rel["kernel_cells"], 4, + "(q4k_gemv@q4_k, rmsnorm@f32) × 2 hosts" + ); + assert_eq!(rel["orphan_rows"], 0); + assert_eq!( + rel["tokenizer_cells"], 1, + "one distinct model across both hosts (#3726)" + ); + let armed = v["armed_shapes"].as_array().expect("armed").len(); + assert_eq!( + armed, 9, + "--shape arms the whole family whatever armed_shapes says" + ); +} + +#[test] +fn a_missing_cell_is_red_naming_it() { + let t = green(); + edit(&models(t.path(), "lambda"), |v| { + let i = row_index(v, "chat", "on", "consumer-max"); + v["cells"].as_array_mut().expect("cells").remove(i); + }); + assert_red_naming( + &gate(t.path(), &[]), + &cell("lambda", "chat", "on", "consumer-max"), + ); +} + +#[test] +fn a_skipped_cell_is_red_naming_it() { + let t = green(); + edit(&models(t.path(), "gx10"), |v| { + row(v, "serve", "off", "4k")["verdict"] = "skip".into(); + }); + assert_red_naming(&gate(t.path(), &[]), &cell("gx10", "serve", "off", "4k")); +} + +#[test] +fn a_fallback_cell_is_red_naming_it() { + let t = green(); + edit(&models(t.path(), "lambda"), |v| { + row(v, "run", "off", "4k")["fallback"] = true.into(); + }); + assert_red_naming(&gate(t.path(), &[]), &cell("lambda", "run", "off", "4k")); +} + +#[test] +fn a_cell_that_does_not_say_whether_it_fell_back_is_red() { + let t = green(); + edit(&models(t.path(), "lambda"), |v| { + row(v, "code", "off", "4k") + .as_object_mut() + .expect("row") + .remove("fallback"); + }); + assert_red_naming(&gate(t.path(), &[]), &cell("lambda", "code", "off", "4k")); +} + +#[test] +fn a_receipt_measured_at_another_commit_is_stale_and_red() { + let t = green(); + edit(&models(t.path(), "lambda"), |v| { + v["apr_sha"] = PARENT.into() + }); + let v = assert_red_naming(&gate(t.path(), &[]), "release-host/0.69.1/lambda"); + let msgs = v["findings"].to_string(); + assert!( + msgs.contains(&cell("lambda", "run", "off", "4k")), + "every lambda cell is stale" + ); + assert!( + !msgs.contains("release-host/0.69.1/gx10"), + "gx10 is untouched" + ); +} + +#[test] +fn a_receipt_for_another_version_is_stale_and_red() { + let t = green(); + edit(&models(t.path(), "gx10"), |v| { + v["version"] = "0.69.0".into() + }); + assert_red_naming(&gate(t.path(), &[]), "release-host/0.69.1/gx10"); +} + +#[test] +fn a_think_on_row_with_an_unclosed_block_and_an_empty_answer_is_red() { + let t = green(); + edit(&models(t.path(), "lambda"), |v| { + let r = row(v, "chat", "on", "4k"); + r["think_closed"] = false.into(); + r["answer_chars"] = 0.into(); + }); + let v = assert_red_naming(&gate(t.path(), &[]), &cell("lambda", "chat", "on", "4k")); + let msgs = v["findings"].to_string(); + assert!(msgs.contains("thinkOk"), "names the unclosed block: {msgs}"); + assert!(msgs.contains("answered"), "names the empty answer: {msgs}"); +} + +#[test] +fn a_missing_think_on_cell_is_red_naming_it() { + let t = green(); + edit(&models(t.path(), "gx10"), |v| { + let i = row_index(v, "code", "on", "declared"); + v["cells"].as_array_mut().expect("cells").remove(i); + }); + assert_red_naming( + &gate(t.path(), &[]), + &cell("gx10", "code", "on", "declared"), + ); +} + +#[test] +fn an_absent_declared_length_rung_is_red_and_so_is_an_unmeasured_length() { + let t = green(); + edit(&models(t.path(), "lambda"), |v| { + let i = row_index(v, "run", "off", "declared"); + v["cells"].as_array_mut().expect("cells").remove(i); + }); + assert_red_naming( + &gate(t.path(), &[]), + &cell("lambda", "run", "off", "declared"), + ); + + // the length is a property of the FILE (one sha256): gx10 dropping it leaves lambda's measurement standing, + // but gx10's own declared cells have no size to fill and go red … + let t = green(); + edit(&models(t.path(), "gx10"), |v| { + v["inventory"][0] + .as_object_mut() + .expect("item") + .remove("context_length"); + }); + assert_red_naming( + &gate(t.path(), &[]), + &cell("gx10", "run", "off", "declared"), + ); + // … and with no host measuring it, the model itself is named + let t = green(); + for h in ["lambda", "gx10"] { + edit(&models(t.path(), h), |v| { + v["inventory"][0] + .as_object_mut() + .expect("item") + .remove("context_length"); + }); + } + assert_red_naming(&gate(t.path(), &[]), "contextLength"); +} + +#[test] +fn two_hosts_that_disagree_on_one_files_length_are_red() { + let t = green(); + edit(&models(t.path(), "gx10"), |v| { + v["inventory"][0]["context_length"] = 4096.into(); + }); + let v = assert_red_naming(&gate(t.path(), &[]), "contextLength"); + assert!(v["findings"].to_string().contains("maxCount")); +} + +#[test] +fn a_prompt_short_of_its_rung_is_red() { + let t = green(); + edit(&models(t.path(), "lambda"), |v| { + row(v, "serve", "on", "consumer-max")["prompt_tokens"] = 1000.into(); + }); + let v = assert_red_naming( + &gate(t.path(), &[]), + &cell("lambda", "serve", "on", "consumer-max"), + ); + assert!(v["findings"].to_string().contains("contextMet")); +} + +#[test] +fn two_rows_for_one_cell_are_ambiguous_and_red() { + let t = green(); + edit(&models(t.path(), "lambda"), |v| { + let dup = row(v, "run", "on", "4k").clone(); + v["cells"].as_array_mut().expect("cells").push(dup); + }); + let v = assert_red_naming(&gate(t.path(), &[]), &cell("lambda", "run", "on", "4k")); + assert!(v["findings"].to_string().contains("maxCount")); +} + +#[test] +fn a_kernel_green_on_sm_121_and_red_on_sm_89_is_named_on_lambda_before_any_model_cell() { + let t = green(); + edit(&kernels(t.path(), "lambda"), |v| { + v["rows"][0]["verdict"] = "fail".into(); + v["rows"][0]["max_err"] = 0.5.into(); + }); + // and one model cell too, so the ORDER is observable + edit(&models(t.path(), "lambda"), |v| { + row(v, "run", "off", "4k")["verdict"] = "fail".into(); + }); + let v = assert_red_naming( + &gate(t.path(), &[]), + "release-kernel/0.69.1/lambda/q4k_gemv/q4_k", + ); + let findings = v["findings"].as_array().expect("findings"); + let first = findings[0]["message"].as_str().unwrap_or(""); + assert!( + first.contains("release-kernel/0.69.1/lambda"), + "kernel first: {first}" + ); + assert!( + !v["findings"] + .to_string() + .contains("release-kernel/0.69.1/gx10"), + "gx10's kernel is green" + ); +} + +#[test] +fn a_kernel_measured_on_one_arch_only_is_a_missing_row_on_the_other() { + let t = green(); + edit(&kernels(t.path(), "gx10"), |v| { + v["rows"].as_array_mut().expect("rows").remove(1); // rmsnorm@f32 unmeasured on gx10 + }); + assert_red_naming( + &gate(t.path(), &[]), + "release-kernel/0.69.1/gx10/rmsnorm/f32", + ); +} + +#[test] +fn a_required_host_with_no_receipt_is_red_naming_the_host() { + let t = green(); + std::fs::remove_file(models(t.path(), "gx10")).expect("rm"); + assert_red_naming(&gate(t.path(), &[]), "release-host/0.69.1/gx10"); +} + +#[test] +fn an_inherited_dogfood_or_none_at_all_is_red() { + let t = green(); + edit(&t.path().join("dogfood-receipt.json"), |v| { + v["commit"] = PARENT.into(); + }); + assert_red_naming(&gate(t.path(), &[]), "release-dogfood/0.69.1"); + + let t = green(); + let contracts = s(&t.path().join("contracts")); + let r = pv( + t.path(), + &[ + "lint", + &contracts, + "--gate", + "shapes", + "--shape", + "release-readiness-v1", + "--release-version", + "0.69.1", + "--release-commit", + MC, + ], + ); + assert_red_naming(&r, "dogfoodReceipt"); +} + +#[test] +fn a_consumer_with_no_recorded_number_is_red_on_the_derived_rung() { + let t = green(); + edit(&t.path().join("evidence/release/context-rungs.json"), |v| { + v["consumers"].as_array_mut().expect("consumers").push( + serde_json::from_str::( + r#"{"consumer": "pending", "basis": "", "source": "fixture://pending"}"#, + ) + .expect("a consumer record"), + ); + }); + assert_red_naming(&gate(t.path(), &[]), "release-context/consumer-max"); +} + +#[test] +fn thinking_that_contradicts_its_template_markers_is_red() { + let t = green(); + edit(&models(t.path(), "lambda"), |v| { + // off only, though the template carries the `enable_thinking` switch (#3723's derivation says both) + v["inventory"][0]["thinking_modes"] = + serde_json::from_str::(r#"["off"]"#).expect("modes"); + }); + assert_red_naming(&gate(t.path(), &[]), "thinkingContradiction"); +} + +#[test] +fn a_v1_receipt_cannot_stand_for_a_host() { + let t = green(); + edit(&models(t.path(), "gx10"), |v| { + v["schema"] = "apr-model-ladder-receipt/v1".into(); + }); + assert_red_naming(&gate(t.path(), &[]), "release-host/0.69.1/gx10"); +} + +#[test] +fn t4_receipts_measured_at_the_bump_head_pass_only_with_receipts_commit() { + // T-4: the committed receipts were measured at the bump head; R7 proved it equal to MC modulo evidence/ + let t = green(); + let other = t.path().join("t4-receipts"); + std::fs::create_dir_all(&other).expect("mkdir"); + for h in ["lambda", "gx10"] { + let dst = other.join(format!("{h}.json")); + std::fs::rename(models(t.path(), h), &dst).expect("mv"); + edit(&dst, |v| v["apr_sha"] = BUMP.into()); + edit(&kernels(t.path(), h), |v| v["apr_sha"] = BUMP.into()); + edit(&tokenizer(t.path()), |v| v["apr_sha"] = BUMP.into()); + } + let dir = s(&other); + let r = gate(t.path(), &["--receipts", &dir, "--receipts-commit", BUMP]); + assert_eq!(r.code, 0, "{}", show(&r)); + let r = gate(t.path(), &["--receipts", &dir]); + assert_red_naming(&r, "release-host/0.69.1/lambda"); +} + +#[test] +fn no_release_subject_declines_it_never_passes() { + let t = green(); + let contracts = s(&t.path().join("contracts")); + let r = pv( + t.path(), + &[ + "lint", + &contracts, + "--gate", + "shapes", + "--shape", + "release-readiness-v1", + ], + ); + assert_eq!( + r.code, + 2, + "zero cells is a decline, not a pass: {}", + show(&r) + ); +} + +#[test] +fn a_malformed_subject_or_a_misplaced_flag_is_the_callers_error() { + let t = green(); + let contracts = s(&t.path().join("contracts")); + let partial = pv( + t.path(), + &[ + "lint", + &contracts, + "--gate", + "shapes", + "--release-version", + "0.69.1", + ], + ); + assert_eq!(partial.code, 3, "{}", show(&partial)); + let short = pv( + t.path(), + &[ + "lint", + &contracts, + "--gate", + "shapes", + "--release-version", + "0.69.1", + "--release-commit", + "225b2a9ab", + ], + ); + assert_eq!( + short.code, + 3, + "a prefix is refused, never matched: {}", + show(&short) + ); + let misplaced = pv( + t.path(), + &["lint", &contracts, "--gate", "sigma", "--shape", "x"], + ); + assert_eq!(misplaced.code, 3, "{}", show(&misplaced)); + let unknown = pv( + t.path(), + &[ + "lint", + &contracts, + "--gate", + "shapes", + "--shape", + "no-such-shape", + ], + ); + assert_eq!(unknown.code, 3, "{}", show(&unknown)); +} + +#[test] +fn pv_extract_writes_the_release_graph_only_where_it_is_told() { + let t = green(); + let contracts = s(&t.path().join("contracts")); + let out = t.path().join("release.nt"); + let dogfood = s(&t.path().join("dogfood-receipt.json")); + let args = [ + "extract", + &contracts, + "--release-version", + "0.69.1", + "--release-commit", + MC, + "--dogfood-receipt", + &dogfood, + ]; + let refused = pv(t.path(), &args); + assert_eq!(refused.code, 3, "no --out: {}", show(&refused)); + assert!( + !t.path().join("contracts/contracts.nt").exists(), + "the tracked file is untouched" + ); + let out_s = s(&out); + let mut with_out = args.to_vec(); + with_out.extend_from_slice(&["--out", &out_s]); + let r = pv(t.path(), &with_out); + assert_eq!(r.code, 0, "{}", show(&r)); + let v = json_of(&r); + assert_eq!(v["release"]["cells"], 48); + let nt = std::fs::read_to_string(&out).expect("written"); + assert!(nt.contains("release/Cell"), "the cells are in the graph"); + assert!(nt.contains("release/KernelDiffReceipt")); + assert!(!nt.contains("_:"), "no blank nodes (R-15)"); + assert!(!t.path().join("contracts/contracts.nt").exists()); +} + +fn tokenizer(t: &Path) -> PathBuf { + t.join("evidence/dogfood/tokenizer/0.69.1/parity.json") +} + +const TOK_CELL: &str = "release-tokenizer/0.69.1/tiny-qwen35.gguf"; + +#[test] +fn one_token_id_that_disagrees_with_llama_cpp_is_red_naming_the_model() { + // #3726: apr mapped every non-ASCII byte to id 0 and `apr parity` could not see it (CPU and GPU share it) + let t = green(); + edit(&tokenizer(t.path()), |v| { + v["rows"][0]["mismatches"] = 1.into() + }); + let v = assert_red_naming(&gate(t.path(), &[]), TOK_CELL); + assert!(v["findings"].to_string().contains("parityExact")); +} + +#[test] +fn a_tokenizer_comparison_over_nothing_or_a_broken_round_trip_or_no_receipt_is_red() { + let t = green(); + edit(&tokenizer(t.path()), |v| { + v["rows"][0]["tokens_compared"] = 0.into() + }); + assert_red_naming(&gate(t.path(), &[]), TOK_CELL); + + let t = green(); + edit(&tokenizer(t.path()), |v| { + v["rows"][0]["roundtrip_ok"] = false.into() + }); + let v = assert_red_naming(&gate(t.path(), &[]), TOK_CELL); + assert!(v["findings"].to_string().contains("roundTrip")); + + let t = green(); + std::fs::remove_file(tokenizer(t.path())).expect("rm"); + assert_red_naming(&gate(t.path(), &[]), TOK_CELL); +} + +// ── the memory rule (#3710, operator decision "a") ────────────────────────────────────────────────────── +// weights 16 GB + 1 MB/token KV + 1 GB workspace: `4k` (21.1 GB) fits the 24 GB card; `consumer-max` (25 GB) and +// `declared` (27 GB) do not, and are owed on lambda as honest pre-load refusals — and on gx10 (119 GB) as passes. + +const GB: u64 = 1_000_000_000; + +fn memory_split(t: &Path) { + for (h, gpu) in [("lambda", 24 * GB), ("gx10", 119 * GB)] { + edit(&models(t, h), |v| { + v["gpu_mem_total_bytes"] = gpu.into(); + let item = &mut v["inventory"][0]; + item["weights_bytes"] = (16 * GB).into(); + item["kv_bytes_per_token"] = 1_000_000u64.into(); + item["workspace_bytes"] = GB.into(); + }); + } + edit(&models(t, "lambda"), |v| { + for c in v["cells"].as_array_mut().expect("cells") { + let need = match c["context"].as_str() { + Some("consumer-max") => 25 * GB, + Some("declared") => 27 * GB, + _ => continue, + }; + c["verdict"] = "refused".into(); + c["required_bytes"] = need.into(); + c["available_bytes"] = (24 * GB).into(); + c["reason"] = + "pre-load refusal: KV for this context needs more than the device holds".into(); + } + }); +} + +#[test] +fn a_27b_that_refuses_honestly_on_lambda_and_passes_on_gx10_is_green() { + let t = green(); + memory_split(t.path()); + let r = gate(t.path(), &[]); + assert_eq!(r.code, 0, "{}", show(&r)); + let v = json_of(&r); + assert_eq!( + v["release"]["refusal_cells"], 16, + "4 verbs × 2 modes × (consumer-max, declared) on lambda" + ); + assert_eq!(v["release"]["cells"], 48, "a refusal is still an owed cell"); +} + +#[test] +fn the_gx10_cell_that_covers_the_rung_missing_is_red() { + let t = green(); + memory_split(t.path()); + edit(&models(t.path(), "gx10"), |v| { + let i = row_index(v, "chat", "off", "declared"); + v["cells"].as_array_mut().expect("cells").remove(i); + }); + assert_red_naming( + &gate(t.path(), &[]), + &cell("gx10", "chat", "off", "declared"), + ); +} + +#[test] +fn an_oom_mid_run_where_a_refusal_is_owed_is_red() { + let t = green(); + memory_split(t.path()); + edit(&models(t.path(), "lambda"), |v| { + let r = row(v, "serve", "on", "declared"); + r["verdict"] = "error".into(); + r["reason"] = "CUDA_ERROR_OUT_OF_MEMORY after 41 s of prefill".into(); + r.as_object_mut().expect("row").remove("required_bytes"); + r.as_object_mut().expect("row").remove("available_bytes"); + }); + let v = assert_red_naming( + &gate(t.path(), &[]), + &cell("lambda", "serve", "on", "declared"), + ); + assert!(v["findings"] + .to_string() + .contains("release-readiness-v1.refusal")); +} + +#[test] +fn a_refusal_whose_arithmetic_shows_no_shortfall_is_red() { + let t = green(); + memory_split(t.path()); + edit(&models(t.path(), "lambda"), |v| { + row(v, "code", "off", "consumer-max")["required_bytes"] = (20 * GB).into(); + }); + let v = assert_red_naming( + &gate(t.path(), &[]), + &cell("lambda", "code", "off", "consumer-max"), + ); + assert!(v["findings"].to_string().contains("arithmeticNamed")); +} + +#[test] +fn a_cell_that_fits_by_the_arithmetic_but_is_missing_is_red() { + let t = green(); + memory_split(t.path()); + edit(&models(t.path(), "lambda"), |v| { + let i = row_index(v, "run", "on", "4k"); + v["cells"].as_array_mut().expect("cells").remove(i); + }); + assert_red_naming(&gate(t.path(), &[]), &cell("lambda", "run", "on", "4k")); +} + +#[test] +fn a_rung_that_fits_on_no_required_host_is_red_naming_the_rung() { + let t = green(); + memory_split(t.path()); + edit(&models(t.path(), "gx10"), |v| { + v["gpu_mem_total_bytes"] = (24 * GB).into(); + }); + assert_red_naming( + &gate(t.path(), &[]), + &format!("release-coverage/0.69.1/{MODEL}/declared"), + ); +} + +#[test] +fn a_refusal_because_a_co_tenant_holds_memory_is_red() { + // required 23 GB ≤ the card's 24 GB total, refused because only 20 GB were free: a busy GPU, not the + // arithmetic — it must not shrink what the release proves (#3712, aprender-62) + let t = green(); + memory_split(t.path()); + edit(&models(t.path(), "lambda"), |v| { + let r = row(v, "chat", "off", "consumer-max"); + r["required_bytes"] = (23 * GB).into(); + r["available_bytes"] = (20 * GB).into(); + }); + let v = assert_red_naming( + &gate(t.path(), &[]), + &cell("lambda", "chat", "off", "consumer-max"), + ); + assert!(v["findings"].to_string().contains("arithmeticNamed")); +} + +#[test] +fn a_model_whose_template_always_opens_think_owes_only_the_on_cells() { + // Qwen3-style prefill that always opens (#3723): ON only — the OFF cells are not owed, so removing + // every OFF row leaves the release green, and removing an ON row does not + let t = green(); + for h in ["lambda", "gx10"] { + edit(&models(t.path(), h), |v| { + v["inventory"][0]["thinking_modes"] = + serde_json::from_str::(r#"["on"]"#).expect("modes"); + v["inventory"][0]["thinking_markers"] = + serde_json::from_str::(r#"[""]"#).expect("markers"); + let cells = v["cells"].as_array_mut().expect("cells"); + cells.retain(|c| c["thinking"] == "on"); + }); + } + let r = gate(t.path(), &[]); + assert_eq!(r.code, 0, "{}", show(&r)); + assert_eq!( + json_of(&r)["release"]["cells"], + 24, + "2 hosts × 4 verbs × ON × 3 rungs" + ); + edit(&models(t.path(), "gx10"), |v| { + let i = row_index(v, "code", "on", "4k"); + v["cells"].as_array_mut().expect("cells").remove(i); + }); + assert_red_naming(&gate(t.path(), &[]), &cell("gx10", "code", "on", "4k")); +} diff --git a/crates/aprender-contracts/src/lint/mod.rs b/crates/aprender-contracts/src/lint/mod.rs index db3b347077..88c8c70cad 100644 --- a/crates/aprender-contracts/src/lint/mod.rs +++ b/crates/aprender-contracts/src/lint/mod.rs @@ -244,6 +244,9 @@ pub enum GateExtra { /// ONT-4b2: Lean theorems extracted, and contract `lean_theorem:` references naming none of them. lean_statements: usize, lean_refs_unresolved: usize, + /// aprender#3715: what `extract:release-evidence` derived — absent unless a release subject was given. + #[serde(skip_serializing_if = "Option::is_none")] + release: Option>, }, } @@ -646,11 +649,22 @@ pub enum NamedGateOutcome { /// report a verdict whose precondition nobody checked — `UnknownGate` is the honest answer, not a silent pass. #[must_use] pub fn run_named_gate(contract_dir: &Path, name: &str) -> NamedGateOutcome { + run_named_gate_with(contract_dir, name, &shapes_gate::ShapesOptions::default()) +} + +/// [`run_named_gate`] with the shapes gate's `--shape` / `--release-*` options (aprender#3715). +pub fn run_named_gate_with( + contract_dir: &Path, + name: &str, + shapes_opts: &shapes_gate::ShapesOptions, +) -> NamedGateOutcome { match name { "relations" => { NamedGateOutcome::Relations(relations_gate::run_relations_gate(contract_dir)) } - "shapes" => NamedGateOutcome::Shapes(shapes_gate::run_shapes_gate(contract_dir)), + "shapes" => { + NamedGateOutcome::Shapes(shapes_gate::run_shapes_gate_with(contract_dir, shapes_opts)) + } "sigma" => NamedGateOutcome::Sigma(sigma_gate::run_sigma_gate(contract_dir)), "validate" => { let (contracts, parse_errors) = load_contracts(contract_dir); diff --git a/crates/aprender-contracts/src/lint/shapes_gate.rs b/crates/aprender-contracts/src/lint/shapes_gate.rs index 5739d10db9..ac6c6242a9 100644 --- a/crates/aprender-contracts/src/lint/shapes_gate.rs +++ b/crates/aprender-contracts/src/lint/shapes_gate.rs @@ -33,8 +33,10 @@ use std::path::Path; use std::time::Instant; use crate::ontology::arming::ArmedShapes; +use crate::ontology::extract::release_inputs::Subject; use crate::ontology::extract::{ - self, apr_model, code, gguf, json, lean, parity_receipt, pv_contract, ExtractFailure, + self, apr_model, code, gguf, json, lean, parity_receipt, pv_contract, release_evidence, + ExtractFailure, }; use crate::ontology::rdf::{iri, Graph, Term, RDF_TYPE}; use crate::ontology::receipts; @@ -196,28 +198,60 @@ fn w3c_checked(shapes_n: usize, focus_nodes_n: usize) -> Result`: grade only that shape FAMILY — the shape `` and every shape whose id starts with + /// `.` — and arm it whatever `armed_shapes` says: a caller that names a shape is asking for its verdict, + /// and "computed and reported" would let a release gate print violations and exit 0. + pub only: Option, + /// `--release-*`: the release subject `extract:release-evidence` reads. `None` → no release graph at all. + pub release: Option, +} + +/// The shapes `only` selects, in declaration order; a name that selects nothing is the CALLER's error (exit 3), +/// never an empty pass. +fn select_family( + declared: Vec<(NodeShape, String)>, + only: Option<&str>, +) -> Result, ShapeError> { + let Some(id) = only else { + return Ok(declared.into_iter().map(|(s, _)| s).collect()); + }; + let prefix = format!("{id}."); + let family: Vec = declared + .into_iter() + .map(|(s, _)| s) + .filter(|s| s.id == id || s.id.starts_with(&prefix)) + .collect(); + if family.iter().any(|s| s.id == id) { + Ok(family) + } else { + Err(ShapeError::Malformed { + shape: id.to_string(), + what: "--shape names no declared shape".into(), + }) + } +} + /// Run the gate over `contract_dir`. #[must_use] pub fn run_shapes_gate(contract_dir: &Path) -> ShapesOutcome { + run_shapes_gate_with(contract_dir, &ShapesOptions::default()) +} + +/// [`run_shapes_gate`] with `--shape` / `--release-*` (aprender#3715). +#[must_use] +pub fn run_shapes_gate_with(contract_dir: &Path, opts: &ShapesOptions) -> ShapesOutcome { let start = Instant::now(); - let (declared, checked) = match collect_shapes(contract_dir) { + let (shapes, arming, checked) = match prepare(contract_dir, opts) { Ok(x) => x, - Err(e) => return ShapesOutcome::Unsupported(e), - }; - if declared.is_empty() { - return ShapesOutcome::NoShapes { - contracts_checked: checked, - }; - } - let arming = match armed_shapes_of(contract_dir) { - Ok(a) => a, - Err(e) => return ShapesOutcome::Unsupported(e), + Err(answer) => return answer, }; - let shapes: Vec = declared.into_iter().map(|(s, _)| s).collect(); // ONE walk (R-18): every extractor, the json documents, the ladder receipts joined to the rungs — the same - // graph `pv extract` writes. - let extraction = match extract::all(contract_dir) { + // graph `pv extract` writes — plus, when a release subject is given, the release evidence (#3715). + let extraction = match extract::all_with(contract_dir, opts.release.as_ref()) { Ok(x) => x, Err(e) => return ShapesOutcome::ExtractFailed(e), }; @@ -225,17 +259,15 @@ pub fn run_shapes_gate(contract_dir: &Path) -> ShapesOutcome { if let Some(refusal) = parity_refusal(&extraction.parity, shapes.len()) { return refusal; } - let graph = extraction.graph; - let needs_receipts = shapes.iter().any(|s| { - s.properties - .iter() - .any(|p| p.resolves.as_deref() == Some("receipt")) - }); + let graph = &extraction.graph; - let (mut report, plant_violations) = validate_with_plant(&graph, &shapes, &arming); + let (mut report, plant_violations) = validate_with_plant(graph, &shapes, &arming); + if opts.only.is_some() { + order_by_family(&mut report, &shapes); + } carry_extract_warnings(&mut report, &extraction.warnings); let pc_extract = extract_controls(); - let unmeasured = needs_receipts && extraction.receipts.is_empty(); + let unmeasured = needs_receipts(&shapes) && extraction.receipts.is_empty(); if let Some(d) = decline( shapes.len(), &report, @@ -253,53 +285,18 @@ pub fn run_shapes_gate(contract_dir: &Path) -> ShapesOutcome { let counted = findings_of( &report, &arming, - &graph, + graph, &extraction.gguf, &extraction.apr_model, ); let passed = counted.violations == 0; - let verdict = if !passed { - Verdict::Fail - } else if counted.warnings > 0 { - Verdict::Unknown(Reason::Warn) - } else { - Verdict::Pass - }; - let mut by_shape: Vec = shapes - .iter() - .map(|s| { - let n = shapes::instances_closed(&graph, &s.target_class).len(); - format!("{}={}", s.id, n) - }) - .collect(); - by_shape.sort(); + let verdict = verdict_of(&counted); + let by_shape = by_shape(graph, &shapes); let (armed_names, not_armed): (Vec, Vec) = shapes .iter() .map(|s| s.id.clone()) .partition(|id| arming.is_armed(id)); - let by_entity_type: BTreeMap = [ - ( - "pv-contract", - graph - .instances_of(&crate::ontology::rdf::ont("Contract")) - .len(), - ), - ( - "gguf", - extraction.gguf.rungs.len() + extraction.gguf.files_read, - ), - ("apr-model", extraction.apr_model.files_read), - // ONT-4c3: registered in Σ and implemented, so it is counted here like every other entity - // type. Without this key a probe asking `by_entity_type["parity-receipt"]` reads ABSENT — - // and an absent key is not zero, so a consumer that treats it as one measures nothing and - // calls it a pass. The same shape as #3610, one map over. - ("parity-receipt", extraction.parity.records), - ("code", extraction.code.symbols), - ("lean", extraction.lean.statements), - ] - .into_iter() - .map(|(k, v)| (k.to_string(), v)) - .collect(); + let by_entity_type = by_entity_type(&extraction); let duration = u64::try_from(start.elapsed().as_millis()).unwrap_or(u64::MAX); let result = GateResult { name: "shapes".into(), @@ -342,6 +339,7 @@ pub fn run_shapes_gate(contract_dir: &Path) -> ShapesOutcome { symbols_unresolved: extraction.code.unresolved, lean_statements: extraction.lean.statements, lean_refs_unresolved: extraction.lean.refs_unresolved.len(), + release: extraction.release.clone().map(Box::new), }), }; ShapesOutcome::Ran { @@ -350,6 +348,99 @@ pub fn run_shapes_gate(contract_dir: &Path) -> ShapesOutcome { } } +/// The shape set this run grades and its arming, or the answer that stands in for a verdict (unsupported or +/// malformed shapes, no shapes at all, a `--shape` naming nothing). A named family is armed whatever +/// `armed_shapes` says (#3715). +fn prepare( + contract_dir: &Path, + opts: &ShapesOptions, +) -> Result<(Vec, ArmedShapes, usize), ShapesOutcome> { + let (declared, checked) = collect_shapes(contract_dir).map_err(ShapesOutcome::Unsupported)?; + if declared.is_empty() { + return Err(ShapesOutcome::NoShapes { + contracts_checked: checked, + }); + } + let baseline = armed_shapes_of(contract_dir).map_err(ShapesOutcome::Unsupported)?; + let arming = if opts.only.is_some() { + ArmedShapes::All + } else { + baseline + }; + let shapes = + select_family(declared, opts.only.as_deref()).map_err(ShapesOutcome::Unsupported)?; + Ok((shapes, arming, checked)) +} + +/// A named family reports in the order its contract declares it: the release, its hosts, its context, its +/// kernels, THEN its model cells (#3715: a kernel that diverges across arches is named before any model cell it +/// breaks). Stable, so the validator's order holds within a shape. +fn order_by_family(report: &mut Report, shapes: &[NodeShape]) { + let rank = |id: &str| shapes.iter().position(|s| s.id == id).unwrap_or(usize::MAX); + report.results.sort_by_key(|r| rank(&r.shape)); +} + +/// Does any shape resolve its values against the tracked ladder receipts? +fn needs_receipts(shapes: &[NodeShape]) -> bool { + shapes.iter().any(|s| { + s.properties + .iter() + .any(|p| p.resolves.as_deref() == Some("receipt")) + }) +} + +/// Violations fail; warnings alone are `Unknown{Warn}`, never a pass; nothing is `Pass`. +fn verdict_of(counted: &Counted) -> Verdict { + if counted.violations > 0 { + Verdict::Fail + } else if counted.warnings > 0 { + Verdict::Unknown(Reason::Warn) + } else { + Verdict::Pass + } +} + +/// `shape=focus-count` per shape, sorted. +fn by_shape(graph: &Graph, shapes: &[NodeShape]) -> Vec { + let mut out: Vec = shapes + .iter() + .map(|s| { + let n = shapes::instances_closed(graph, &s.target_class).len(); + format!("{}={}", s.id, n) + }) + .collect(); + out.sort(); + out +} + +/// Focus nodes each extractor produced. +fn by_entity_type(extraction: &extract::Extraction) -> BTreeMap { + [ + ( + "pv-contract", + extraction + .graph + .instances_of(&crate::ontology::rdf::ont("Contract")) + .len(), + ), + ( + "gguf", + extraction.gguf.rungs.len() + extraction.gguf.files_read, + ), + ("apr-model", extraction.apr_model.files_read), + // ONT-4c3: registered in Σ and implemented, so it is counted here like every other entity + // type. Without this key a probe asking `by_entity_type["parity-receipt"]` reads ABSENT — + // and an absent key is not zero, so a consumer that treats it as one measures nothing and + // calls it a pass. The same shape as #3610, one map over. + ("parity-receipt", extraction.parity.records), + ("code", extraction.code.symbols), + ("lean", extraction.lean.statements), + ] + .into_iter() + .map(|(k, v)| (k.to_string(), v)) + .collect() +} + /// The answers that are not corpus verdicts, in the order they are asked: no focus node, receipts needed and /// none tracked, the plant silent, an extractor control silent. `None` when the corpus gets a verdict. fn decline( @@ -399,6 +490,8 @@ fn extract_controls() -> BTreeMap { "parity-receipt", parity_receipt::positive_control(&parity_receipt::control_sample()), ), + // aprender#3715: drawn every run, subject or not — a cell owed without a receipt stays a node + ("release-evidence", release_evidence::positive_control()), ] .into_iter() .map(|(k, fired)| { diff --git a/crates/aprender-contracts/src/ontology/extract/mod.rs b/crates/aprender-contracts/src/ontology/extract/mod.rs index 3f2c54939c..da7e46f0bc 100644 --- a/crates/aprender-contracts/src/ontology/extract/mod.rs +++ b/crates/aprender-contracts/src/ontology/extract/mod.rs @@ -21,6 +21,8 @@ pub mod json; pub mod lean; pub mod parity_receipt; pub mod pv_contract; +pub mod release_evidence; +pub mod release_inputs; /// Every extractor's output over `contract_dir`, plus the input-side warnings the extractors chose to carry /// rather than hide (a torn JSONL line), plus what the model extractors and the receipt resolver counted. @@ -44,6 +46,8 @@ pub struct Extraction { pub lean: lean::LeanStats, /// ONT-4c3: the logit-parity receipts under `evidence/parity/**`, and the files this extractor refused. pub parity: parity_receipt::ParityStats, + /// aprender#3715: the release evidence — `None` unless a release subject was given (an ordinary PR has none). + pub release: Option, } /// What a walk could not do. Every variant is the DECLARATION's fault (exit 3), never a corpus verdict. @@ -53,6 +57,8 @@ pub enum ExtractFailure { Json(json::ExtractError), /// A ladder receipt under `evidence/dogfood/models/` is unreadable or carries a foreign schema. Receipt(receipts::ReceiptError), + /// aprender#3715: the release subject is malformed, or a release input is unreadable or foreign. + Release(release_inputs::ReleaseError), } impl std::fmt::Display for ExtractFailure { @@ -60,6 +66,7 @@ impl std::fmt::Display for ExtractFailure { match self { Self::Json(e) => write!(f, "{e}"), Self::Receipt(e) => write!(f, "receipt {e}"), + Self::Release(e) => write!(f, "{e}"), } } } @@ -70,6 +77,15 @@ impl std::error::Error for ExtractFailure {} /// contract dir's parent — the repo root, the same base `ont:file` uses); then the `gguf` and `apr-model` /// entities and the ladder receipts joined to the rungs. `Err` is a declaration's fault. pub fn all(contract_dir: &Path) -> Result { + all_with(contract_dir, None) +} + +/// [`all`], plus — when `release` names a release subject — `extract:release-evidence` (aprender#3715): the +/// release's receipts as the graph `release-readiness-v1` grades. Same one walk (R-18). +pub fn all_with( + contract_dir: &Path, + release: Option<&release_inputs::Subject>, +) -> Result { let mut out = Extraction { graph: pv_contract::extract(contract_dir), ..Extraction::default() @@ -91,6 +107,12 @@ pub fn all(contract_dir: &Path) -> Result { out.code = code::extract(contract_dir, &mut out.graph); out.lean = lean::extract(contract_dir, &mut out.graph); out.parity = parity_receipt::extract(root, &mut out.graph); + if let Some(subject) = release { + out.release = Some( + release_evidence::extract(&mut out.graph, contract_dir, subject) + .map_err(ExtractFailure::Release)?, + ); + } Ok(out) } diff --git a/crates/aprender-contracts/src/ontology/extract/release_evidence.rs b/crates/aprender-contracts/src/ontology/extract/release_evidence.rs new file mode 100644 index 0000000000..2db3277a29 --- /dev/null +++ b/crates/aprender-contracts/src/ontology/extract/release_evidence.rs @@ -0,0 +1,1119 @@ +//! ONT-001 §3.7 — `extract:release-evidence` (aprender#3715): the release's receipts become a graph in which +//! every obligation is a NODE, so absence is a missing edge that `minCount 1` rejects — never a row nobody read. +//! +//! Operator, 2026-09-21: *"you need to improve our dogfood process and pv SHACL so leaks don't exist pre-release."* +//! Every 0.69.0 leak was absence read as conformance (a rung marked optional, a GPU error read as a skip, an +//! inherited dogfood, models on a host that were not on the ladder). So the universe is DERIVED here, from what +//! was measured, and each (thing × host × verb × context) the release must prove gets a node of its own: +//! +//! | node | IRI | from | +//! |---|---|---| +//! | `release:Release` | `release-subject/` | the CLI's `--release-*` flags (never inferred) | +//! | `release:DogfoodReceipt` | `release-dogfood/` | `--dogfood-receipt`: the file R5 judged | +//! | `release:Host` | `release-host//` | every `ladder.hosts[]` entry with `required: true` | +//! | `release:HostReceipt` | `release-receipt///` | the host's `apr-model-ladder-receipt` (v2, #3712) | +//! | `release:Model` | `model/` (shared with `extract:gguf`) | the host's measured `inventory[]` ∪ the ladder's cuda rungs listed for it | +//! | `release:Verb` | `release-verb/` | run, chat, serve, code | +//! | `release:ContextRung` | `release-context/` | `evidence/release/context-rungs.json`; `consumer-max` is COMPUTED from the consumer records, and `declared` is each model's own GGUF `context_length` | +//! | `release:Cell` | `release-cell/////think-/` | model × verb × thinking × rung, per host — the focus of `release-readiness-v1` | +//! | `release:CellReceipt` | `release-row////` | each `cells[]` row that keys onto a cell | +//! | `release:RefusalCell` | as `release:Cell` | a cell that does not FIT the host by the declared memory arithmetic: owed as an honest pre-load refusal (#3710 rule) | +//! | `release:RungCoverage` | `release-coverage///` | every (model, rung) — owed on ≥ 1 required host | +//! | `release:KernelCell` | `release-kernel////` | every (kernel, quant) on any required host's dispatch path × every required host | +//! | `release:KernelDiffReceipt` | `release-krow////` | each kernel-diff row | +//! | `release:TokenizerCell` | `release-tokenizer//` | every distinct model in any required host's universe (#3726) | +//! | `release:TokenizerParityReceipt` | `release-trow///` | each tokenizer-parity row keyed onto a model | +//! +//! The cell universe per model (#3710, operator: "MORE than enough tokens … chat with and without thinking, ditto +//! run, ditto code"): every verb × thinking ∈ {on, off} (only `off` when the host MEASURED no thinking mode) × +//! every declared rung up to the model's own `context_length`, plus `declared` itself. A model whose length was +//! not measured owes every rung, and the `.model` shape names the missing length. +//! +//! Rust computes only what SHACL-Core cannot say (the universe, the join, `fresh`, `contextMet`, `thinkOk`, +//! `answered`, `withinBound`); +//! the shapes in `contracts/release-readiness-v1.yaml` say what a pass IS, and the fixtures under +//! `tests/fixtures/ont/release-*` falsify both. Nothing here runs unless a release subject is given: an ordinary +//! PR has no release, and inventing one would grade a tree against a commit it is not. + +use std::collections::{BTreeMap, BTreeSet}; +use std::path::Path; + +use crate::ontology::extract::gguf::{self, Rung}; +use crate::ontology::extract::pv_contract::{self, scalar}; +use crate::ontology::extract::release_inputs::{ + self as inputs, Consumer, ContextRung, Dogfood, KernelReceipt, ReleaseError, Subject, + TokReceipt, +}; +use crate::ontology::rdf::{iri, iri_path, Graph, Term, ONT_BASE, RDF_TYPE}; +use crate::ontology::receipts::{self, CellRow, Receipt}; + +/// The verbs every model must pass on every required host (#3710: "run, chat, serve and code"). +pub const VERBS: [&str; 4] = ["run", "chat", "serve", "code"]; +/// Both thinking modes (#3710 bar raise: "chat with and without thinking, ditto run, ditto code"). +pub const THINKING: [&str; 2] = ["on", "off"]; +/// The per-model rung: the model's own declared context length, read from its GGUF on the host. +pub use crate::ontology::extract::release_inputs::DECLARED; + +/// The vocabulary root: `https://ont.paiml.dev/v1alpha1/release/` (`release:` in a shape). +#[must_use] +pub fn rel(name: &str) -> String { + format!("{ONT_BASE}release/{name}") +} + +/// What one run derived, for the gate's report. +#[derive(Debug, Clone, Default, PartialEq, Eq, serde::Serialize)] +pub struct ReleaseStats { + pub required_hosts: usize, + pub model_receipts: usize, + pub kernel_receipts: usize, + /// Distinct (host, model) pairs in the universe. + pub models: usize, + pub cells: usize, + /// Cells with at least one keyed row (a row is not a pass — the shape decides that). + pub cells_with_row: usize, + pub kernel_cells: usize, + /// One per distinct model (sha256) in any required host's universe (#3726). + pub tokenizer_cells: usize, + /// Cells owed as an honest pre-load refusal because they do not fit the host (counted inside `cells`). + pub refusal_cells: usize, + /// (model, rung) pairs that must be owed on ≥ 1 required host (#3710 rule). + pub rung_coverage: usize, + pub context_rungs: usize, + /// Rows that keyed onto no cell (a model outside the universe, an unknown verb or rung): counted, not graded. + pub orphan_rows: usize, + /// Every cell, named `///think-/` — so a Pass reads "N/N cells", each named + /// (#3715 done_when 5), and a reader can see which cells a verdict covered, not only how many. + pub cell_names: Vec, +} + +/// A `ladder.hosts[]` entry with `required: true`. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct HostDecl { + pub id: String, + pub cc: String, +} + +/// `ladder.cells.long_rungs_for` (#3712 amendment 2): the families that owe every long rung, and one +/// representative file per other architecture. Absent from the ladder → every model owes the long rungs. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct LongRungsFor { + pub families: BTreeSet, + /// `general.architecture` → the one file basename that owes the long rungs for that arch. + pub representatives: BTreeMap, +} + +impl LongRungsFor { + fn owes(&self, arch: Option<&str>, file: &str) -> bool { + arch.is_some_and(|a| { + self.families.contains(a) || self.representatives.get(a).is_some_and(|f| f == file) + }) + } +} + +/// What the ladder contract(s) declare for the release. +#[derive(Debug, Clone, Default)] +pub struct Ladder { + pub hosts: Vec, + pub rungs: Vec, + pub long_rungs_for: Option, +} + +/// One model the release must prove on one host. +#[derive(Debug, Clone, PartialEq, Eq)] +struct ModelInfo { + file: String, + arch: Option, + quant: Option, + context_length: Option, + thinking_modes: Option>, + thinking_markers: Option>, + /// The producer's echo, compared with `owes_long`. + owes_long_echo: Option, + /// Derived here, from the ladder contract. + owes_long: bool, + /// (weights, KV bytes per token, workspace) — the declared memory arithmetic, when all three were measured. + mem: Option<(u64, u64, u64)>, + kv_dtype: Option, +} + +impl ModelInfo { + /// Does a `tokens`-long context fit on a host with `gpu` bytes (#3710 rule, operator "a")? `None` when any + /// term is unmeasured — and an unknown fit leaves the cell OWED, the stricter reading. + fn fits(&self, gpu: Option, tokens: Option) -> Option { + Some(self.required_bytes(tokens?)? <= gpu?) + } + + fn required_bytes(&self, tokens: u64) -> Option { + let (w, kv, ws) = self.mem?; + Some( + w.saturating_add(kv.saturating_mul(tokens)) + .saturating_add(ws), + ) + } + + /// `off` always; `on` unless the host MEASURED that the model has no thinking mode. + /// The modes the model owes: its measured `thinking_modes` (#3723); BOTH when unmeasured — and both when the + /// measured list names no valid mode, so a malformed list cannot make a model vanish (the `.model` shape names + /// it as a contradiction instead). + fn modes(&self) -> Vec<&'static str> { + let owed: Vec<&'static str> = match &self.thinking_modes { + Some(ms) => THINKING + .iter() + .copied() + .filter(|t| ms.iter().any(|m| m == t)) + .collect(), + None => THINKING.to_vec(), + }; + if owed.is_empty() { + THINKING.to_vec() + } else { + owed + } + } + + /// The rungs this model owes: every declared rung whose size fits its declared length (all of them while the + /// length is unmeasured, or the rung's own size is), then `declared`, whose target IS the length. + fn rungs<'r>(&self, rungs: &'r [ContextRung]) -> Vec<(&'r str, Option)> { + let fits = |r: &&ContextRung| match (r.tokens, self.context_length) { + (Some(t), Some(l)) => t <= l, + _ => true, + }; + let mut out: Vec<(&str, Option)> = rungs + .iter() + .filter(|r| self.owes_long || !r.long) + .filter(fits) + .map(|r| (r.id.as_str(), r.tokens)) + .collect(); + if self.owes_long { + out.push((DECLARED, self.context_length)); + } + out + } + + /// `thinking_modes` must be what its template evidence can prove (#3723's derivation): `enable_thinking` + /// among the markers ⇔ both modes; no marker at all ⇒ `["off"]`; a list naming no valid mode is itself a + /// contradiction. (`["on"]` — a prompt that always opens `` — is the producer's to prove; pv checks only + /// what the markers decide.) `None` when both are unmeasured or they agree. + fn thinking_contradiction(&self) -> Option { + let modes = self.thinking_modes.as_ref()?; + let valid: Vec<&String> = modes + .iter() + .filter(|m| THINKING.contains(&m.as_str())) + .collect(); + let both = THINKING.iter().all(|t| modes.iter().any(|m| m == t)); + let markers = self.thinking_markers.as_deref().unwrap_or_default(); + let switch = markers.iter().any(|m| m == "enable_thinking"); + let bad = valid.is_empty() + || valid.len() != modes.len() + || switch != both + || (self.thinking_markers.is_some() && markers.is_empty() && modes != &["off"]); + bad.then(|| { + format!( + "thinking_modes = [{}] but thinking_markers = [{}]", + modes.join(", "), + markers.join(", ") + ) + }) + } +} + +/// One required host, with everything derived for it. +struct HostView<'a> { + decl: &'a HostDecl, + node: String, + receipts: Vec<&'a Receipt>, + kernels: Vec<&'a KernelReceipt>, + /// sha256 → model, the host's universe. + models: BTreeMap, + /// The host's measured GPU memory (the fit arithmetic's right-hand side). + gpu_mem: Option, +} + +/// The required hosts, the rungs and `cells.long_rungs_for`, from every contract that carries a `ladder:` block. +#[must_use] +pub fn ladder_of(contract_dir: &Path) -> Ladder { + let mut out = Ladder::default(); + for (stem, _rel, doc) in pv_contract::documents(contract_dir) { + let Some(l) = doc.get("ladder") else { + continue; + }; + let decls = l.get("hosts").and_then(serde_yaml::Value::as_sequence); + for h in decls.into_iter().flatten() { + if h.get("required").and_then(serde_yaml::Value::as_bool) == Some(true) { + out.hosts.push(HostDecl { + id: scalar(h.get("id")).unwrap_or_default(), + cc: scalar(h.get("cc")).unwrap_or_default(), + }); + } + } + out.rungs.extend(gguf::rungs_of(&stem, &doc)); + if let Some(lr) = l.get("cells").and_then(|c| c.get("long_rungs_for")) { + out.long_rungs_for = Some(long_rungs_for(lr)); + } + } + out.hosts.sort_by(|a, b| a.id.cmp(&b.id)); + out.hosts.dedup_by(|a, b| a.id == b.id); + out +} + +fn long_rungs_for(v: &serde_yaml::Value) -> LongRungsFor { + let families = v + .get("families") + .and_then(serde_yaml::Value::as_sequence) + .into_iter() + .flatten() + .filter_map(|f| scalar(Some(f))) + .collect(); + let representatives = v + .get("representatives") + .and_then(serde_yaml::Value::as_mapping) + .into_iter() + .flatten() + .filter_map(|(k, f)| Some((scalar(Some(k))?, scalar(Some(f))?))) + .collect(); + LongRungsFor { + families, + representatives, + } +} + +/// Read the release's evidence and write its graph into `g`. `Err` is a declaration's fault (exit 3). +pub fn extract( + g: &mut Graph, + contract_dir: &Path, + subject: &Subject, +) -> Result { + let root = super::repo_root(contract_dir); + let ladder = ladder_of(contract_dir); + let (ctx_rungs, consumers) = inputs::read_context_rungs(&root)?.unwrap_or_default(); + let model_receipts = + receipts::read_dir(&subject.model_dir(&root), &root).map_err(ReleaseError::Receipt)?; + let kernel_receipts = inputs::read_kernel_receipts(&subject.kernel_dir(&root), &root)?; + let tok_receipts = inputs::read_tokenizer_receipts(&subject.tokenizer_dir(&root), &root)?; + let dogfood = subject + .dogfood_receipt + .as_deref() + .map(inputs::read_dogfood) + .transpose()?; + Ok(build( + g, + subject, + &Inputs { + ladder: &ladder, + ctx_rungs: &ctx_rungs, + consumers: &consumers, + model_receipts: &model_receipts, + kernel_receipts: &kernel_receipts, + tok_receipts: &tok_receipts, + dogfood: dogfood.as_ref(), + }, + )) +} + +/// Everything [`build`] reads, already parsed — so the positive control drives the same code in memory. +pub struct Inputs<'a> { + pub ladder: &'a Ladder, + pub ctx_rungs: &'a [ContextRung], + pub consumers: &'a [Consumer], + pub model_receipts: &'a [Receipt], + pub kernel_receipts: &'a [KernelReceipt], + pub tok_receipts: &'a [TokReceipt], + pub dogfood: Option<&'a Dogfood>, +} + +/// The release graph from parsed inputs: pure, no filesystem (R-15). +pub fn build(g: &mut Graph, subject: &Subject, i: &Inputs<'_>) -> ReleaseStats { + let hosts = &i.ladder.hosts; + let mut stats = ReleaseStats { + required_hosts: hosts.len(), + model_receipts: i.model_receipts.len(), + kernel_receipts: i.kernel_receipts.len(), + context_rungs: i.ctx_rungs.len(), + ..ReleaseStats::default() + }; + emit_release(g, subject, hosts, i.ctx_rungs, i.dogfood); + emit_vocabulary_nodes(g, i.ctx_rungs, i.consumers); + let views: Vec> = hosts + .iter() + .map(|h| host_view(h, subject, i.model_receipts, i.kernel_receipts, i.ladder)) + .collect(); + let mut coverage = Coverage::new(); + for v in &views { + emit_host(g, subject, v, &mut stats); + emit_cells(g, subject, v, i.ctx_rungs, &mut coverage, &mut stats); + } + emit_coverage(g, subject, &coverage, &mut stats); + emit_kernels(g, subject, &views, &mut stats); + emit_tokenizer(g, subject, &views, i.tok_receipts, &mut stats); + stats +} + +/// The positive control (R-3, PMAT-3704): drawn on EVERY gate run, with or without a release subject. One +/// required host holds one model that owes one cell; the SAMPLE receipt carries that cell's fresh Pass row, the +/// PLANTED one omits it. Fires iff the sample's cell has exactly one fresh row AND the planted cell still exists +/// with ZERO rows — i.e. the extractor still materializes absence as a node for `minCount 1` to reject, which is +/// the whole reason this shape exists. An extractor that emitted only cells it had rows for would turn absence +/// back into silence, and this control would go `not-fired`. +#[must_use] +pub fn positive_control() -> bool { + let Ok(subject) = Subject::new("0.0.0-pc", PC_COMMIT) else { + return false; + }; + let (Ok(sample), Ok(planted)) = ( + receipts::parse("__pc_sample__.json", &pc_receipt(PC_ROW)), + receipts::parse("__pc_planted__.json", &pc_receipt("")), + ) else { + return false; + }; + let cell = iri_path( + "release-cell", + &["0.0.0-pc", "pc-host", "pc.gguf", "run", "think-off", "pc"], + ); + let rows_of = |r: &Receipt| { + let mut g = Graph::new(); + let ladder = Ladder { + hosts: vec![HostDecl { + id: "pc-host".into(), + cc: "sm_0".into(), + }], + ..Ladder::default() + }; + let rung = ContextRung { + id: "pc".into(), + tokens: Some(1), + long: false, + derived_from: vec!["pc".into()], + unmeasured_consumers: Vec::new(), + }; + build( + &mut g, + &subject, + &Inputs { + ladder: &ladder, + ctx_rungs: std::slice::from_ref(&rung), + consumers: &[], + model_receipts: std::slice::from_ref(r), + kernel_receipts: &[], + tok_receipts: &[], + dogfood: None, + }, + ); + let is_cell = g + .objects(&cell, RDF_TYPE) + .iter() + .any(|t| t.as_iri() == Some(rel("Cell").as_str())); + let fresh: Vec = g + .objects(&cell, &rel("row")) + .iter() + .filter_map(|t| t.as_iri()) + .map(|row| { + g.objects(row, &rel("fresh")) + .iter() + .any(|t| t.as_literal().map(|(v, _)| v) == Some("true")) + }) + .collect(); + (is_cell, fresh) + }; + let (sample_cell, sample_rows) = rows_of(&sample); + let (planted_cell, planted_rows) = rows_of(&planted); + sample_cell && sample_rows == [true] && planted_cell && planted_rows.is_empty() +} + +const PC_COMMIT: &str = "1111111111111111111111111111111111111111"; +const PC_ROW: &str = r#"{"sha256":"cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc","file":"pc.gguf","verb":"run","thinking":"off","context":"pc","prompt_tokens":2,"max_tokens":8,"answer_chars":1,"verdict":"pass","backend":"cuda","fallback":false,"rc":0}"#; + +fn pc_receipt(cells: &str) -> String { + format!( + r#"{{"schema":"apr-model-ladder-receipt/v2","host":"pc-host","version":"0.0.0-pc","sha":"1","apr_sha":"{PC_COMMIT}","inventory":[{{"file":"pc.gguf","sha256":"cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc","arch":"pc","context_length":10,"thinking_modes":["off"]}}],"cells":[{cells}],"rungs":[]}}"# + ) +} + +fn emit_release( + g: &mut Graph, + subject: &Subject, + hosts: &[HostDecl], + rungs: &[ContextRung], + dogfood: Option<&Dogfood>, +) { + let v = subject.version.as_str(); + let n = iri_path("release-subject", &[v]); + g.insert(n.clone(), RDF_TYPE, Term::iri(rel("Release"))); + g.insert(n.clone(), rel("version"), Term::string(v)); + g.insert(n.clone(), rel("commit"), Term::string(&subject.commit)); + g.insert( + n.clone(), + rel("measuredCommit"), + Term::string(subject.measured_commit()), + ); + for h in hosts { + g.insert( + n.clone(), + rel("requiredHost"), + Term::iri(iri_path("release-host", &[v, &h.id])), + ); + } + for r in rungs { + g.insert( + n.clone(), + rel("contextRung"), + Term::iri(iri("release-context", &r.id)), + ); + } + if let Some(d) = dogfood { + let dn = iri_path("release-dogfood", &[v]); + g.insert(dn.clone(), RDF_TYPE, Term::iri(rel("DogfoodReceipt"))); + g.insert(dn.clone(), rel("file"), Term::string(&d.file)); + g.insert(dn.clone(), rel("verdict"), Term::string(&d.verdict)); + g.insert(dn.clone(), rel("commit"), Term::string(&d.commit)); + g.insert(dn.clone(), rel("version"), Term::string(&d.version)); + let fresh = d.commit == subject.commit && d.version == subject.version; + g.insert(dn.clone(), rel("fresh"), Term::boolean(fresh)); + g.insert(n, rel("dogfoodReceipt"), Term::iri(dn)); + } +} + +/// The verbs, the context rungs and the consumers they were derived from. +fn emit_vocabulary_nodes(g: &mut Graph, rungs: &[ContextRung], consumers: &[Consumer]) { + for verb in VERBS { + let n = iri("release-verb", verb); + g.insert(n.clone(), RDF_TYPE, Term::iri(rel("Verb"))); + g.insert(n, rel("name"), Term::string(verb)); + } + // `declared` has no one size — each model's own length is its target — so it is not a `release:ContextRung` + // (whose shape requires a `tokens`); the model's `contextLength` carries it, and `.model` requires that. + let declared = iri("release-context", DECLARED); + g.insert(declared.clone(), RDF_TYPE, Term::iri(rel("DeclaredRung"))); + g.insert(declared.clone(), rel("rungId"), Term::string(DECLARED)); + g.insert( + declared, + rel("derivedFrom"), + Term::string("each model's own GGUF context_length, measured on the host (#3710)"), + ); + for r in rungs { + let n = iri("release-context", &r.id); + g.insert(n.clone(), RDF_TYPE, Term::iri(rel("ContextRung"))); + g.insert(n.clone(), rel("rungId"), Term::string(&r.id)); + if let Some(t) = r.tokens { + g.insert(n.clone(), rel("tokens"), Term::integer(t)); + } + for d in &r.derived_from { + g.insert(n.clone(), rel("derivedFrom"), Term::string(d)); + } + for c in &r.unmeasured_consumers { + g.insert(n.clone(), rel("unmeasuredConsumer"), Term::string(c)); + } + } + for c in consumers { + let n = iri("release-consumer", &c.name); + g.insert(n.clone(), RDF_TYPE, Term::iri(rel("Consumer"))); + if let Some(t) = c.max_prompt_tokens { + g.insert(n.clone(), rel("maxPromptTokens"), Term::integer(t)); + } + g.insert(n.clone(), rel("basis"), Term::string(&c.basis)); + g.insert(n, rel("source"), Term::string(&c.source)); + } +} + +/// The host's receipts and its universe: every inventory model with a measured hash, plus every ladder rung that +/// claims cuda and lists this host (or lists none). Keyed by sha256, so a copy under a second name is one model. +fn host_view<'a>( + decl: &'a HostDecl, + subject: &Subject, + model_receipts: &'a [Receipt], + kernel_receipts: &'a [KernelReceipt], + ladder: &Ladder, +) -> HostView<'a> { + let receipts: Vec<&Receipt> = model_receipts + .iter() + .filter(|r| r.host == decl.id) + .collect(); + let kernels: Vec<&KernelReceipt> = kernel_receipts + .iter() + .filter(|r| r.host == decl.id) + .collect(); + let mut models: BTreeMap = BTreeMap::new(); + for item in receipts.iter().flat_map(|r| &r.inventory) { + if let Some(sha) = &item.sha256 { + models.entry(sha.clone()).or_insert_with(|| ModelInfo { + file: item.file.clone(), + arch: item.arch.clone(), + quant: item.quant.clone(), + context_length: item.context_length, + thinking_modes: item.thinking_modes.clone(), + thinking_markers: item.thinking_markers.clone(), + owes_long_echo: item.owes_long_rungs, + owes_long: true, + mem: item + .weights_bytes + .zip(item.kv_bytes_per_token) + .zip(item.workspace_bytes) + .map(|((w, kv), ws)| (w, kv, ws)), + kv_dtype: item.kv_dtype.clone(), + }); + } + } + let listed = |r: &&Rung| r.hosts.is_empty() || r.hosts.iter().any(|h| h == &decl.id); + for r in ladder + .rungs + .iter() + .filter(listed) + .filter(|r| r.backends.iter().any(|b| b == "cuda")) + { + models + .entry(r.sha256.to_ascii_lowercase()) + .or_insert_with(|| ModelInfo { + file: r.gguf.clone(), + arch: Some(r.arch.clone()), + quant: None, + context_length: None, + thinking_modes: None, + thinking_markers: None, + owes_long_echo: None, + owes_long: true, + mem: None, + kv_dtype: None, + }); + } + if let Some(lr) = &ladder.long_rungs_for { + for m in models.values_mut() { + m.owes_long = lr.owes(m.arch.as_deref(), &m.file); + } + } + let gpu_mem = receipts.iter().find_map(|r| r.gpu_mem_total_bytes); + HostView { + decl, + node: iri_path("release-host", &[&subject.version, &decl.id]), + receipts, + kernels, + models, + gpu_mem, + } +} + +/// Is this receipt of THIS release — its version, and the sha its `apr` was built from? +fn fresh(version: &str, apr_sha: Option<&str>, subject: &Subject) -> bool { + version == subject.version && apr_sha == Some(subject.measured_commit()) +} + +fn emit_host(g: &mut Graph, subject: &Subject, v: &HostView<'_>, stats: &mut ReleaseStats) { + let h = &v.node; + g.insert(h.clone(), RDF_TYPE, Term::iri(rel("Host"))); + g.insert(h.clone(), rel("hostId"), Term::string(&v.decl.id)); + g.insert(h.clone(), rel("cc"), Term::string(&v.decl.cc)); + for r in &v.receipts { + let n = iri_path("release-receipt", &[&subject.version, &v.decl.id, &r.file]); + g.insert(n.clone(), RDF_TYPE, Term::iri(rel("HostReceipt"))); + g.insert(n.clone(), rel("file"), Term::string(&r.file)); + g.insert(n.clone(), rel("schema"), Term::string(&r.schema)); + g.insert(n.clone(), rel("version"), Term::string(&r.version)); + if let Some(s) = &r.apr_sha { + g.insert(n.clone(), rel("aprSha"), Term::string(s)); + } + let ok = fresh(&r.version, r.apr_sha.as_deref(), subject); + g.insert(n.clone(), rel("fresh"), Term::boolean(ok)); + g.insert(h.clone(), rel("hostReceipt"), Term::iri(n)); + for item in r.inventory.iter().filter(|i| i.sha256.is_none()) { + g.insert( + h.clone(), + rel("unmeasuredModel"), + Term::string(format!("{}: {}", r.file, item.file)), + ); + } + for c in r.cells.iter().filter(|c| c.sha256.is_none()) { + g.insert( + h.clone(), + rel("unkeyedRow"), + Term::string(format!( + "{}: {} {} think-{} {} verdict={}", + r.file, c.file, c.verb, c.thinking, c.context, c.verdict + )), + ); + } + } + for (sha, m) in &v.models { + stats.models += 1; + let n = iri("model", sha); + g.insert(n.clone(), RDF_TYPE, Term::iri(rel("Model"))); + g.insert(n.clone(), rel("sha256"), Term::string(sha)); + g.insert(n.clone(), rel("file"), Term::string(&m.file)); + if let Some(a) = &m.arch { + g.insert(n.clone(), rel("arch"), Term::string(a)); + } + if let Some(q) = &m.quant { + g.insert(n.clone(), rel("quant"), Term::string(q)); + } + if let Some(l) = m.context_length { + g.insert(n.clone(), rel("contextLength"), Term::integer(l)); + } + emit_model_evidence(g, &n, m); + g.insert(h.clone(), rel("holds"), Term::iri(n)); + } +} + +/// A model's thinking evidence and long-rung obligation, and every disagreement between what the producer +/// claimed and what pv derives (#3712 amendments 1, 2). +fn emit_model_evidence(g: &mut Graph, n: &str, m: &ModelInfo) { + let n = n.to_string(); + for t in m.thinking_modes.iter().flatten() { + g.insert(n.clone(), rel("thinkingMode"), Term::string(t)); + } + for mk in m.thinking_markers.iter().flatten() { + g.insert(n.clone(), rel("thinkingMarker"), Term::string(mk)); + } + if let Some(c) = m.thinking_contradiction() { + g.insert(n.clone(), rel("thinkingContradiction"), Term::string(c)); + } + g.insert(n.clone(), rel("owesLongRungs"), Term::boolean(m.owes_long)); + if let Some(k) = &m.kv_dtype { + g.insert(n.clone(), rel("kvDtype"), Term::string(k)); + } + if let Some(echo) = m.owes_long_echo.filter(|e| *e != m.owes_long) { + g.insert( + n, + rel("longRungsMismatch"), + Term::string(format!( + "{}: receipt says owes_long_rungs={echo}, the ladder contract derives {}", + m.file, m.owes_long + )), + ); + } +} + +/// One cell's key: (verb, thinking mode, rung id, the rung's target size for this model). +type CellKey<'r> = (&'static str, &'static str, &'r str, Option); + +/// One `release:Cell` per (model, verb, thinking, rung) on this host, and an edge to every row that keys onto it. +/// (model sha256, model file, rung id) → the hosts on which that rung is OWED (it fits, or its fit is unknown). +type Coverage = BTreeMap<(String, String, String), BTreeSet>; + +fn emit_cells( + g: &mut Graph, + subject: &Subject, + v: &HostView<'_>, + rungs: &[ContextRung], + coverage: &mut Coverage, + stats: &mut ReleaseStats, +) { + let mut keyed: BTreeSet<(String, usize)> = BTreeSet::new(); + for (sha, m) in &v.models { + let owed = m.rungs(rungs); + for (id, t) in &owed { + let hosts = coverage + .entry((sha.clone(), m.file.clone(), (*id).to_string())) + .or_default(); + if m.fits(v.gpu_mem, *t) != Some(false) { + hosts.insert(v.node.clone()); + } + } + let modes = m.modes(); + let keys: Vec> = VERBS + .iter() + .flat_map(|verb| modes.iter().map(move |mode| (*verb, *mode))) + .flat_map(|(verb, mode)| owed.iter().map(move |(id, t)| (verb, mode, *id, *t))) + .collect(); + for key in keys { + let cell = emit_cell_node(g, subject, v, sha, m, key); + stats.cells += 1; + if m.fits(v.gpu_mem, key.3) == Some(false) { + stats.refusal_cells += 1; + } + let (verb, mode, rung, _) = key; + stats.cell_names.push(format!( + "{}/{}/{verb}/think-{mode}/{rung}", + v.decl.id, m.file + )); + let rows = rows_for(g, subject, v, sha, key, &mut keyed); + if !rows.is_empty() { + stats.cells_with_row += 1; + } + for r in rows { + g.insert(cell.clone(), rel("row"), Term::iri(r)); + } + } + } + let all: usize = v.receipts.iter().map(|r| r.cells.len()).sum(); + stats.orphan_rows += all - keyed.len(); +} + +fn emit_cell_node( + g: &mut Graph, + subject: &Subject, + v: &HostView<'_>, + sha: &str, + m: &ModelInfo, + (verb, mode, rung, target): CellKey<'_>, +) -> String { + let think = format!("think-{mode}"); + let cell = iri_path( + "release-cell", + &[&subject.version, &v.decl.id, &m.file, verb, &think, rung], + ); + // a cell that does not FIT this host by the declared arithmetic is owed as an honest pre-load refusal + // instead (#3710 rule): a different class, a different shape — never an OOM mid-run + let class = if m.fits(v.gpu_mem, target) == Some(false) { + "RefusalCell" + } else { + "Cell" + }; + g.insert(cell.clone(), RDF_TYPE, Term::iri(rel(class))); + if let Some(req) = target.and_then(|t| m.required_bytes(t)) { + g.insert(cell.clone(), rel("requiredBytes"), Term::integer(req)); + } + g.insert(cell.clone(), rel("host"), Term::iri(v.node.clone())); + g.insert(cell.clone(), rel("model"), Term::iri(iri("model", sha))); + g.insert( + cell.clone(), + rel("verb"), + Term::iri(iri("release-verb", verb)), + ); + g.insert(cell.clone(), rel("thinking"), Term::string(mode)); + g.insert( + cell.clone(), + rel("context"), + Term::iri(iri("release-context", rung)), + ); + cell +} + +/// Emit every row of this host's receipts that keys onto (sha, verb, thinking, rung); return their IRIs. +fn rows_for( + g: &mut Graph, + subject: &Subject, + v: &HostView<'_>, + sha: &str, + key: CellKey<'_>, + keyed: &mut BTreeSet<(String, usize)>, +) -> Vec { + let (verb, mode, rung, _) = key; + let mut out = Vec::new(); + for r in &v.receipts { + for (i, c) in r.cells.iter().enumerate() { + let hit = c.sha256.as_deref() == Some(sha) + && c.verb == verb + && c.thinking == mode + && c.context == rung; + if hit { + keyed.insert((r.file.clone(), i)); + out.push(emit_row(g, subject, r, i, c, key, v.gpu_mem)); + } + } + } + out +} + +/// Did the row fill its rung? A fixed rung: the prompt alone reaches it. `declared`: prompt plus the output +/// budget fill the model's own length. An unknown on either side is `false` — an unmeasured size met nothing. +fn context_met(c: &CellRow, (_, _, rung, target): CellKey<'_>) -> bool { + let Some(t) = target else { + return false; + }; + match (rung == DECLARED, c.prompt_tokens, c.max_tokens) { + (false, Some(p), _) => p >= t, + (true, Some(p), Some(o)) => p.saturating_add(o) >= t, + _ => false, + } +} + +fn emit_row( + g: &mut Graph, + subject: &Subject, + r: &Receipt, + i: usize, + c: &CellRow, + key: CellKey<'_>, + gpu_total: Option, +) -> String { + let n = iri_path( + "release-row", + &[&subject.version, &r.host, &r.file, &i.to_string()], + ); + g.insert(n.clone(), RDF_TYPE, Term::iri(rel("CellReceipt"))); + g.insert( + n.clone(), + rel("verdict"), + Term::string(c.verdict.to_ascii_lowercase()), + ); + g.insert(n.clone(), rel("backend"), Term::string(&c.backend)); + if let Some(fb) = c.fallback { + g.insert(n.clone(), rel("fallback"), Term::boolean(fb)); + } + let ok = fresh(&r.version, r.apr_sha.as_deref(), subject); + g.insert(n.clone(), rel("fresh"), Term::boolean(ok)); + g.insert( + n.clone(), + rel("contextMet"), + Term::boolean(context_met(c, key)), + ); + // thinking ON must CLOSE its block (#3720: an unclosed block is reported, never an empty answer) + let think_ok = c.thinking == "off" || c.think_closed == Some(true); + g.insert(n.clone(), rel("thinkOk"), Term::boolean(think_ok)); + let answered = c.answer_chars.is_some_and(|a| a > 0); + // a refusal NAMES its arithmetic: what it needs is more than what the host has + // honest only against the TOTAL: a refusal because a co-tenant holds memory (free < required ≤ total) is + // not the arithmetic, it is a busy GPU silently shrinking what the release proves (#3712, 62) + let named = matches!((c.required_bytes, gpu_total), (Some(r), Some(t)) if r > t); + g.insert(n.clone(), rel("arithmeticNamed"), Term::boolean(named)); + g.insert(n.clone(), rel("answered"), Term::boolean(answered)); + emit_row_detail(g, &n, r, c); + n +} + +/// What the row measured, carried for the reader: never graded here. +fn emit_row_detail(g: &mut Graph, n: &str, r: &Receipt, c: &CellRow) { + let n = n.to_string(); + if let Some(p) = c.prompt_tokens { + g.insert(n.clone(), rel("promptTokens"), Term::integer(p)); + } + if let Some(o) = c.max_tokens { + g.insert(n.clone(), rel("maxTokens"), Term::integer(o)); + } + if let Some(t) = c.ttft_ms { + g.insert(n.clone(), rel("ttftMs"), Term::double(t)); + } + if let Some(rc) = c.rc { + g.insert(n.clone(), rel("rc"), Term::signed(rc)); + } + if !c.reason.is_empty() { + g.insert(n.clone(), rel("reason"), Term::string(&c.reason)); + } + if let Some(s) = &r.apr_sha { + g.insert(n.clone(), rel("aprSha"), Term::string(s)); + } + g.insert(n, rel("receiptFile"), Term::string(&r.file)); +} + +/// Every rung a model declares must be owed — and so Pass — on at least one required host (#3710 rule): the +/// 27B's 262,144 cells refuse on the 24 GB 4090 and are owed on gx10. A rung that fits nowhere is named here. +fn emit_coverage(g: &mut Graph, subject: &Subject, coverage: &Coverage, stats: &mut ReleaseStats) { + for ((sha, file, rung), hosts) in coverage { + stats.rung_coverage += 1; + let n = iri_path("release-coverage", &[&subject.version, file, rung]); + g.insert(n.clone(), RDF_TYPE, Term::iri(rel("RungCoverage"))); + g.insert(n.clone(), rel("model"), Term::iri(iri("model", sha))); + g.insert( + n.clone(), + rel("context"), + Term::iri(iri("release-context", rung)), + ); + for h in hosts { + g.insert(n.clone(), rel("owedOn"), Term::iri(h.clone())); + } + } +} + +/// The kernel differential (#3715 addendum): the universe K is every kernel on ANY required host's dispatch +/// path, and each (kernel, host) — each GPU arch — is a `release:KernelCell`. A kernel measured on one arch only +/// is therefore a missing row on the other, and one that passes on sm_121 and fails on sm_89 is a failing row on +/// the sm_89 node: the class that made qwen2.5-coder red on lambda alone. +fn emit_kernels( + g: &mut Graph, + subject: &Subject, + views: &[HostView<'_>], + stats: &mut ReleaseStats, +) { + let universe: BTreeSet<(&str, &str)> = views + .iter() + .flat_map(|v| &v.kernels) + .flat_map(|r| &r.dispatch) + .flat_map(|d| &d.kernels) + .map(|(k, q)| (k.as_str(), q.as_str())) + .collect(); + for v in views { + emit_kernel_host(g, subject, v); + for key in &universe { + stats.kernel_cells += 1; + emit_kernel_cell(g, subject, v, *key); + } + } +} + +/// The host's kernel receipts, its dispatch kernels, and every held model the dispatch lists do not cover. +fn emit_kernel_host(g: &mut Graph, subject: &Subject, v: &HostView<'_>) { + let h = &v.node; + let mut covered: BTreeSet<&str> = BTreeSet::new(); + for r in &v.kernels { + let n = iri_path("release-kreceipt", &[&subject.version, &v.decl.id, &r.file]); + g.insert(n.clone(), RDF_TYPE, Term::iri(rel("KernelReceiptFile"))); + g.insert(n.clone(), rel("file"), Term::string(&r.file)); + g.insert(n.clone(), rel("sm"), Term::string(&r.sm)); + let ok = fresh(&r.version, r.apr_sha.as_deref(), subject); + g.insert(n.clone(), rel("fresh"), Term::boolean(ok)); + g.insert(h.clone(), rel("kernelReceipt"), Term::iri(n)); + for d in &r.dispatch { + if let Some(s) = &d.sha256 { + covered.insert(s); + } + for (k, q) in &d.kernels { + g.insert( + h.clone(), + rel("dispatchKernel"), + Term::string(kernel_label(k, q)), + ); + } + } + } + for (sha, m) in &v.models { + if !covered.contains(sha.as_str()) { + g.insert( + h.clone(), + rel("modelWithoutDispatch"), + Term::string(&m.file), + ); + } + } +} + +/// `q4k_gemv@q4_k`, or the bare kernel when it carries no quant (an f32 elementwise kernel). +fn kernel_label(kernel: &str, quant: &str) -> String { + if quant.is_empty() { + kernel.to_string() + } else { + format!("{kernel}@{quant}") + } +} + +fn emit_kernel_cell( + g: &mut Graph, + subject: &Subject, + v: &HostView<'_>, + (kernel, quant): (&str, &str), +) { + // kernel and quant as their own segments: `release-kernel////`, `-` for none + let q = if quant.is_empty() { "-" } else { quant }; + let cell = iri_path("release-kernel", &[&subject.version, &v.decl.id, kernel, q]); + g.insert(cell.clone(), RDF_TYPE, Term::iri(rel("KernelCell"))); + g.insert(cell.clone(), rel("kernel"), Term::string(kernel)); + g.insert(cell.clone(), rel("quant"), Term::string(quant)); + g.insert(cell.clone(), rel("host"), Term::iri(v.node.clone())); + g.insert(cell.clone(), rel("cc"), Term::string(&v.decl.cc)); + for r in &v.kernels { + let rows = r.rows.iter().enumerate(); + for (i, row) in rows.filter(|(_, x)| x.kernel == kernel && x.quant == quant) { + let n = emit_krow(g, subject, r, i, row); + g.insert(cell.clone(), rel("row"), Term::iri(n)); + } + } +} + +fn emit_krow( + g: &mut Graph, + subject: &Subject, + r: &KernelReceipt, + i: usize, + row: &inputs::KernelRow, +) -> String { + let n = iri_path( + "release-krow", + &[&subject.version, &r.host, &r.file, &i.to_string()], + ); + g.insert(n.clone(), RDF_TYPE, Term::iri(rel("KernelDiffReceipt"))); + g.insert(n.clone(), rel("verdict"), Term::string(&row.verdict)); + g.insert(n.clone(), rel("reference"), Term::string(&row.reference)); + if let Some(e) = row.max_err { + g.insert(n.clone(), rel("maxErr"), Term::double(e)); + } + if let Some(b) = row.bound { + g.insert(n.clone(), rel("bound"), Term::double(b)); + } + if let Some(m) = row.index_mismatch { + g.insert(n.clone(), rel("indexMismatch"), Term::integer(m)); + } + if let Some(t) = row.tie_margin { + g.insert(n.clone(), rel("tieMargin"), Term::double(t)); + } + g.insert( + n.clone(), + rel("withinBound"), + Term::boolean(row.within_bound()), + ); + if !row.bound_source.is_empty() { + g.insert( + n.clone(), + rel("boundSource"), + Term::string(&row.bound_source), + ); + } + let ok = fresh(&r.version, r.apr_sha.as_deref(), subject); + g.insert(n.clone(), rel("fresh"), Term::boolean(ok)); + if !row.reason.is_empty() { + g.insert(n.clone(), rel("reason"), Term::string(&row.reason)); + } + n +} + +/// Tokenizer parity (aprender#3726): apr's token ids against the pinned llama.cpp, one cell per distinct model +/// in any required host's universe — host-independent, because the tokenizer is. It is the cell that sees what +/// every apr-vs-apr gate cannot: CPU and GPU share the tokenizer, so `apr parity` passes a tokenizer that maps +/// every non-ASCII byte to id 0. +fn emit_tokenizer( + g: &mut Graph, + subject: &Subject, + views: &[HostView<'_>], + receipts: &[TokReceipt], + stats: &mut ReleaseStats, +) { + let models: BTreeMap<&str, &ModelInfo> = views + .iter() + .flat_map(|v| v.models.iter().map(|(sha, m)| (sha.as_str(), m))) + .collect(); + for (sha, m) in models { + stats.tokenizer_cells += 1; + let cell = iri_path("release-tokenizer", &[&subject.version, &m.file]); + g.insert(cell.clone(), RDF_TYPE, Term::iri(rel("TokenizerCell"))); + g.insert(cell.clone(), rel("model"), Term::iri(iri("model", sha))); + if let Some(a) = &m.arch { + g.insert(cell.clone(), rel("family"), Term::string(a)); + } + for r in receipts { + let rows = r.rows.iter().enumerate(); + for (i, row) in rows.filter(|(_, x)| x.sha256.as_deref() == Some(sha)) { + let n = emit_trow(g, subject, r, i, row); + g.insert(cell.clone(), rel("row"), Term::iri(n)); + } + } + } +} + +fn emit_trow( + g: &mut Graph, + subject: &Subject, + r: &TokReceipt, + i: usize, + row: &inputs::TokRow, +) -> String { + let n = iri_path("release-trow", &[&subject.version, &r.file, &i.to_string()]); + g.insert( + n.clone(), + RDF_TYPE, + Term::iri(rel("TokenizerParityReceipt")), + ); + g.insert(n.clone(), rel("verdict"), Term::string(&row.verdict)); + g.insert(n.clone(), rel("parityExact"), Term::boolean(row.exact())); + g.insert( + n.clone(), + rel("roundTrip"), + Term::boolean(row.roundtrip_ok == Some(true)), + ); + let opt = |g: &mut Graph, p: &str, v: &str| { + if !v.is_empty() { + g.insert(n.clone(), rel(p), Term::string(v)); + } + }; + opt(g, "corpusSha", &row.corpus_sha256); + opt(g, "comparator", &row.comparator); + opt(g, "comparatorSha", &row.comparator_sha); + opt(g, "reason", &row.reason); + if let Some(m) = row.mismatches { + g.insert(n.clone(), rel("mismatches"), Term::integer(m)); + } + if let Some(t) = row.tokens_compared { + g.insert(n.clone(), rel("tokensCompared"), Term::integer(t)); + } + let ok = fresh(&r.version, r.apr_sha.as_deref(), subject); + g.insert(n.clone(), rel("fresh"), Term::boolean(ok)); + n +} + +#[cfg(test)] +#[path = "release_evidence_tests.rs"] +mod tests; diff --git a/crates/aprender-contracts/src/ontology/extract/release_evidence_tests.rs b/crates/aprender-contracts/src/ontology/extract/release_evidence_tests.rs new file mode 100644 index 0000000000..e2fcce47cf --- /dev/null +++ b/crates/aprender-contracts/src/ontology/extract/release_evidence_tests.rs @@ -0,0 +1,458 @@ +//! aprender#3715 — `extract:release-evidence` in isolation: the universe, the join, and what is computed. The +//! shapes' verdicts over the same graphs are `tests/fixtures/ont/release-*` on the CLI +//! (`crates/aprender-contracts-cli/tests/ont_release_readiness.rs`). + +use super::*; +use crate::ontology::extract::release_inputs::{derive_rungs, Consumer}; + +const MC: &str = "1111111111111111111111111111111111111111"; +const BUMP: &str = "2222222222222222222222222222222222222222"; +const SHA_A: &str = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"; +const SHA_B: &str = "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"; + +/// A throwaway repo: a ladder contract with two required hosts, a context-rung file, and whatever receipts +/// the case writes. Returns (tempdir, contract dir). +fn repo(ladder_rungs: &str) -> (tempfile::TempDir, std::path::PathBuf) { + let t = tempfile::tempdir().expect("tempdir"); + let c = t.path().join("contracts"); + std::fs::create_dir_all(&c).expect("contracts dir"); + std::fs::write( + c.join("ladder.yaml"), + format!( + "name: ladder\nentity: {{type: gguf}}\nladder:\n hosts:\n - {{id: lambda, cc: sm_89, required: true}}\n - {{id: gx10, cc: sm_121, required: true}}\n - {{id: yoga, cc: sm_86, required: false}}\n rungs:\n{ladder_rungs}" + ), + ) + .expect("ladder"); + let ev = t.path().join("evidence/release"); + std::fs::create_dir_all(&ev).expect("evidence dir"); + std::fs::write( + ev.join("context-rungs.json"), + r#"{"schema":"apr-release-context-rungs/v1", + "consumers":[{"consumer":"rah","max_prompt_tokens":1000,"basis":"measured","source":"u1"}], + "rungs":[{"id":"golden","tokens":0,"derived_from":["g"]},{"id":"consumer-max"}]}"#, + ) + .expect("rungs"); + (t, c) +} + +fn write_receipt(root: &Path, host: &str, body: &str) { + let d = root.join("evidence/dogfood/models/0.69.1"); + std::fs::create_dir_all(&d).expect("receipt dir"); + std::fs::write(d.join(format!("{host}.json")), body).expect("receipt"); +} + +fn receipt(host: &str, apr_sha: &str, inventory: &str, cells: &str) -> String { + format!( + r#"{{"schema":"apr-model-ladder-receipt/v2","host":"{host}","version":"0.69.1","sha":"x", + "apr_sha":"{apr_sha}","cc":"8.9","gpu":"g","inventory":[{inventory}],"cells":[{cells}],"rungs":[]}}"# + ) +} + +fn cell(sha: &str, verb: &str, ctx: &str) -> String { + cell_t(sha, verb, "off", ctx) +} + +fn cell_t(sha: &str, verb: &str, thinking: &str, ctx: &str) -> String { + format!( + r#"{{"sha256":"{sha}","file":"m.gguf","verb":"{verb}","thinking":"{thinking}","context":"{ctx}", + "prompt_tokens":2000,"max_tokens":512,"answer_chars":12, + "verdict":"pass","backend":"cuda","fallback":false,"rc":0}}"# + ) +} + +fn subject() -> Subject { + Subject::new("0.69.1", MC).expect("subject") +} + +#[test] +fn a_subject_refuses_a_short_or_empty_sha_and_an_empty_version() { + assert!( + Subject::new("0.69.1", "225b2a9ab").is_err(), + "a prefix is refused, never matched" + ); + assert!(Subject::new("", MC).is_err()); + assert!(subject().with_receipts_commit("abc").is_err()); + let s = subject().with_receipts_commit(BUMP).expect("40 hex"); + assert_eq!(s.measured_commit(), BUMP); + assert_eq!(subject().measured_commit(), MC); +} + +#[test] +fn consumer_max_is_the_max_over_the_records_and_names_a_consumer_with_no_number() { + let consumers = vec![ + Consumer { + name: "rah".into(), + max_prompt_tokens: Some(148_000), + basis: "measured".into(), + source: "u1".into(), + }, + Consumer { + name: "rmedia".into(), + max_prompt_tokens: Some(128_000), + basis: "plan".into(), + source: "u2".into(), + }, + Consumer { + name: "arbiter".into(), + max_prompt_tokens: None, + basis: String::new(), + source: "u3".into(), + }, + ]; + let rung = ContextRung { + id: "consumer-max".into(), + tokens: Some(7), + long: true, + derived_from: vec![], + unmeasured_consumers: vec![], + }; + let out = derive_rungs(vec![rung], &consumers); + assert_eq!( + out[0].tokens, + Some(148_000), + "a declared number is ignored: the rung is derived" + ); + assert_eq!(out[0].derived_from.len(), 3); + assert_eq!(out[0].unmeasured_consumers, vec!["arbiter".to_string()]); +} + +#[test] +fn the_universe_is_the_inventory_plus_the_cuda_rungs_listed_for_the_host_and_every_cell_gets_a_node( +) { + // rung a: cuda, every host · rung b: cpu only (not a cuda obligation) · rung c: cuda, gx10 only + let (t, c) = repo(&format!( + " - {{id: a, sha256: {SHA_A}, arch: qwen2, gguf: a.gguf, backends: [cpu, cuda], required: true}}\n - {{id: b, sha256: {}, arch: qwen2, gguf: b.gguf, backends: [cpu], required: true}}\n - {{id: c, sha256: {}, arch: qwen3, gguf: c.gguf, backends: [cuda], hosts: [gx10], required: true}}\n", + "c".repeat(64), + "d".repeat(64) + )); + // lambda holds b (inventory) and an unlisted model e + let inv = format!(r#"{{"file":"e.gguf","sha256":"{SHA_B}"}},{{"file":"nohash.gguf"}}"#); + write_receipt( + t.path(), + "lambda", + &receipt("lambda", MC, &inv, &cell(SHA_A, "run", "golden")), + ); + let mut g = Graph::new(); + let st = extract(&mut g, &c, &subject()).expect("extracts"); + assert_eq!(st.required_hosts, 2, "yoga is not required"); + // lambda: a (rung) + e (inventory) = 2 · gx10: a + c = 2 (no receipt, but the rungs listed for it stay) + assert_eq!(st.models, 4); + // no measured length or thinking mode: every rung (golden, consumer-max) + declared, both modes + assert_eq!(st.cells, 4 * 4 * 2 * 3, "models × verbs × thinking × rungs"); + assert_eq!(st.cells_with_row, 1); + let lambda = iri_path("release-host", &["0.69.1", "lambda"]); + let unmeasured = g.objects(&lambda, &rel("unmeasuredModel")); + assert_eq!( + unmeasured.len(), + 1, + "an inventory row with no hash is named, never dropped" + ); + let gx10 = iri_path("release-host", &["0.69.1", "gx10"]); + assert!( + g.objects(&gx10, &rel("hostReceipt")).is_empty(), + "no receipt is a missing edge" + ); + assert!(!g.to_ntriples().contains("_:"), "no blank nodes (R-15)"); +} + +#[test] +fn a_row_keys_by_sha_verb_and_rung_and_carries_fresh_and_context_met() { + let (t, c) = repo(&format!( + " - {{id: a, sha256: {SHA_A}, arch: qwen2, gguf: a.gguf, backends: [cuda], hosts: [lambda], required: true}}\n" + )); + let rows = [ + cell(SHA_A, "chat", "consumer-max"), + cell(SHA_A, "chat", "no-such-rung"), + cell(SHA_B, "chat", "golden"), + ] + .join(","); + let inv = format!(r#"{{"file":"a.gguf","sha256":"{SHA_A}"}}"#); + write_receipt(t.path(), "lambda", &receipt("lambda", BUMP, &inv, &rows)); + let mut g = Graph::new(); + let st = extract(&mut g, &c, &subject()).expect("extracts"); + assert_eq!(st.cells_with_row, 1); + assert_eq!( + st.orphan_rows, 2, + "an unknown rung and a model outside the universe key onto no cell" + ); + let cellnode = iri_path( + "release-cell", + &[ + "0.69.1", + "lambda", + "a.gguf", + "chat", + "think-off", + "consumer-max", + ], + ); + let row = g.objects(&cellnode, &rel("row"))[0] + .as_iri() + .expect("iri") + .to_string(); + let lit = |p: &str| { + g.objects(&row, &rel(p))[0] + .as_literal() + .map(|(v, _)| v.to_string()) + .expect("literal") + }; + assert_eq!( + lit("fresh"), + "false", + "measured at BUMP, released at MC, no --receipts-commit" + ); + assert_eq!( + lit("contextMet"), + "true", + "2000 prompt tokens ≥ consumer-max 1000" + ); + let mut g2 = Graph::new(); + extract( + &mut g2, + &c, + &subject().with_receipts_commit(BUMP).expect("sha"), + ) + .expect("extracts"); + let fresh2 = g2.objects(&row, &rel("fresh"))[0] + .as_literal() + .map(|(v, _)| v.to_string()); + assert_eq!( + fresh2.as_deref(), + Some("true"), + "T-4: R7 proved BUMP ≡ MC and said so" + ); +} + +#[test] +fn a_kernel_on_one_hosts_dispatch_path_is_an_obligation_on_every_required_host() { + let (t, c) = repo(&format!( + " - {{id: a, sha256: {SHA_A}, arch: qwen2, gguf: a.gguf, backends: [cuda], required: true}}\n" + )); + let kd = t.path().join("evidence/dogfood/kernels/0.69.1"); + std::fs::create_dir_all(&kd).expect("kernel dir"); + std::fs::write( + kd.join("gx10.json"), + format!( + r#"{{"schema":"apr-kernel-diff-receipt/v1","host":"gx10","version":"0.69.1","apr_sha":"{MC}","sm":"sm_121", + "dispatch":[{{"sha256":"{SHA_A}","file":"a.gguf","kernels":[{{"kernel":"q4k_gemv","quant":"q4_k"}}]}}], + "rows":[{{"kernel":"q4k_gemv","quant":"q4_k","reference":"cpu","max_err":0.001,"bound":0.004,"bound_source":"pair","verdict":"pass"}}, + {{"kernel":"q4k_gemv","quant":"q6_k","reference":"cpu","max_err":0.001,"bound":0.004,"bound_source":"pair","verdict":"pass"}}]}}"# + ), + ) + .expect("kernel receipt"); + let mut g = Graph::new(); + let st = extract(&mut g, &c, &subject()).expect("extracts"); + assert_eq!(st.kernel_cells, 2, "one kernel × two required hosts"); + let lambda = iri_path("release-kernel", &["0.69.1", "lambda", "q4k_gemv", "q4_k"]); + let gx10 = iri_path("release-kernel", &["0.69.1", "gx10", "q4k_gemv", "q4_k"]); + assert!( + g.objects(&lambda, &rel("row")).is_empty(), + "measured on gx10 only: lambda's row is missing" + ); + assert_eq!( + g.objects(&gx10, &rel("row")).len(), + 1, + "the q6_k row is a different cell (#3712, eb)" + ); + let host = iri_path("release-host", &["0.69.1", "lambda"]); + assert_eq!( + g.objects(&host, &rel("modelWithoutDispatch")).len(), + 1, + "lambda holds a.gguf and has no dispatch list for it" + ); +} + +#[test] +fn a_foreign_context_or_kernel_schema_is_refused_by_name() { + let (t, c) = repo(""); + std::fs::write( + t.path().join("evidence/release/context-rungs.json"), + r#"{"schema":"something/v9"}"#, + ) + .expect("rewrite"); + let e = extract(&mut Graph::new(), &c, &subject()).expect_err("refused"); + assert!(e.to_string().contains("something/v9"), "{e}"); +} + +#[test] +fn a_measured_length_and_thinking_mode_shape_the_cells_and_a_think_on_row_must_close_and_answer() { + let (t, c) = repo(""); + // e.gguf: 1500-token context (golden 0 fits, consumer-max 1000 fits), no thinking mode + // f.gguf: 800-token context (consumer-max 1000 does NOT fit), thinking mode unmeasured → both modes + let inv = format!( + r#"{{"file":"e.gguf","sha256":"{SHA_A}","context_length":1500,"thinking_modes":["off"],"thinking_markers":[]}}, + {{"file":"f.gguf","sha256":"{SHA_B}","context_length":800}}"# + ); + let unclosed = cell_t(SHA_B, "chat", "on", "golden").replace( + r#""answer_chars":12"#, + r#""answer_chars":0,"think_closed":false"#, + ); + let declared_row = cell_t(SHA_A, "run", "off", "declared"); // 2000 + 512 ≥ 1500 + let rows = [unclosed, declared_row].join(","); + write_receipt(t.path(), "lambda", &receipt("lambda", MC, &inv, &rows)); + let mut g = Graph::new(); + let st = extract(&mut g, &c, &subject()).expect("extracts"); + // lambda: e = 4 verbs × 1 mode × 3 rungs = 12 · f = 4 × 2 × 2 rungs (golden, declared) = 16 + // gx10: no receipt and no rung listed → no models, no cells (its Host node carries the missing receipt) + assert_eq!(st.cells, 12 + 16); + let on = |file: &str| { + iri_path( + "release-cell", + &["0.69.1", "lambda", file, "chat", "think-on", "golden"], + ) + }; + assert!( + g.objects(&on("e.gguf"), RDF_TYPE).is_empty(), + "no thinking mode measured → no ON cell" + ); + let row = g.objects(&on("f.gguf"), &rel("row"))[0] + .as_iri() + .expect("iri") + .to_string(); + let lit = |n: &str, p: &str| { + g.objects(n, &rel(p))[0] + .as_literal() + .map(|(v, _)| v.to_string()) + }; + assert_eq!( + lit(&row, "thinkOk").as_deref(), + Some("false"), + "an unclosed think block" + ); + assert_eq!( + lit(&row, "answered").as_deref(), + Some("false"), + "an empty answer" + ); + let big = iri_path( + "release-cell", + &[ + "0.69.1", + "lambda", + "f.gguf", + "chat", + "think-off", + "consumer-max", + ], + ); + assert!( + g.objects(&big, &rel("model")).is_empty(), + "a rung past the declared length is not owed" + ); + let decl = iri_path( + "release-cell", + &["0.69.1", "lambda", "e.gguf", "run", "think-off", "declared"], + ); + let drow = g.objects(&decl, &rel("row"))[0] + .as_iri() + .expect("iri") + .to_string(); + assert_eq!( + lit(&drow, "contextMet").as_deref(), + Some("true"), + "prompt + budget fill the declared length" + ); +} + +#[test] +fn a_rung_file_that_redeclares_the_per_model_rung_is_refused() { + let (t, c) = repo(""); + std::fs::write( + t.path().join("evidence/release/context-rungs.json"), + r#"{"schema":"apr-release-context-rungs/v1","consumers":[],"rungs":[{"id":"declared","tokens":4096}]}"#, + ) + .expect("rewrite"); + let e = extract(&mut Graph::new(), &c, &subject()).expect_err("refused"); + assert!(e.to_string().contains("declared"), "{e}"); +} + +#[test] +fn long_rungs_are_owed_per_the_ladders_long_rungs_for_and_an_echo_that_disagrees_is_named() { + // qwen35 is a family that owes everything; for qwen2 only the representative file does + let (t, c) = repo( + " cells:\n long_rungs_for: {families: [qwen35], representatives: {qwen2: rep.gguf}}\n", + ); + std::fs::write( + t.path().join("evidence/release/context-rungs.json"), + r#"{"schema":"apr-release-context-rungs/v1", + "consumers":[{"consumer":"rah","max_prompt_tokens":1000,"basis":"measured","source":"u1"}], + "rungs":[{"id":"4k","tokens":40,"derived_from":["g"]},{"id":"consumer-max","long":true}]}"#, + ) + .expect("rungs"); + let inv = format!( + r#"{{"file":"big.gguf","sha256":"{SHA_A}","arch":"qwen35","context_length":262144,"thinking_modes":["on","off"],"thinking_markers":["","enable_thinking"]}}, + {{"file":"small.gguf","sha256":"{SHA_B}","arch":"qwen2","context_length":32768,"thinking_modes":["off"],"thinking_markers":["","enable_thinking"],"owes_long_rungs":true}}"# + ); + write_receipt(t.path(), "lambda", &receipt("lambda", MC, &inv, "")); + let mut g = Graph::new(); + let st = extract(&mut g, &c, &subject()).expect("extracts"); + // big: 4 verbs × 2 modes × (4k, consumer-max, declared) = 24 · small: 4 × 1 × (4k) = 4 + assert_eq!(st.cells, 24 + 4); + let small = iri("model", SHA_B); + let lit = |p: &str| { + g.objects(&small, &rel(p)) + .first() + .and_then(|t| t.as_literal()) + .map(|(v, _)| v.to_string()) + }; + assert_eq!( + lit("owesLongRungs").as_deref(), + Some("false"), + "qwen2 and not the representative" + ); + assert!( + lit("longRungsMismatch").is_some_and(|m| m.contains("small.gguf")), + "the echo said true" + ); + assert!( + lit("thinkingContradiction").is_some_and(|m| m.contains("")), + "off only, though the template carries the enable_thinking switch" + ); + let big = iri("model", SHA_A); + assert!(g.objects(&big, &rel("thinkingContradiction")).is_empty()); + assert!( + g.objects(&big, &rel("longRungsMismatch")).is_empty(), + "no echo, nothing to disagree with" + ); +} + +#[test] +fn a_discrete_kernel_is_within_bound_only_on_zero_mismatches_or_near_ties() { + use crate::ontology::extract::release_inputs::KernelRow; + let row = |mism: Option, tie: Option, err: Option| KernelRow { + kernel: "topk".into(), + quant: "f32".into(), + reference: "cpu".into(), + max_err: err, + index_mismatch: mism, + tie_margin: tie, + bound: Some(0.01), + bound_source: "pair".into(), + verdict: "pass".into(), + reason: String::new(), + }; + assert!(row(Some(0), None, None).within_bound(), "same expert sets"); + assert!( + row(Some(3), Some(0.002), None).within_bound(), + "three near-ties" + ); + assert!( + !row(Some(1), Some(0.5), Some(0.0)).within_bound(), + "a real divergence; max_err is moot" + ); + assert!( + !row(Some(1), None, None).within_bound(), + "a mismatch with no margin measured" + ); + assert!( + !row(None, None, None).within_bound(), + "nothing measured is outside every bound" + ); +} + +#[test] +fn the_positive_control_fires_without_any_release_subject_or_file() { + // PMAT-3704 R-3: drawn on every gate run — a sample cell with one fresh row, and a planted cell with none + // that must still be a node (the mutant that emits only measured cells turns this false; measured) + assert!(positive_control()); +} diff --git a/crates/aprender-contracts/src/ontology/extract/release_inputs.rs b/crates/aprender-contracts/src/ontology/extract/release_inputs.rs new file mode 100644 index 0000000000..ccf04c0eaa --- /dev/null +++ b/crates/aprender-contracts/src/ontology/extract/release_inputs.rs @@ -0,0 +1,494 @@ +//! aprender#3715 — the inputs `extract:release-evidence` reads, each refused BY NAME when unreadable: the release +//! subject (the CLI's `--release-*` flags), the consumer-derived context rungs, the dogfood receipt R5 judged, and +//! the per-host kernel-diff receipts. The per-host MODEL receipts are `crate::ontology::receipts` (one reader for +//! the ladder join and the release matrix). +//! +//! Every reader here is pure and deterministic (R-15): files in byte order, no clock, no network, no git. What a +//! file does not say is carried as `None`, never defaulted to the passing value — an absent `fallback` is not +//! `false`, an absent `max_err` is not within any bound. + +use std::path::{Path, PathBuf}; + +/// The context-rung declaration, relative to the repository root. +pub const CONTEXT_RUNGS_FILE: &str = "evidence/release/context-rungs.json"; +pub const CONTEXT_SCHEMA: &str = "apr-release-context-rungs/v1"; +/// The per-host kernel-diff receipts: `evidence/dogfood/kernels//.json` (#3715 addendum). +pub const KERNEL_EVIDENCE_DIR: &str = "evidence/dogfood/kernels"; +pub const KERNEL_SCHEMA: &str = "apr-kernel-diff-receipt/v1"; +/// The tokenizer-parity receipts (aprender#3726): apr's token ids against the pinned llama.cpp comparator. +pub const TOKENIZER_EVIDENCE_DIR: &str = "evidence/dogfood/tokenizer"; +pub const TOKENIZER_SCHEMA: &str = "apr-tokenizer-parity-receipt/v1"; +/// The rung whose token count is DERIVED as the max over the consumer records, never written down. +pub const CONSUMER_MAX: &str = "consumer-max"; +/// Reserved: the per-model rung whose size is the model's own GGUF `context_length` (added by the extractor). +pub const DECLARED: &str = "declared"; + +/// What a release-readiness run is asked about. Built from `--release-version` / `--release-commit` and friends. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Subject { + pub version: String, + /// The release commit (MC), 40 lowercase hex: the dogfood receipt's `commit` must equal it. + pub commit: String, + /// T-4 only: the sha the committed receipts were measured at, after R7 proved the tree equal modulo + /// `evidence/`. `None` → the receipts must carry [`Subject::commit`]. + pub receipts_commit: Option, + /// `--receipts`: the model-receipt dir. `None` → `evidence/dogfood/models//`. + pub receipts_dir: Option, + /// `--kernel-receipts`: `None` → `evidence/dogfood/kernels//`. + pub kernel_receipts_dir: Option, + /// `--dogfood-receipt`: the exact file rule_r5 judged. `None` → the release has no dogfood receipt (a violation). + pub dogfood_receipt: Option, + /// `--tokenizer-receipts`: `None` → `evidence/dogfood/tokenizer//`. + pub tokenizer_receipts_dir: Option, +} + +impl Subject { + /// Refuse a subject no receipt could ever match: an empty version, or a commit that is not 40 hex. A short sha + /// is refused rather than prefix-matched — two releases can share a prefix, and the gate would not know. + pub fn new(version: &str, commit: &str) -> Result { + if version.trim().is_empty() { + return Err(ReleaseError::Subject("--release-version is empty".into())); + } + Ok(Self { + version: version.trim().to_string(), + commit: full_sha("--release-commit", commit)?, + receipts_commit: None, + receipts_dir: None, + kernel_receipts_dir: None, + dogfood_receipt: None, + tokenizer_receipts_dir: None, + }) + } + + /// Set `--receipts-commit` (T-4), under the same 40-hex rule. + pub fn with_receipts_commit(mut self, sha: &str) -> Result { + self.receipts_commit = Some(full_sha("--receipts-commit", sha)?); + Ok(self) + } + + /// The sha every model and kernel receipt's `apr_sha` must equal. + #[must_use] + pub fn measured_commit(&self) -> &str { + self.receipts_commit.as_deref().unwrap_or(&self.commit) + } + + #[must_use] + pub fn model_dir(&self, root: &Path) -> PathBuf { + self.receipts_dir.clone().unwrap_or_else(|| { + root.join(crate::ontology::receipts::EVIDENCE_DIR) + .join(&self.version) + }) + } + + #[must_use] + pub fn tokenizer_dir(&self, root: &Path) -> PathBuf { + self.tokenizer_receipts_dir + .clone() + .unwrap_or_else(|| root.join(TOKENIZER_EVIDENCE_DIR).join(&self.version)) + } + + #[must_use] + pub fn kernel_dir(&self, root: &Path) -> PathBuf { + self.kernel_receipts_dir + .clone() + .unwrap_or_else(|| root.join(KERNEL_EVIDENCE_DIR).join(&self.version)) + } +} + +fn full_sha(flag: &str, s: &str) -> Result { + let s = s.trim().to_ascii_lowercase(); + if s.len() == 40 && s.bytes().all(|b| b.is_ascii_hexdigit()) { + Ok(s) + } else { + Err(ReleaseError::Subject(format!( + "{flag} {s:?} is not a full 40-hex commit (a prefix is refused, never matched)" + ))) + } +} + +/// Why the release evidence could not be read. Every variant is the DECLARATION's fault (exit 3). +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum ReleaseError { + Subject(String), + Receipt(crate::ontology::receipts::ReceiptError), + Input { file: String, what: String }, +} + +impl std::fmt::Display for ReleaseError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Subject(w) => write!(f, "release subject: {w}"), + Self::Receipt(e) => write!(f, "receipt {e}"), + Self::Input { file, what } => write!(f, "{file}: {what}"), + } + } +} + +impl std::error::Error for ReleaseError {} + +fn input_err(file: &str, what: impl Into) -> ReleaseError { + ReleaseError::Input { + file: file.to_string(), + what: what.into(), + } +} + +/// Read `path` as JSON carrying `schema == want`; any other schema is refused by name. +fn read_json(path: &Path, name: &str, want: &str) -> Result { + let text = + std::fs::read_to_string(path).map_err(|e| input_err(name, format!("unreadable: {e}")))?; + let v: serde_json::Value = + serde_json::from_str(&text).map_err(|e| input_err(name, format!("not JSON: {e}")))?; + let schema = v + .get("schema") + .and_then(serde_json::Value::as_str) + .unwrap_or(""); + if schema != want { + return Err(input_err( + name, + format!("schema {schema:?} is not {want} — refused by name"), + )); + } + Ok(v) +} + +fn str_of(v: &serde_json::Value, key: &str) -> Option { + v.get(key) + .and_then(serde_json::Value::as_str) + .map(str::to_string) +} + +fn list<'a>(v: &'a serde_json::Value, key: &str) -> impl Iterator { + v.get(key) + .and_then(serde_json::Value::as_array) + .into_iter() + .flatten() +} + +/// One consumer's measured (or planned) maximum prompt, quoted from its own record on #3710 / #3716. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Consumer { + pub name: String, + /// `None` = the consumer has not recorded a number yet — which makes `consumer-max` underived. + pub max_prompt_tokens: Option, + /// `measured` or `plan`, as the consumer's own record says. + pub basis: String, + pub source: String, +} + +/// One context rung, as declared. `tokens` is `None` for the derived rung until [`derive_rungs`] fills it. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ContextRung { + pub id: String, + pub tokens: Option, + /// A LONG rung is owed only by the models `ladder.cells.long_rungs_for` names (#3712 amendment 2); a short + /// rung is owed by every model it fits. + pub long: bool, + pub derived_from: Vec, + /// Consumers the derived rung could not include because they recorded no number. + pub unmeasured_consumers: Vec, +} + +/// The context-rung declaration as read: the rungs (with `consumer-max` derived) and the consumer records. +pub type ContextDecl = (Vec, Vec); + +/// Read the context-rung declaration. `Ok(None)` when the file is absent — which the shape turns into a +/// violation (a release with no context rung proves nothing about context); a present-but-foreign file is exit 3. +pub fn read_context_rungs(root: &Path) -> Result, ReleaseError> { + let path = root.join(CONTEXT_RUNGS_FILE); + if !path.exists() { + return Ok(None); + } + let v = read_json(&path, CONTEXT_RUNGS_FILE, CONTEXT_SCHEMA)?; + let consumers: Vec = list(&v, "consumers") + .map(|c| Consumer { + name: str_of(c, "consumer").unwrap_or_default(), + max_prompt_tokens: c + .get("max_prompt_tokens") + .and_then(serde_json::Value::as_u64), + basis: str_of(c, "basis").unwrap_or_default(), + source: str_of(c, "source").unwrap_or_default(), + }) + .collect(); + if let Some(r) = list(&v, "rungs").find(|r| str_of(r, "id").as_deref() == Some(DECLARED)) { + return Err(input_err( + CONTEXT_RUNGS_FILE, + format!("rung {r} redeclares `{DECLARED}`, which is each model's own GGUF context_length and is never written down"), + )); + } + let rungs = list(&v, "rungs") + .map(|r| ContextRung { + id: str_of(r, "id").unwrap_or_default(), + tokens: r.get("tokens").and_then(serde_json::Value::as_u64), + long: r + .get("long") + .and_then(serde_json::Value::as_bool) + .unwrap_or(false), + derived_from: list(r, "derived_from") + .filter_map(serde_json::Value::as_str) + .map(str::to_string) + .collect(), + unmeasured_consumers: Vec::new(), + }) + .collect(); + Ok(Some((derive_rungs(rungs, &consumers), consumers))) +} + +/// `consumer-max` is COMPUTED here: the max over every consumer record that states a number, derived from every +/// consumer's source. A declared `tokens` on it is ignored, and a consumer with no number is carried by name so +/// the shape can refuse the rung (a max over a subset is a smaller number wearing the full name). +#[must_use] +pub fn derive_rungs(rungs: Vec, consumers: &[Consumer]) -> Vec { + rungs + .into_iter() + .map(|mut r| { + if r.id == CONSUMER_MAX { + r.tokens = consumers.iter().filter_map(|c| c.max_prompt_tokens).max(); + r.derived_from = consumers + .iter() + .filter(|c| !c.source.is_empty()) + .map(|c| format!("{}: {}", c.name, c.source)) + .collect(); + r.unmeasured_consumers = consumers + .iter() + .filter(|c| c.max_prompt_tokens.is_none()) + .map(|c| c.name.clone()) + .collect(); + } + r + }) + .collect() +} + +/// The dogfood receipt R5 judged (`scripts/dogfood.sh`: `{verdict, commit, version, phase, deferred, …}`). +/// Only verdict, commit and version are read: the pre-publish deferral whitelist is R5's, and a second copy of it +/// here would be a second declaration. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Dogfood { + pub file: String, + pub verdict: String, + pub commit: String, + pub version: String, +} + +pub fn read_dogfood(path: &Path) -> Result { + let name = path.display().to_string(); + let text = + std::fs::read_to_string(path).map_err(|e| input_err(&name, format!("unreadable: {e}")))?; + let v: serde_json::Value = + serde_json::from_str(&text).map_err(|e| input_err(&name, format!("not JSON: {e}")))?; + Ok(Dogfood { + file: name, + verdict: str_of(&v, "verdict").unwrap_or_default(), + commit: str_of(&v, "commit") + .unwrap_or_default() + .to_ascii_lowercase(), + version: str_of(&v, "version").unwrap_or_default(), + }) +} + +/// One inventory model's dispatch path on one host: the kernels `apr parity --per-op` reports for it. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Dispatch { + pub sha256: Option, + pub file: String, + /// (kernel, quant). A GEMV is keyed on its quant — qwen3moe runs `ffn_down_exps` as Q4_K in 24 layers and + /// Q6_K in 24 (#3712, aprender-eb) — so a kernel listed as a bare string carries quant `""`. + pub kernels: Vec<(String, String)>, +} + +/// One kernel-vs-reference measurement. +#[derive(Debug, Clone, PartialEq)] +pub struct KernelRow { + pub kernel: String, + pub quant: String, + pub reference: String, + pub max_err: Option, + /// A DISCRETE kernel (router top-k): token×layer slots whose selected sets differ, and the largest CPU logit + /// gap between rank k and k+1 among them. `max_err` means nothing when the two sides chose different sets. + pub index_mismatch: Option, + pub tie_margin: Option, + pub bound: Option, + pub bound_source: String, + pub verdict: String, + pub reason: String, +} + +/// One kernel-diff receipt file (`apr-kernel-diff-receipt/v1`). +#[derive(Debug, Clone, PartialEq)] +pub struct KernelReceipt { + pub file: String, + pub host: String, + pub version: String, + pub apr_sha: Option, + pub sm: String, + pub dispatch: Vec, + pub rows: Vec, +} + +/// Every `*.json` under `dir`, in byte order (none when the dir is absent — the shape names the host). +pub fn read_kernel_receipts(dir: &Path, root: &Path) -> Result, ReleaseError> { + let mut files = Vec::new(); + walk_json(dir, &mut files); + files.sort(); + files + .iter() + .map(|f| { + let name = f + .strip_prefix(root) + .unwrap_or(f) + .to_string_lossy() + .replace('\\', "/"); + let v = read_json(f, &name, KERNEL_SCHEMA)?; + Ok(parse_kernel_receipt(&name, &v)) + }) + .collect() +} + +fn walk_json(dir: &Path, out: &mut Vec) { + let Ok(rd) = std::fs::read_dir(dir) else { + return; + }; + for e in rd.flatten() { + let p = e.path(); + if p.is_dir() { + walk_json(&p, out); + } else if p.extension().and_then(|x| x.to_str()) == Some("json") { + out.push(p); + } + } +} + +fn parse_kernel_receipt(file: &str, v: &serde_json::Value) -> KernelReceipt { + KernelReceipt { + file: file.to_string(), + host: str_of(v, "host").unwrap_or_default(), + version: str_of(v, "version").unwrap_or_default(), + apr_sha: str_of(v, "apr_sha").map(|s| s.to_ascii_lowercase()), + sm: str_of(v, "sm").unwrap_or_default(), + dispatch: list(v, "dispatch") + .map(|d| Dispatch { + sha256: str_of(d, "sha256").map(|s| s.to_ascii_lowercase()), + file: str_of(d, "file").unwrap_or_default(), + kernels: list(d, "kernels").filter_map(kernel_key).collect(), + }) + .collect(), + rows: list(v, "rows") + .map(|r| KernelRow { + kernel: str_of(r, "kernel").unwrap_or_default(), + quant: str_of(r, "quant").unwrap_or_default().to_ascii_lowercase(), + reference: str_of(r, "reference").unwrap_or_default(), + max_err: r.get("max_err").and_then(serde_json::Value::as_f64), + index_mismatch: r.get("index_mismatch").and_then(serde_json::Value::as_u64), + tie_margin: r.get("tie_margin").and_then(serde_json::Value::as_f64), + bound: r.get("bound").and_then(serde_json::Value::as_f64), + bound_source: str_of(r, "bound_source").unwrap_or_default(), + verdict: str_of(r, "verdict") + .unwrap_or_default() + .to_ascii_lowercase(), + reason: str_of(r, "reason").unwrap_or_default(), + }) + .collect(), + } +} + +/// A dispatch entry: `"kernel"` or `{"kernel": …, "quant": …}`. +fn kernel_key(k: &serde_json::Value) -> Option<(String, String)> { + match k { + serde_json::Value::String(s) => Some((s.clone(), String::new())), + serde_json::Value::Object(_) => Some(( + str_of(k, "kernel")?, + str_of(k, "quant").unwrap_or_default().to_ascii_lowercase(), + )), + _ => None, + } +} + +impl KernelRow { + /// Within the measured bound. Continuous: `max_err ≤ bound`. Discrete (`index_mismatch` present): no mismatch, + /// or every mismatch a near-tie (`tie_margin ≤ bound`). Anything unmeasured is outside. + #[must_use] + pub fn within_bound(&self) -> bool { + match self.index_mismatch { + Some(0) => true, + Some(_) => matches!((self.tie_margin, self.bound), (Some(t), Some(b)) if t <= b), + None => matches!((self.max_err, self.bound), (Some(e), Some(b)) if e <= b), + } + } +} + +/// One model's tokenizer comparison (#3726): apr's ids vs the pinned llama.cpp on a named corpus. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct TokRow { + pub sha256: Option, + pub file: String, + pub family: String, + pub corpus_sha256: String, + pub comparator: String, + pub comparator_sha: String, + pub tokens_compared: Option, + pub mismatches: Option, + /// `decode(encode(x)) == x` over the corpus. + pub roundtrip_ok: Option, + pub verdict: String, + pub reason: String, +} + +impl TokRow { + /// Identical ids over a corpus that is not empty: zero tokens compared is not parity, it is no measurement. + #[must_use] + pub fn exact(&self) -> bool { + self.mismatches == Some(0) && self.tokens_compared.is_some_and(|n| n > 0) + } +} + +/// One tokenizer-parity receipt file. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct TokReceipt { + pub file: String, + pub version: String, + pub apr_sha: Option, + pub rows: Vec, +} + +/// Every `*.json` under `dir` (none when absent — every model's tokenizer cell is then a missing row). +pub fn read_tokenizer_receipts(dir: &Path, root: &Path) -> Result, ReleaseError> { + let mut files = Vec::new(); + walk_json(dir, &mut files); + files.sort(); + files + .iter() + .map(|f| { + let name = f + .strip_prefix(root) + .unwrap_or(f) + .to_string_lossy() + .replace('\\', "/"); + let v = read_json(f, &name, TOKENIZER_SCHEMA)?; + Ok(TokReceipt { + file: name, + version: str_of(&v, "version").unwrap_or_default(), + apr_sha: str_of(&v, "apr_sha").map(|s| s.to_ascii_lowercase()), + rows: list(&v, "rows").map(parse_tok_row).collect(), + }) + }) + .collect() +} + +fn parse_tok_row(r: &serde_json::Value) -> TokRow { + TokRow { + sha256: str_of(r, "sha256").map(|s| s.to_ascii_lowercase()), + file: str_of(r, "file").unwrap_or_default(), + family: str_of(r, "family").unwrap_or_default(), + corpus_sha256: str_of(r, "corpus_sha256").unwrap_or_default(), + comparator: str_of(r, "comparator").unwrap_or_default(), + comparator_sha: str_of(r, "comparator_sha").unwrap_or_default(), + tokens_compared: r.get("tokens_compared").and_then(serde_json::Value::as_u64), + mismatches: r.get("mismatches").and_then(serde_json::Value::as_u64), + roundtrip_ok: r.get("roundtrip_ok").and_then(serde_json::Value::as_bool), + verdict: str_of(r, "verdict") + .unwrap_or_default() + .to_ascii_lowercase(), + reason: str_of(r, "reason").unwrap_or_default(), + } +} diff --git a/crates/aprender-contracts/src/ontology/rdf.rs b/crates/aprender-contracts/src/ontology/rdf.rs index 901dc93f06..f7949781de 100644 --- a/crates/aprender-contracts/src/ontology/rdf.rs +++ b/crates/aprender-contracts/src/ontology/rdf.rs @@ -221,6 +221,19 @@ pub fn iri(kind: &str, id: &str) -> String { format!("{ONT_BASE}{kind}/{}", percent_encode(id)) } +/// A multi-segment instance IRI, `///…`, each segment percent-encoded on its own so a `/` +/// inside one (a file name, a host) cannot fake a boundary — and the path still reads as the cell it names +/// (`release-cell/0.69.1/lambda//chat/consumer-max`, aprender#3715). +#[must_use] +pub fn iri_path(kind: &str, segments: &[&str]) -> String { + let mut out = format!("{ONT_BASE}{kind}"); + for s in segments { + out.push('/'); + out.push_str(&percent_encode(s)); + } + out +} + /// A vocabulary term: `https://ont.paiml.dev/v1alpha1/`. `ont:Contract`, `ont:id`, `ont:depends_on`, …. #[must_use] pub fn ont(name: &str) -> String { diff --git a/crates/aprender-contracts/src/ontology/receipts.rs b/crates/aprender-contracts/src/ontology/receipts.rs index 1e23034dae..f92cea8b07 100644 --- a/crates/aprender-contracts/src/ontology/receipts.rs +++ b/crates/aprender-contracts/src/ontology/receipts.rs @@ -1,9 +1,12 @@ //! ONT-001 §5 ONT-4c1 — `resolves: receipt`: the tracked model-capability-ladder receipts, read once, joined //! to the rungs by sha256, and materialized as edges a shape can count. //! -//! One reader, one schema: `evidence/dogfood/models//.json` with -//! `schema: apr-model-ladder-receipt/v1` (aprender `scripts/model_ladder.sh`). Any other `schema` value is -//! refused BY NAME — a receipt this reader does not understand is not silently read as empty. A rung row with +//! One reader, two versions of one schema: `evidence/dogfood/models//.json` with +//! `schema: apr-model-ladder-receipt/v1` or `/v2` (aprender `scripts/model_ladder.sh`; v2 is #3712's, and adds the +//! host's measured `inventory[]`, the full `apr_sha`, and — with the widened 0.69.1 bar — `cells[]`, one row per +//! (model × verb × context rung), which the release-readiness resolver reads, #3715). Any other `schema` value is +//! refused BY NAME — a receipt this reader does not understand is not silently read as empty. v2 rows are v1 rows +//! plus `file`/`inventory_only`, so the rung join below reads both versions the same way. A rung row with //! `sha256` equal to the contract's is a **witness**; a row WITHOUT `sha256` is not a witness (the 0.68.1 //! receipts committed before aprender#3507 have none — "no measurement", never a match); a row with a //! different hex is a **hex mismatch**, materialized so a shape rejects it naming rung, receipt and both hexes. @@ -32,6 +35,10 @@ use crate::ontology::extract::gguf::{model, Rung}; use crate::ontology::rdf::{iri, Graph, Term, RDF_TYPE}; pub const SCHEMA: &str = "apr-model-ladder-receipt/v1"; +/// #3712's successor: v1 plus `inventory[]`, `apr_sha` and `cells[]`. +pub const SCHEMA_V2: &str = "apr-model-ladder-receipt/v2"; +/// Every schema this reader understands, in version order. +pub const SCHEMAS: [&str; 2] = [SCHEMA, SCHEMA_V2]; /// Where the receipts live, relative to the repository root. pub const EVIDENCE_DIR: &str = "evidence/dogfood/models"; @@ -50,17 +57,87 @@ pub struct Row { pub backends: BTreeMap, } -/// One receipt file. +/// One model the host HOLDS (v2 `inventory[]`): the universe the release must prove, measured on the host. #[derive(Debug, Clone, PartialEq, Eq)] +pub struct InventoryItem { + pub file: String, + /// `None` when the row carries no hash: no measurement, never a match. + pub sha256: Option, + pub arch: Option, + pub quant: Option, + /// The GGUF's own `*.context_length`, read from the file on the host (#3710: "not picked here"). + pub context_length: Option, + /// The thinking modes the model's chat template supports, DERIVED from `tokenizer.chat_template` (#3723, + /// the cop's rule): `enable_thinking` → `["on","off"]`; a generation prompt that always opens `` → + /// `["on"]`; otherwise `["off"]`. `None` (no template read) owes BOTH modes. + pub thinking_modes: Option>, + /// The evidence for `thinking_modes`: the markers found in `tokenizer.chat_template`, and that template's hash. + pub thinking_markers: Option>, + pub chat_template_sha256: Option, + /// The producer's ECHO of whether this model owes the long rungs; pv re-derives it from the ladder contract + /// and a disagreement is a violation (#3712 amendment 2). + pub owes_long_rungs: Option, + /// The declared memory arithmetic (#3710 rule, operator "a"): weights + KV per token (at the chosen + /// precision) + workspace, compared with the host's measured `gpu_mem_total_bytes`. + pub weights_bytes: Option, + pub kv_bytes_per_token: Option, + pub workspace_bytes: Option, + /// The KV precision `kv_bytes_per_token` was computed at (#3712, 62: device KV was measured as f32). + pub kv_dtype: Option, +} + +/// One (model × verb × thinking × context rung) row of v2 `cells[]` (#3712 / #3715). +#[derive(Debug, Clone, PartialEq)] +pub struct CellRow { + pub sha256: Option, + pub file: String, + pub verb: String, + /// `on` / `off` (#3710, operator: "chat with and without thinking, ditto run, ditto code"). + pub thinking: String, + pub context: String, + pub prompt_tokens: Option, + /// The output budget the request asked for; with `prompt_tokens`, what fills the declared context. + pub max_tokens: Option, + /// Thinking ON: did the think block close? `None` on a thinking-OFF row (and on a row that did not say). + pub think_closed: Option, + /// Characters of answer AFTER any think block. `Some(0)` is the empty answer #3720 forbids. + pub answer_chars: Option, + pub ttft_ms: Option, + /// A pre-load refusal's arithmetic: what the cell needs, and what the host has. `verdict: "refused"`. + pub required_bytes: Option, + pub available_bytes: Option, + pub verdict: String, + pub backend: String, + /// `None` when the row does not say — which is not `false`. + pub fallback: Option, + pub rc: Option, + pub reason: String, +} + +/// One receipt file. +#[derive(Debug, Clone, PartialEq)] pub struct Receipt { - /// Path relative to the repository root, `/`-separated. + /// Path relative to the repository root, `/`-separated (or as given, for a receipt dir outside the tree). pub file: String, + pub schema: String, pub host: String, pub version: String, pub sha: String, + /// v2: the full 40-hex commit the measured `apr` was built from. `None` on v1. + pub apr_sha: Option, pub cc: String, pub gpu: String, + /// v2: the host's measured TOTAL GPU memory (unified on GB10), the right-hand side of the fit arithmetic. + /// Owed iff required ≤ total; a refusal is honest only when required > total (#3712, 62): a co-tenant that + /// holds memory must not shrink the owed set. + pub gpu_mem_total_bytes: Option, + /// v2: free GPU memory when measured: carried for the reader, never part of the fit. + pub gpu_mem_free_bytes: Option, pub rows: Vec, + /// v2 `inventory[]`; empty on v1 (which cannot say what the host holds). + pub inventory: Vec, + /// v2 `cells[]`; empty when the measuring side wrote none. + pub cells: Vec, } /// A receipt file this reader refuses, by name. @@ -79,11 +156,16 @@ impl std::fmt::Display for ReceiptError { impl std::error::Error for ReceiptError {} /// Read every `*.json` under `/evidence/dogfood/models/**`, in byte order. A file whose `schema` is not -/// [`SCHEMA`] is an error naming the file and the schema it carries. +/// one of [`SCHEMAS`] is an error naming the file and the schema it carries. pub fn read_all(root: &Path) -> Result, ReceiptError> { - let dir = root.join(EVIDENCE_DIR); + read_dir(&root.join(EVIDENCE_DIR), root) +} + +/// Read every `*.json` under `dir/**`, in byte order, naming each file relative to `root` when it lies under it +/// (a T-1 receipt dir such as `$AP/models-t1` does not, and is named as given). +pub fn read_dir(dir: &Path, root: &Path) -> Result, ReceiptError> { let mut files = Vec::new(); - walk(&dir, &mut files); + walk(dir, &mut files); files.sort(); let mut out = Vec::new(); for f in files { @@ -127,9 +209,10 @@ pub fn parse(file: &str, text: &str) -> Result { .get("schema") .and_then(serde_json::Value::as_str) .unwrap_or(""); - if schema != SCHEMA { + if !SCHEMAS.contains(&schema) { return Err(err(format!( - "schema {schema:?} is not {SCHEMA} — refused by name" + "schema {schema:?} is not {} — refused by name", + SCHEMAS.join(" or ") ))); } let s = |k: &str| { @@ -140,70 +223,115 @@ pub fn parse(file: &str, text: &str) -> Result { }) .unwrap_or_default() }; - let mut rows = Vec::new(); - for r in v - .get("rungs") - .and_then(serde_json::Value::as_array) - .into_iter() - .flatten() - { - let b = |k: &str| { - r.get(k) - .and_then(serde_json::Value::as_bool) - .unwrap_or(false) - }; - let cap = r.get("capability_match"); - let capability_passed = cap - .and_then(|c| c.get("passed")) - .and_then(serde_json::Value::as_bool) - .unwrap_or(false) - && !cap - .and_then(|c| c.get("skipped")) - .and_then(serde_json::Value::as_bool) - .unwrap_or(false); - let mut backends = BTreeMap::new(); - if let Some(bs) = r.get("backends").and_then(serde_json::Value::as_object) { - for (name, st) in bs { - let ran = st - .get("ran") - .and_then(serde_json::Value::as_bool) - .unwrap_or(false); - let fallback = st - .get("fallback") - .and_then(serde_json::Value::as_bool) - .unwrap_or(false); - backends.insert(name.clone(), (ran, fallback)); - } - } - rows.push(Row { - id: r - .get("id") - .and_then(serde_json::Value::as_str) - .unwrap_or_default() - .to_string(), - present: b("present"), - sha_ok: b("sha_ok"), - sha256: r - .get("sha256") - .and_then(serde_json::Value::as_str) - .map(str::to_ascii_lowercase), - required: b("required"), - green: b("green"), - capability_passed, - backends, - }); - } Ok(Receipt { file: file.to_string(), + schema: schema.to_string(), host: s("host"), version: s("version"), sha: s("sha"), + apr_sha: str_of(&v, "apr_sha").map(|x| x.to_ascii_lowercase()), cc: s("cc"), gpu: s("gpu"), - rows, + gpu_mem_total_bytes: v + .get("gpu_mem_total_bytes") + .and_then(serde_json::Value::as_u64), + gpu_mem_free_bytes: v + .get("gpu_mem_free_bytes") + .and_then(serde_json::Value::as_u64), + rows: list(&v, "rungs").map(parse_row).collect(), + inventory: list(&v, "inventory").map(parse_inventory_item).collect(), + cells: list(&v, "cells").map(parse_cell).collect(), }) } +/// The elements of array `key`, or none. +fn list<'a>(v: &'a serde_json::Value, key: &str) -> impl Iterator { + v.get(key) + .and_then(serde_json::Value::as_array) + .into_iter() + .flatten() +} + +fn str_of(v: &serde_json::Value, key: &str) -> Option { + v.get(key) + .and_then(serde_json::Value::as_str) + .map(str::to_string) +} + +fn bool_of(v: &serde_json::Value, key: &str) -> bool { + v.get(key) + .and_then(serde_json::Value::as_bool) + .unwrap_or(false) +} + +/// `passed ∧ ¬skipped`: a skipped gate reports `passed: true`, and that is not a pass. +fn gate_passed(cap: Option<&serde_json::Value>) -> bool { + cap.is_some_and(|c| bool_of(c, "passed") && !bool_of(c, "skipped")) +} + +fn parse_row(r: &serde_json::Value) -> Row { + let mut backends = BTreeMap::new(); + if let Some(bs) = r.get("backends").and_then(serde_json::Value::as_object) { + for (name, st) in bs { + backends.insert(name.clone(), (bool_of(st, "ran"), bool_of(st, "fallback"))); + } + } + Row { + id: str_of(r, "id").unwrap_or_default(), + present: bool_of(r, "present"), + sha_ok: bool_of(r, "sha_ok"), + sha256: str_of(r, "sha256").map(|x| x.to_ascii_lowercase()), + required: bool_of(r, "required"), + green: bool_of(r, "green"), + capability_passed: gate_passed(r.get("capability_match")), + backends, + } +} + +fn parse_inventory_item(r: &serde_json::Value) -> InventoryItem { + InventoryItem { + file: str_of(r, "file").unwrap_or_default(), + sha256: str_of(r, "sha256").map(|x| x.to_ascii_lowercase()), + arch: str_of(r, "arch"), + quant: str_of(r, "quant"), + context_length: r.get("context_length").and_then(serde_json::Value::as_u64), + thinking_modes: strings(r, "thinking_modes"), + thinking_markers: strings(r, "thinking_markers"), + chat_template_sha256: str_of(r, "chat_template_sha256"), + owes_long_rungs: r + .get("owes_long_rungs") + .and_then(serde_json::Value::as_bool), + weights_bytes: r.get("weights_bytes").and_then(serde_json::Value::as_u64), + kv_bytes_per_token: r + .get("kv_bytes_per_token") + .and_then(serde_json::Value::as_u64), + workspace_bytes: r.get("workspace_bytes").and_then(serde_json::Value::as_u64), + kv_dtype: str_of(r, "kv_dtype"), + } +} + +fn parse_cell(r: &serde_json::Value) -> CellRow { + CellRow { + sha256: str_of(r, "sha256").map(|x| x.to_ascii_lowercase()), + file: str_of(r, "file").unwrap_or_default(), + verb: str_of(r, "verb").unwrap_or_default(), + thinking: str_of(r, "thinking").unwrap_or_default(), + context: str_of(r, "context").unwrap_or_default(), + prompt_tokens: r.get("prompt_tokens").and_then(serde_json::Value::as_u64), + max_tokens: r.get("max_tokens").and_then(serde_json::Value::as_u64), + think_closed: r.get("think_closed").and_then(serde_json::Value::as_bool), + answer_chars: r.get("answer_chars").and_then(serde_json::Value::as_u64), + ttft_ms: r.get("ttft_ms").and_then(serde_json::Value::as_f64), + required_bytes: r.get("required_bytes").and_then(serde_json::Value::as_u64), + available_bytes: r.get("available_bytes").and_then(serde_json::Value::as_u64), + verdict: str_of(r, "verdict").unwrap_or_default(), + backend: str_of(r, "backend").unwrap_or_default(), + fallback: r.get("fallback").and_then(serde_json::Value::as_bool), + rc: r.get("rc").and_then(serde_json::Value::as_i64), + reason: str_of(r, "reason").unwrap_or_default(), + } +} + /// What resolution found, for the gate's report. #[derive(Debug, Clone, Default, PartialEq, Eq)] pub struct ResolveStats { @@ -344,6 +472,16 @@ fn emit_receipt_node(g: &mut Graph, rec: &Receipt, row: &Row, rung: &Rung) -> St node } +/// An array of strings at `key`, or `None` when the key is absent (which is not an empty list). +fn strings(r: &serde_json::Value, key: &str) -> Option> { + r.get(key).and_then(serde_json::Value::as_array).map(|a| { + a.iter() + .filter_map(serde_json::Value::as_str) + .map(str::to_string) + .collect() + }) +} + #[cfg(test)] mod tests { use super::*; @@ -378,6 +516,42 @@ mod tests { let e = parse("x.json", r#"{"schema":"something/v9","rungs":[]}"#).expect_err("refused"); assert!(e.what.contains("something/v9"), "{e}"); assert!(e.what.contains(SCHEMA), "{e}"); + assert!(e.what.contains(SCHEMA_V2), "{e}"); + } + + #[test] + fn a_v2_receipt_joins_its_rungs_like_v1_and_carries_inventory_apr_sha_and_cells() { + // #3712's v2: the rung rows are v1 rows plus `file`/`inventory_only`, so the ladder join is unchanged — + // the arm that goes RED when v2 is refused (the collision #3715 closes) or read as empty. + let row = OK.replace( + r#""id":"a","#, + r#""id":"a","file":"a.gguf","inventory_only":false,"#, + ); + let text = format!( + r#"{{"schema":"apr-model-ladder-receipt/v2","host":"lambda","version":"0.69.1","sha":"abc", + "apr_sha":"ABCDEF0123456789ABCDEF0123456789ABCDEF01","cc":"8.9","gpu":"g", + "inventory":[{{"file":"a.gguf","sha256":"AAAA","bytes":4}}], + "cells":[{{"sha256":"aaaa","file":"a.gguf","verb":"chat","context":"golden","prompt_tokens":40, + "verdict":"pass","backend":"cuda","fallback":false,"rc":0,"reason":""}}], + "rungs":[{row}]}}"# + ); + let rec = parse("evidence/dogfood/models/0.69.1/lambda.json", &text).expect("v2 parses"); + assert_eq!(rec.schema, SCHEMA_V2); + assert_eq!( + rec.apr_sha.as_deref(), + Some("abcdef0123456789abcdef0123456789abcdef01") + ); + assert_eq!(rec.inventory.len(), 1); + assert_eq!(rec.inventory[0].sha256.as_deref(), Some("aaaa")); + assert_eq!(rec.cells.len(), 1); + assert_eq!(rec.cells[0].verb, "chat"); + assert_eq!(rec.cells[0].fallback, Some(false)); + let mut g = Graph::new(); + let st = resolve(&mut g, &[rung("a", "aaaa", &[], true)], &[rec]); + assert_eq!((st.witnesses, st.green_on), (1, 1)); + let v3 = + parse("x.json", r#"{"schema":"apr-model-ladder-receipt/v3"}"#).expect_err("v3 refused"); + assert!(v3.what.contains("v3"), "{v3}"); } #[test] diff --git a/crates/aprender-core/src/format/gguf/api.rs b/crates/aprender-core/src/format/gguf/api.rs index 490cd15ae3..eba45f1764 100644 --- a/crates/aprender-core/src/format/gguf/api.rs +++ b/crates/aprender-core/src/format/gguf/api.rs @@ -339,9 +339,11 @@ pub fn load_gguf_raw>(path: P) -> Result { // Contract: apr-inspect-metadata-propagation-v1 F-INSPECT-META-001 (paiml/aprender#622). // Propagate raw GGUF KV metadata to downstream consumers (inspect/rosetta) so they // can display authentic on-disk keys instead of fabricated ML-shorthand names. + // #3733: EVERY header key, including those outside the reader's parse allowlist. let raw_metadata: BTreeMap = reader .metadata .iter() + .chain(&reader.display_only_metadata) .map(|(k, v)| (k.clone(), gguf_value_display(v))) .collect(); diff --git a/crates/aprender-core/src/format/gguf/api_tests.rs b/crates/aprender-core/src/format/gguf/api_tests.rs index 069d46f1c2..7fdbcc11a6 100644 --- a/crates/aprender-core/src/format/gguf/api_tests.rs +++ b/crates/aprender-core/src/format/gguf/api_tests.rs @@ -449,3 +449,6 @@ fn test_gguf_model_config_clone() { #[path = "api_tests_raw_tensor_rope.rs"] mod api_tests_raw_tensor_rope; + +#[path = "api_tests_all_keys_3733.rs"] +mod api_tests_all_keys_3733; diff --git a/crates/aprender-core/src/format/gguf/api_tests_all_keys_3733.rs b/crates/aprender-core/src/format/gguf/api_tests_all_keys_3733.rs new file mode 100644 index 0000000000..6c7e1cf37b --- /dev/null +++ b/crates/aprender-core/src/format/gguf/api_tests_all_keys_3733.rs @@ -0,0 +1,149 @@ +//! #3733: `apr inspect --json` must show EVERY key the GGUF header carries. +//! +//! The reader's parse allowlist (`tokenizer.`, `general.`, six arch prefixes) +//! decided what inspect could show, so a Qwen3.5 file listed 24 of its 46 header +//! keys and none of the 18 `qwen35.*` ones: `qwen35.` does not start with +//! `qwen3.`. The fixture below uses that prefix, a prefix nothing has heard of, +//! and the `quantize.*` group real quantized files carry. + +use super::*; +use crate::format::gguf::GGUF_MAGIC; +use crate::format::rosetta::RosettaStone; + +/// Every key the fixture's header carries. The count assertions compare against +/// THIS list, which is exactly what `export_tensors_to_gguf` writes. +fn header() -> Vec<(String, GgufValue)> { + vec![ + ( + "general.architecture".into(), + GgufValue::String("qwen35".into()), + ), + ( + "general.name".into(), + GgufValue::String("pygmy-hybrid".into()), + ), + ("qwen35.context_length".into(), GgufValue::Uint32(262_144)), + ("qwen35.block_count".into(), GgufValue::Uint32(32)), + ( + "qwen35.full_attention_interval".into(), + GgufValue::Uint32(4), + ), + ( + "qwen35.attention.head_count_kv".into(), + GgufValue::Uint32(4), + ), + ("qwen35.attention.key_length".into(), GgufValue::Uint32(256)), + ( + "qwen35.rope.dimension_sections".into(), + GgufValue::ArrayInt32(vec![11, 11, 10, 0]), + ), + ("qwen35.rope.freq_base".into(), GgufValue::Float32(1.0e7)), + ( + "quantize.imatrix.file".into(), + GgufValue::String("imatrix.gguf".into()), + ), + ("zzz_never_heard_of.depth".into(), GgufValue::Uint64(7)), + ( + "tokenizer.ggml.model".into(), + GgufValue::String("gpt2".into()), + ), + ] +} + +fn fixture() -> NamedTempFile { + let file = NamedTempFile::with_suffix(".gguf").expect("temp gguf"); + let tensors = vec![GgufTensor { + name: "token_embd.weight".to_string(), + shape: vec![4, 4], + dtype: GgmlType::F32, + data: vec![0u8; 64], + }]; + let mut bytes = Vec::new(); + export_tensors_to_gguf(&mut bytes, &tensors, &header()).expect("export gguf"); + std::fs::write(file.path(), &bytes).expect("write gguf"); + file +} + +fn assert_every_header_key(shown: &BTreeMap, layer: &str) { + let header = header(); + let missing: Vec<&str> = header + .iter() + .map(|(k, _)| k.as_str()) + .filter(|k| !shown.contains_key(*k)) + .collect(); + assert!( + missing.is_empty(), + "{layer} dropped header keys: {missing:?}" + ); + assert_eq!( + shown.len(), + header.len(), + "{layer} must show exactly the header's KV count" + ); +} + +#[test] +fn raw_metadata_carries_every_header_key() { + let file = fixture(); + let raw = load_gguf_raw(file.path()).expect("load"); + assert_every_header_key(&raw.raw_metadata, "load_gguf_raw"); + assert_eq!(raw.raw_metadata["qwen35.context_length"], "262144"); + assert_eq!( + raw.raw_metadata["qwen35.rope.dimension_sections"], + "[len=4]" + ); + assert_eq!(raw.raw_metadata["zzz_never_heard_of.depth"], "7"); +} + +/// The path `apr inspect` takes for a GGUF (`run_rosetta_inspect`). +#[test] +fn rosetta_inspect_carries_every_header_key() { + let file = fixture(); + let report = RosettaStone::new().inspect(file.path()).expect("inspect"); + assert_every_header_key(&report.metadata, "RosettaStone::inspect"); +} + +/// The keys are shown, not consumed: the config accessors still read only the +/// allowlisted map, so an architecture the loaders do not know gets no config it +/// never had (a widened accessor would hand `apr import` a qwen35 hidden size). +#[test] +fn display_only_keys_do_not_reach_the_config_accessors() { + let file = fixture(); + let reader = GgufReader::from_file(file.path()).expect("read"); + assert!(reader + .display_only_metadata + .contains_key("qwen35.context_length")); + assert!(!reader.metadata.contains_key("qwen35.context_length")); + assert_eq!(reader.context_length(), None); + assert_eq!(reader.num_layers(), None); + + // `keep_all` (the shard merge) still puts everything in `metadata`. + let full = GgufReader::from_file_full(file.path()).expect("read full"); + assert!(full.display_only_metadata.is_empty()); + assert_eq!(full.metadata.len(), header().len()); +} + +/// Parsing (not skipping) keys outside the allowlist must not turn a hostile +/// array count into an allocation or an out-of-bounds index. +#[test] +fn an_array_count_past_the_end_of_the_file_is_an_error_not_a_panic() { + for elem_type in [4u32, 5, 6, 8] { + let key = "zzz.hostile"; + let mut data = Vec::new(); + data.extend_from_slice(&GGUF_MAGIC.to_le_bytes()); + data.extend_from_slice(&3u32.to_le_bytes()); + data.extend_from_slice(&0u64.to_le_bytes()); // tensors + data.extend_from_slice(&1u64.to_le_bytes()); // one KV + data.extend_from_slice(&(key.len() as u64).to_le_bytes()); + data.extend_from_slice(key.as_bytes()); + data.extend_from_slice(&9u32.to_le_bytes()); // array + data.extend_from_slice(&elem_type.to_le_bytes()); + data.extend_from_slice(&(u64::MAX / 2).to_le_bytes()); + data.extend_from_slice(&[0u8; 16]); + let err = GgufReader::from_bytes(data).expect_err("a count past EOF must be refused"); + assert!( + err.to_string().contains("overruns"), + "type {elem_type}: {err}" + ); + } +} diff --git a/crates/aprender-core/src/format/gguf/builder.rs b/crates/aprender-core/src/format/gguf/builder.rs index 5aca8570aa..91f5c7bcd1 100644 --- a/crates/aprender-core/src/format/gguf/builder.rs +++ b/crates/aprender-core/src/format/gguf/builder.rs @@ -161,10 +161,13 @@ fn test_from_bytes_tensor_at_max_dims() { } // ======================================================================== -// skip_metadata_value Tests (via from_bytes with non-parsed keys) +// Keys outside the parse allowlist (via from_bytes with non-parsed keys). +// They were skipped by length arithmetic until #3733; they are now decoded into +// `display_only_metadata`, so `apr inspect` shows them, and still kept out of +// `metadata`, which the config accessors read. // ======================================================================== -/// Build a GGUF with metadata that will be skipped (key prefix not in parsed set) +/// Build a GGUF with one metadata key outside the parse allowlist fn build_gguf_with_skipped_metadata(key: &str, value_type: u32, value_bytes: &[u8]) -> Vec { let mut data = Vec::new(); @@ -175,7 +178,7 @@ fn build_gguf_with_skipped_metadata(key: &str, value_type: u32, value_bytes: &[u data.extend_from_slice(&1u64.to_le_bytes()); // metadata_count = 1 // Metadata KV: key with prefix that does NOT match tokenizer./general./llama./qwen2./phi./mistral. - // so it will be skipped via skip_metadata_value + // so it lands in display_only_metadata (#3733) data.extend_from_slice(&(key.len() as u64).to_le_bytes()); data.extend_from_slice(key.as_bytes()); data.extend_from_slice(&value_type.to_le_bytes()); @@ -189,6 +192,7 @@ fn test_skip_metadata_value_uint8() { let data = build_gguf_with_skipped_metadata("custom.u8", 0, &[42u8]); let reader = GgufReader::from_bytes(data).expect("skip uint8"); assert!(!reader.metadata.contains_key("custom.u8")); + assert!(reader.display_only_metadata.contains_key("custom.u8")); } #[test] @@ -196,6 +200,7 @@ fn test_skip_metadata_value_int8() { let data = build_gguf_with_skipped_metadata("custom.i8", 1, &[0xFEu8]); let reader = GgufReader::from_bytes(data).expect("skip int8"); assert!(!reader.metadata.contains_key("custom.i8")); + assert!(reader.display_only_metadata.contains_key("custom.i8")); } #[test] @@ -203,6 +208,7 @@ fn test_skip_metadata_value_uint16() { let data = build_gguf_with_skipped_metadata("custom.u16", 2, &1000u16.to_le_bytes()); let reader = GgufReader::from_bytes(data).expect("skip uint16"); assert!(!reader.metadata.contains_key("custom.u16")); + assert!(reader.display_only_metadata.contains_key("custom.u16")); } #[test] @@ -210,6 +216,7 @@ fn test_skip_metadata_value_int16() { let data = build_gguf_with_skipped_metadata("custom.i16", 3, &(-500i16).to_le_bytes()); let reader = GgufReader::from_bytes(data).expect("skip int16"); assert!(!reader.metadata.contains_key("custom.i16")); + assert!(reader.display_only_metadata.contains_key("custom.i16")); } #[test] @@ -217,6 +224,7 @@ fn test_skip_metadata_value_bool() { let data = build_gguf_with_skipped_metadata("custom.flag", 7, &[1u8]); let reader = GgufReader::from_bytes(data).expect("skip bool"); assert!(!reader.metadata.contains_key("custom.flag")); + assert!(reader.display_only_metadata.contains_key("custom.flag")); } #[test] @@ -229,6 +237,7 @@ fn test_skip_metadata_value_string() { let data = build_gguf_with_skipped_metadata("custom.str", 8, &value_bytes); let reader = GgufReader::from_bytes(data).expect("skip string"); assert!(!reader.metadata.contains_key("custom.str")); + assert!(reader.display_only_metadata.contains_key("custom.str")); } #[test] @@ -236,6 +245,7 @@ fn test_skip_metadata_value_uint64() { let data = build_gguf_with_skipped_metadata("custom.u64", 10, &999u64.to_le_bytes()); let reader = GgufReader::from_bytes(data).expect("skip uint64"); assert!(!reader.metadata.contains_key("custom.u64")); + assert!(reader.display_only_metadata.contains_key("custom.u64")); } #[test] @@ -243,6 +253,7 @@ fn test_skip_metadata_value_int64() { let data = build_gguf_with_skipped_metadata("custom.i64", 11, &(-1i64).to_le_bytes()); let reader = GgufReader::from_bytes(data).expect("skip int64"); assert!(!reader.metadata.contains_key("custom.i64")); + assert!(reader.display_only_metadata.contains_key("custom.i64")); } #[test] @@ -251,6 +262,7 @@ fn test_skip_metadata_value_float64() { build_gguf_with_skipped_metadata("custom.f64", 12, &std::f64::consts::E.to_le_bytes()); let reader = GgufReader::from_bytes(data).expect("skip float64"); assert!(!reader.metadata.contains_key("custom.f64")); + assert!(reader.display_only_metadata.contains_key("custom.f64")); } #[test] @@ -259,6 +271,7 @@ fn test_skip_metadata_value_unknown_type() { let data = build_gguf_with_skipped_metadata("custom.unk", 99, &[0u8; 4]); let reader = GgufReader::from_bytes(data).expect("skip unknown"); assert!(!reader.metadata.contains_key("custom.unk")); + assert!(reader.display_only_metadata.contains_key("custom.unk")); } #[test] diff --git a/crates/aprender-core/src/format/gguf/reader.rs b/crates/aprender-core/src/format/gguf/reader.rs index 198931c5a5..1e23f114a8 100644 --- a/crates/aprender-core/src/format/gguf/reader.rs +++ b/crates/aprender-core/src/format/gguf/reader.rs @@ -86,6 +86,28 @@ fn read_metadata_array(data: &[u8], offset: usize) -> Result<(GgufValue, usize)> let count = read_u64(data, offset + 4)? as usize; let mut consumed = 12; // type (4) + count (8) + // #3733: every header key is now parsed (keys outside the allowlist used to + // be skipped by length arithmetic alone), so a count taken from the file must + // not size an allocation or index past the end. The span is checked first; + // a string needs at least its 8-byte length, which bounds that count too. + let available = data.len().saturating_sub(offset + consumed); + let min_elem = match elem_type { + 0..=1 | 7 => 1, + 2..=3 => 2, + 8 | 10..=12 => 8, + _ => 4, + }; + if count + .checked_mul(min_elem) + .is_none_or(|need| need > available) + { + return Err(AprenderError::FormatError { + message: format!( + "GGUF metadata array of {count} elements (type {elem_type}) overruns the file" + ), + }); + } + match elem_type { 8 => { let mut strings = Vec::with_capacity(count); @@ -107,13 +129,7 @@ fn read_metadata_array(data: &[u8], offset: usize) -> Result<(GgufValue, usize)> 5 => { let mut values = Vec::with_capacity(count); for _ in 0..count { - let v = i32::from_le_bytes([ - data[offset + consumed], - data[offset + consumed + 1], - data[offset + consumed + 2], - data[offset + consumed + 3], - ]); - values.push(v); + values.push(read_i32_le(data, offset + consumed)?); consumed += 4; } Ok((GgufValue::ArrayInt32(values), consumed)) @@ -121,25 +137,13 @@ fn read_metadata_array(data: &[u8], offset: usize) -> Result<(GgufValue, usize)> 6 => { let mut values = Vec::with_capacity(count); for _ in 0..count { - let v = f32::from_le_bytes([ - data[offset + consumed], - data[offset + consumed + 1], - data[offset + consumed + 2], - data[offset + consumed + 3], - ]); - values.push(v); + values.push(read_f32_le(data, offset + consumed)?); consumed += 4; } Ok((GgufValue::ArrayFloat32(values), consumed)) } _ => { - let elem_size = match elem_type { - 0..=1 | 7 => 1, - 2..=3 => 2, - 10..=12 => 8, - _ => 4, - }; - consumed += count * elem_size; + consumed += count * min_elem; Ok((GgufValue::ArrayUint32(vec![]), consumed)) } } @@ -261,42 +265,6 @@ pub(crate) fn read_metadata_value( } } -/// Skip a metadata value (we don't need to parse all metadata types) -fn skip_metadata_value(data: &[u8], offset: usize, value_type: u32) -> Result { - match value_type { - 0..=1 | 7 => Ok(1), // Uint8, Int8, Bool - 2..=3 => Ok(2), // Uint16, Int16 - 8 => { - // String - let len = read_u64(data, offset)? as usize; - Ok(8 + len) - } - 9 => { - // Array - need to read element type and count, then skip elements - let elem_type = read_u32(data, offset)?; - let count = read_u64(data, offset + 4)? as usize; - let elem_size = match elem_type { - 0..=1 | 7 => 1, // Uint8, Int8, Bool - 2..=3 => 2, // Uint16, Int16 - 8 => { - // Array of strings - need to iterate - let mut skip = 12; // type (4) + count (8) - for _ in 0..count { - let (_, slen) = read_string(data, offset + skip)?; - skip += slen; - } - return Ok(skip); - } - 10..=12 => 8, // Uint64, Int64, Float64 - _ => 4, // Uint32, Int32, Float32, Unknown - }; - Ok(12 + count * elem_size) - } - 10..=12 => Ok(8), // Uint64, Int64, Float64 - _ => Ok(4), // Uint32, Int32, Float32, Unknown (4 bytes default) - } -} - /// Parsed GGUF file for import #[derive(Debug)] pub struct GgufReader { @@ -312,6 +280,16 @@ pub struct GgufReader { pub data_offset: usize, /// Metadata key-value pairs (extracted from GGUF) pub metadata: BTreeMap, + /// Every header key OUTSIDE the parse allowlist, kept for inventory only + /// (`apr inspect`). Empty when the reader was built with `keep_all`. + /// + /// #3733: the allowlist (`tokenizer.`, `general.`, and six arch prefixes) + /// decided what `apr inspect --json` could show, so a Qwen3.5 file lost all + /// 22 of its `qwen35.*` keys, `context_length` among them: `qwen35.` does not + /// start with `qwen3.`. The keys live here rather than in `metadata` because + /// the config accessors read `metadata`, and widening THEM would hand every + /// unlisted architecture a config the import path has never been given. + pub display_only_metadata: BTreeMap, } /// Tensor metadata from GGUF file diff --git a/crates/aprender-core/src/format/gguf/reader_parsing.rs b/crates/aprender-core/src/format/gguf/reader_parsing.rs index 9f7a0f5a25..5b8d5ea65a 100644 --- a/crates/aprender-core/src/format/gguf/reader_parsing.rs +++ b/crates/aprender-core/src/format/gguf/reader_parsing.rs @@ -26,9 +26,9 @@ impl GgufReader { Self::from_bytes_keep(data, false) } - /// Parse GGUF from bytes. When `keep_all` is true EVERY metadata key is - /// retained; otherwise only `tokenizer.` / `general.` / known-arch keys are - /// parsed (the rest skipped for efficiency). + /// Parse GGUF from bytes. When `keep_all` is true EVERY metadata key goes to + /// `metadata`; otherwise only `tokenizer.` / `general.` / known-arch keys do, + /// and the rest go to `display_only_metadata` (#3733). No key is dropped. pub fn from_bytes_keep(data: Vec, keep_all: bool) -> Result { if data.len() < 24 { return Err(AprenderError::FormatError { @@ -78,6 +78,7 @@ impl GgufReader { // Parse metadata section (extract vocabulary and other tokenizer data) let mut offset = 24; let mut metadata = BTreeMap::new(); + let mut display_only_metadata = BTreeMap::new(); for _ in 0..metadata_kv_count { // Read key let (key, key_len) = read_string(&data, offset)?; @@ -103,8 +104,10 @@ impl GgufReader { metadata.insert(key, value); offset += value_len; } else { - // Skip other metadata for efficiency - let value_len = skip_metadata_value(&data, offset, value_type)?; + // #3733: kept, not skipped. `apr inspect` must show every key the + // header carries; only the config accessors are kept to the list. + let (value, value_len) = read_metadata_value(&data, offset, value_type)?; + display_only_metadata.insert(key, value); offset += value_len; } } @@ -164,6 +167,7 @@ impl GgufReader { tensors, data_offset, metadata, + display_only_metadata, }) } diff --git a/crates/aprender-core/src/format/gguf/tests/core_tensor_export.rs b/crates/aprender-core/src/format/gguf/tests/core_tensor_export.rs index cd2531114e..c06dc5be17 100644 --- a/crates/aprender-core/src/format/gguf/tests/core_tensor_export.rs +++ b/crates/aprender-core/src/format/gguf/tests/core_tensor_export.rs @@ -121,6 +121,7 @@ fn make_test_reader(metadata: std::collections::BTreeMap) -> tensors: vec![], data_offset: 0, metadata, + display_only_metadata: std::collections::BTreeMap::new(), } } diff --git a/crates/aprender-orchestrate/src/bug_hunter/tests_patterns.rs b/crates/aprender-orchestrate/src/bug_hunter/tests_patterns.rs index 3d8ecef050..92affd6556 100644 --- a/crates/aprender-orchestrate/src/bug_hunter/tests_patterns.rs +++ b/crates/aprender-orchestrate/src/bug_hunter/tests_patterns.rs @@ -187,11 +187,12 @@ fn test_bh_mod_024_hunt_with_spec_basic() { let result = hunt_with_spec(&temp, &temp.join("spec.md"), None, config); assert!(result.is_ok()); - let (hunt_result, parsed_spec) = result.unwrap(); + let (_hunt_result, parsed_spec) = result.unwrap(); assert!(!parsed_spec.claims.is_empty()); assert_eq!(parsed_spec.claims[0].id, "TST-01"); - // Hunt ran (non-zero duration) or produced no findings - assert!(hunt_result.duration_ms > 0); + // #3703: no wall-clock assertion. `duration_ms` is a truncated `as_millis()`, which reads 0 + // when the hunt finishes in under a millisecond; `result.is_ok()` and the parsed claims above + // already prove the hunt ran. let _ = std::fs::remove_dir_all(&temp); } diff --git a/crates/aprender-serve/Cargo.toml b/crates/aprender-serve/Cargo.toml index 0e18d512f2..3128866767 100644 --- a/crates/aprender-serve/Cargo.toml +++ b/crates/aprender-serve/Cargo.toml @@ -47,6 +47,8 @@ name = "realizar" # High-performance SIMD compute library with GPU support # PAR-073: Using local path for development, version for crates.io trueno = { workspace = true, features = ["gpu"] } +# #3726: Unicode general categories for the canonical byte-level BPE pre-tokenizer +unicode-properties = { workspace = true } # CUDA PTX generation and runtime for NVIDIA GPUs (optional, pure Rust, no LLVM/nvcc) # v0.4.12: BatchedQ6K, MultiWarp, RopeNeox, FlashDecoding kernels trueno-gpu = { version = "0.69.0", path = "../aprender-gpu", package = "aprender-gpu", optional = true, features = ["cuda"] } # APR-MONO single-source-of-truth @@ -106,6 +108,11 @@ serde_json = "1" # #3604: the F2 guard receipt is keyed on the model file's sha256. sha2 = { workspace = true } +# #3568: schema-constrained decoding. llguidance (MIT) masks logits against a JSON schema or a Lark +# grammar; it sits behind `constrain::TokenConstraint`, so the engine stays swappable. Optional, enabled +# by `structured-output`, which apr-cli's `inference` feature turns on (cop ruling, 2026-09-21). +llguidance = { version = "1.8.0", optional = true, default-features = false, features = ["lark", "rayon", "ahash"] } + # Lock-free concurrent data structures arc-swap = { version = "1.7", optional = true } @@ -234,6 +241,7 @@ visualization = ["dep:trueno-viz"] # Terminal/PNG benchmark visualization registry = ["dep:pacha"] # Model registry with Pacha (pull/push/cache) tui = ["dep:presentar-terminal", "dep:crossterm", "dep:ureq", "dep:clap"] # PARITY-107: Real-time monitoring TUI (presentar-terminal) bench-http = ["dep:reqwest"] # Real HTTP benchmarking of external model servers +structured-output = ["dep:llguidance"] # #3568: schema-constrained decoding (`constrain` module) # Testing features load-test-enabled = ["server"] # Enable load tests (requires running server) @@ -340,6 +348,11 @@ required-features = ["cuda"] name = "imp800_gpu_parity" required-features = ["cuda"] +# #3568: per-token cost of schema-constrained decoding on a real vocabulary +[[example]] +name = "constrain_mask_overhead" +required-features = ["structured-output"] + [[example]] name = "imp900_optimized_gpu" required-features = ["cuda"] diff --git a/crates/aprender-serve/examples/constrain_mask_overhead.rs b/crates/aprender-serve/examples/constrain_mask_overhead.rs new file mode 100644 index 0000000000..f8396de733 --- /dev/null +++ b/crates/aprender-serve/examples/constrain_mask_overhead.rs @@ -0,0 +1,93 @@ +//! #3568: what schema-constrained decoding costs per token, on a REAL vocabulary. +//! +//! cargo run --release -p aprender-serve --features structured-output \ +//! --example constrain_mask_overhead -- [schema | @schema.json] +//! +//! It indexes the model's vocabulary once and compiles the schema (default: RAH's quorum-lane +//! schema, #3716). Then it feeds a real verdict document, tokenized by the model's own +//! tokenizer, through `mask` + `accept` one token at a time, the way a generation loop will. +//! It reports the index time, the compile time, and the per-token mask cost (p50 / p95 / max). +//! No weights run. This is the constraint's own overhead, which is the same on CPU and CUDA; a +//! CUDA loop also pays one host-logits download per token (#3568 PR 3 measures that). + +use std::time::Instant; + +use realizar::constrain::{load_schema, ConstraintEnv}; +use realizar::gguf::MappedGGUFModel; + +const DOC: &str = r#"{"verdict":"PASS","summary":"The diff classifies every receipt with a strict RFC 8259 parse in awk and removes the UNMEASURED exit-3 path.","findings":[{"file":"scripts/parity_receipt_denominator.sh","line":42,"claim":"classify() parses rather than substring-matches","grounding":"cited"},{"file":"crates/aprender-contracts-cli/tests/ont4c3_parity_receipts.rs","line":213,"claim":"the exit-3 acceptance is gone","grounding":"measured","fix":"none needed"}]}"#; + +fn rah_lane_schema() -> serde_json::Value { + serde_json::json!({ + "type": "object", "required": ["verdict", "summary", "findings"], + "properties": { + "verdict": {"type": "string", "enum": ["PASS", "FAIL", "do-not-implement-as-written"]}, + "summary": {"type": "string"}, "tree_witness": {"type": "string"}, + "findings": {"type": "array", "items": {"type": "object", "required": ["file", "claim", "grounding"], + "properties": {"file": {"type": "string"}, "line": {"type": "integer"}, "claim": {"type": "string"}, + "grounding": {"type": "string", "enum": ["cited", "measured", "asserted"]}, "fix": {"type": "string"}}}} + } + }) +} + +fn main() { + let args: Vec = std::env::args().collect(); + let path = args + .get(1) + .expect("usage: constrain_mask_overhead [schema | @schema.json]"); + let mapped = MappedGGUFModel::from_path(path).expect("map the GGUF"); + let vocab = mapped + .model + .constraint_vocab() + .expect("the GGUF has a vocabulary and an eos id"); + + let t = Instant::now(); + let env = ConstraintEnv::new(&vocab).expect("index the vocabulary"); + let index_ms = t.elapsed().as_secs_f64() * 1e3; + + let schema = args.get(2).map_or_else(rah_lane_schema, |a| { + load_schema(a).expect("the schema argument") + }); + let t = Instant::now(); + let mut constraint = env.json_schema(&schema).expect("compile the schema"); + let compile_ms = t.elapsed().as_secs_f64() * 1e3; + + let tokens = mapped + .model + .encode(DOC) + .expect("the model's tokenizer encodes the document"); + assert_eq!( + mapped.model.decode(&tokens), + DOC, + "the tokenization round-trips" + ); + + let mut logits = vec![0.0f32; vocab.token_bytes.len()]; + let mut mask_us = Vec::with_capacity(tokens.len()); + for &token in &tokens { + logits.iter_mut().for_each(|l| *l = 0.0); + let t = Instant::now(); + constraint + .mask(&mut logits) + .expect("a mask at every position of a valid document"); + mask_us.push(t.elapsed().as_secs_f64() * 1e6); + assert!( + logits[token as usize].is_finite(), + "the document's own token must be allowed" + ); + constraint + .accept(token) + .expect("the document's own token is accepted"); + } + let complete = constraint.is_complete(); + mask_us.sort_by(f64::total_cmp); + let pct = |p: f64| mask_us[((mask_us.len() - 1) as f64 * p).round() as usize]; + println!( + "vocab={} tokens={} index_ms={index_ms:.1} compile_ms={compile_ms:.2} mask_us p50={:.1} p95={:.1} max={:.1} complete={complete}", + vocab.token_bytes.len(), + tokens.len(), + pct(0.50), + pct(0.95), + pct(1.0), + ); +} diff --git a/crates/aprender-serve/src/api/cuda_chat_backend.rs b/crates/aprender-serve/src/api/cuda_chat_backend.rs index 1b0b556d9a..b6139865c9 100644 --- a/crates/aprender-serve/src/api/cuda_chat_backend.rs +++ b/crates/aprender-serve/src/api/cuda_chat_backend.rs @@ -319,7 +319,16 @@ fn try_quantized_backend( state.should_trace(trace_level), cancel, ); - let max_tokens = q_config.max_tokens; + // #3718: the dense CPU loop spends `effective_max_tokens` (the request clamped + // to the context room), so the finish is judged against THAT budget. Judged + // against the request's number, every context-clamped cut read as "stop". A + // prompt that cannot fit at all is the request's fault: refused here with 400, + // not a 500 after the fact or a streamed error that still ends "stop" at 200. + let max_tokens = match quantized_model.effective_max_tokens(prompt_tokens, q_config.max_tokens) + { + Ok(budget) => budget, + Err(e) => return Some(fail_response(state, crate::api::generation_error_status(&e), e)), + }; if request.stream { let (tx, rx) = tokio::sync::mpsc::channel::>(16); @@ -358,7 +367,7 @@ fn try_quantized_backend( // Non-streaming quantized let generated = match quantized_model.generate_with_cache(&prompt_ids, &q_config) { Ok(g) => g, - Err(e) => return Some(fail_response(state, StatusCode::INTERNAL_SERVER_ERROR, e)), + Err(e) => return Some(fail_response(state, crate::api::generation_error_status(&e), e)), }; let token_ids: Vec = generated.iter().skip(prompt_tokens).copied().collect(); diff --git a/crates/aprender-serve/src/api/tests/mod.rs b/crates/aprender-serve/src/api/tests/mod.rs index ae725c1292..262dc6e7b9 100644 --- a/crates/aprender-serve/src/api/tests/mod.rs +++ b/crates/aprender-serve/src/api/tests/mod.rs @@ -66,3 +66,5 @@ mod chat_stream_route_2375; // aprender#2375(4): POST /v1/chat/completions/strea mod apr_model_routes_2609; // aprender#2609: routed endpoints on an AprTransformer server; one condition, one status mod effective_config_route_pp2; // PP-LLAMA-001 §12 row 6 / PP-2 / PP-13 / PP-14 / PP-24 / PP-30: GET /v1/effective-config reports residency, not cfg! mod stream_mode_pp27; // PP-LLAMA-001 PP-27 / §3: an SSE stream declares live vs replayed, and the terminal chunk carries usage + measured timings +#[cfg(feature = "gpu")] // create_test_quantized_model is gpu-gated +mod usage_finish_3718; // aprender#3718: a context-clamped cut is "length", an unfittable prompt is 400 diff --git a/crates/aprender-serve/src/api/tests/usage_finish_3718.rs b/crates/aprender-serve/src/api/tests/usage_finish_3718.rs new file mode 100644 index 0000000000..b2657a65fb --- /dev/null +++ b/crates/aprender-serve/src/api/tests/usage_finish_3718.rs @@ -0,0 +1,160 @@ +//! aprender#3718: `/v1/chat/completions` on a GGUF server must never report a cut as +//! a finished reply, and a prompt that cannot fit must be refused as the client's +//! error. +//! +//! The dense CPU backend clamps `max_tokens` to the room left in the context +//! (`effective_max_tokens`, aprender#2376), then judged `finish_reason` against the +//! UNCLAMPED request. So a reply cut at the context edge said `"stop"`, and a +//! consumer could not tell it from a complete answer. A prompt longer than the +//! context was a 500 (non-streaming) or a 200 stream carrying an error event that +//! still ended `"stop"`. +//! +//! The fixture's greedy pick is token 0, which is also the EOS fallback when a +//! model declares none (`chat_gen_params`). With `ignore_eos` a run ends on its +//! budget, which is what the clamp tests vary; without it the run stops at once, +//! which is the converse. + +use axum::body::Body; +use axum::http::{Request, StatusCode}; +use tower::util::ServiceExt; + +use crate::api::{create_router, AppState}; + +/// A quantized-only server with a `context_length`-token window. +fn quantized_state(context_length: usize) -> AppState { + use crate::api::test_helpers::create_test_quantized_model; + use crate::gguf::{ArchConstraints, GGUFConfig}; + + let config = GGUFConfig { + architecture: "llama".to_string(), + constraints: ArchConstraints::from_architecture("llama"), + hidden_dim: 64, + intermediate_dim: 128, + num_layers: 2, + num_heads: 4, + num_kv_heads: 4, + vocab_size: 256, + context_length, + rope_theta: 10000.0, + eps: 1e-5, + rope_type: 0, + explicit_head_dim: None, + query_pre_attn_scalar: None, + bos_token_id: None, + eos_token_id: None, + }; + AppState::with_quantized_model(create_test_quantized_model(&config)) + .expect("build quantized AppState") +} + +async fn chat( + state: AppState, + max_tokens: usize, + stream: bool, + ignore_eos: bool, +) -> (StatusCode, String) { + let body = serde_json::json!({ + "model": "default", + "messages": [{"role": "user", "content": "Hi"}], + "max_tokens": max_tokens, + "temperature": 0.0, + "stream": stream, + "ignore_eos": ignore_eos, + }); + let response = create_router(state) + .oneshot( + Request::builder() + .method("POST") + .uri("/v1/chat/completions") + .header("content-type", "application/json") + .body(Body::from(body.to_string())) + .expect("request"), + ) + .await + .expect("response"); + let status = response.status(); + let bytes = axum::body::to_bytes(response.into_body(), usize::MAX) + .await + .expect("body"); + (status, String::from_utf8_lossy(&bytes).into_owned()) +} + +/// How many tokens the templated `"Hi"` prompt is, measured through the server +/// itself with a budget small enough that nothing clamps. +async fn templated_prompt_tokens() -> usize { + let (status, body) = chat(quantized_state(512), 1, false, true).await; + assert_eq!(status, StatusCode::OK, "{body}"); + let v: serde_json::Value = serde_json::from_str(&body).expect("json"); + v["usage"]["prompt_tokens"].as_u64().expect("prompt_tokens") as usize +} + +#[tokio::test] +async fn a_context_clamped_cut_is_length_not_stop() { + let prompt = templated_prompt_tokens().await; + let context = prompt + 6; + let (status, body) = chat(quantized_state(context), 1000, false, true).await; + assert_eq!(status, StatusCode::OK, "{body}"); + let v: serde_json::Value = serde_json::from_str(&body).expect("json"); + assert_eq!(v["usage"]["prompt_tokens"], prompt, "{body}"); + assert_eq!( + v["usage"]["completion_tokens"], 6, + "control: the run must stop at the context edge (6 tokens of room), or this \ + is not measuring a clamp: {body}" + ); + assert_eq!(v["usage"]["total_tokens"], context, "{body}"); + assert_eq!(v["choices"][0]["finish_reason"], "length", "{body}"); +} + +#[tokio::test] +async fn a_context_clamped_stream_ends_length_not_stop() { + let prompt = templated_prompt_tokens().await; + let (status, body) = chat(quantized_state(prompt + 6), 1000, true, true).await; + assert_eq!(status, StatusCode::OK, "{body}"); + assert!( + body.contains(r#""finish_reason":"length""#), + "the terminal chunk must say the stream was cut: {body}" + ); + assert!(!body.contains(r#""finish_reason":"stop""#), "{body}"); +} + +#[tokio::test] +async fn a_prompt_longer_than_the_context_is_a_400() { + let prompt = templated_prompt_tokens().await; + for stream in [false, true] { + let (status, body) = chat(quantized_state(prompt - 1), 8, stream, true).await; + assert_eq!( + status, + StatusCode::BAD_REQUEST, + "stream={stream}: the request cannot fit, which is the client's error: {body}" + ); + assert!(!body.contains("finish_reason"), "stream={stream}: nothing ran: {body}"); + } +} + +/// The converse, so the clamp tests cannot pass by reporting "length" for +/// everything: with EOS honoured the fixture stops on its first pick, under +/// budget and under the clamp, and that is a finished reply. +#[tokio::test] +async fn a_stop_under_budget_is_stop() { + let prompt = templated_prompt_tokens().await; + for stream in [false, true] { + let (status, body) = chat(quantized_state(prompt + 6), 1000, stream, false).await; + assert_eq!(status, StatusCode::OK, "stream={stream}: {body}"); + assert!( + body.contains(r#""completion_tokens":0"#), + "control: the fixture must stop on its first pick: {body}" + ); + assert!(body.contains(r#""finish_reason":"stop""#), "stream={stream}: {body}"); + assert!(!body.contains(r#""finish_reason":"length""#), "stream={stream}: {body}"); + } +} + +/// With room to spare, the budget that ends the run is the REQUEST's. +#[tokio::test] +async fn an_unclamped_run_is_cut_at_the_request_budget() { + let (status, body) = chat(quantized_state(512), 4, false, true).await; + assert_eq!(status, StatusCode::OK, "{body}"); + let v: serde_json::Value = serde_json::from_str(&body).expect("json"); + assert_eq!(v["usage"]["completion_tokens"], 4, "{body}"); + assert_eq!(v["choices"][0]["finish_reason"], "length", "{body}"); +} diff --git a/crates/aprender-serve/src/api/types.rs b/crates/aprender-serve/src/api/types.rs index 1475a33a76..a13f613a7d 100644 --- a/crates/aprender-serve/src/api/types.rs +++ b/crates/aprender-serve/src/api/types.rs @@ -196,55 +196,10 @@ pub struct ModelsResponse { /// Why a completion stopped, in OpenAI's vocabulary. /// -/// Dogfood 0.63.0 (#2375 finding 6): the STREAMING chat path emitted the string -/// literal `"stop"` in its terminal chunk no matter what happened, while the -/// non-streaming path on the identical request correctly reported `"length"` -/// when the generation hit `max_tokens`. A client that streams therefore cannot -/// tell a truncated answer from a finished one, and every "continue from where -/// you stopped" flow silently breaks. -/// -/// The type exists so that literal cannot come back: the terminal-chunk -/// constructor takes a `FinishReason`, which is only obtainable from -/// [`FinishReason::from_generation`] (or an explicit, named variant). There is -/// no `&str` parameter left to hardcode. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum FinishReason { - /// The model emitted a stop token or a stop string matched. - Stop, - /// Generation was cut off at the `max_tokens` budget. - Length, -} - -impl FinishReason { - /// Decide the reason from what the generation actually did. - /// - /// Mirrors `finalize_chat_text` / `completion_finish_reason`: a matched stop - /// string wins over the budget, and a model that terminated early is - /// `Stop`. Only "ran to the budget with no stop match" is `Length`. - #[must_use] - pub fn from_generation(stopped: bool, completion_tokens: usize, max_tokens: usize) -> Self { - if !stopped && completion_tokens >= max_tokens { - Self::Length - } else { - Self::Stop - } - } - - /// The wire string OpenAI clients match on. - #[must_use] - pub fn as_str(self) -> &'static str { - match self { - Self::Stop => "stop", - Self::Length => "length", - } - } -} - -impl std::fmt::Display for FinishReason { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.write_str(self.as_str()) - } -} +/// Declared once in [`crate::infer::run_report`] so `apr run --json` and this +/// server spell the same outcome the same way (#3718); `infer` is not behind the +/// `server` feature, so the CLI can reach it without the HTTP stack. +pub use crate::infer::run_report::FinishReason; /// The OpenAI `n` field: how many completions the client asked for. /// diff --git a/crates/aprender-serve/src/constrain/llg.rs b/crates/aprender-serve/src/constrain/llg.rs new file mode 100644 index 0000000000..e97655cedb --- /dev/null +++ b/crates/aprender-serve/src/constrain/llg.rs @@ -0,0 +1,189 @@ +//! The `llguidance` engine behind [`TokenConstraint`] (#3568). +//! +//! `LlgEnv` indexes a vocabulary once: a byte trie over every token, with each +//! special token prefixed by `TokTrie::SPECIAL_TOKEN_MARKER` so it can never +//! match constrained text. `LlgConstraint` is one request's matcher. +//! +//! The error mapping is structural, not a reading of message text. +//! `load_schema` reports input that is not a schema at all as `SchemaInvalid`. +//! Anything the engine then fails to compile is `SchemaUnsupported`, with the +//! engine's own message verbatim, because llguidance returns untyped errors, and +//! sorting them by substring would be a text match standing in for a type. + +use std::sync::Arc; + +use llguidance::api::TopLevelGrammar; +use llguidance::toktrie::{ApproximateTokEnv, TokEnv, TokRxInfo, TokTrie}; +use llguidance::{Matcher, ParserFactory}; + +use super::{ConstraintError, ConstraintVocab, TokenConstraint}; + +/// A vocabulary indexed for llguidance, built once per model. +pub struct LlgEnv { + factory: Arc, +} + +impl LlgEnv { + /// Index `vocab`. Special tokens get the marker byte, so no grammar can emit them as text. + pub fn new(vocab: &ConstraintVocab) -> Result { + vocab.validate()?; + let words: Vec> = vocab + .token_bytes + .iter() + .zip(&vocab.special) + .map(|(bytes, &special)| { + if special { + let mut marked = Vec::with_capacity(bytes.len() + 1); + marked.push(TokTrie::SPECIAL_TOKEN_MARKER); + marked.extend_from_slice(bytes); + marked + } else { + bytes.clone() + } + }) + .collect(); + let vocab_size = u32::try_from(words.len()).map_err(|_| { + ConstraintError::Vocab(format!("{} tokens do not fit a u32 id", words.len())) + })?; + let info = TokRxInfo::new(vocab_size, vocab.eos); + let trie = TokTrie::from(&info, &words); + let env: TokEnv = Arc::new(ApproximateTokEnv::new(trie)); + let mut factory = ParserFactory::new_simple(&env).map_err(|e| { + ConstraintError::Vocab(format!("llguidance refused the vocabulary: {e}")) + })?; + factory.quiet(); + Ok(Self { + factory: Arc::new(factory), + }) + } + + /// One request's constraint for a JSON schema. + /// + /// Compact JSON (`,` and `:`, no free whitespace) unless the caller chose its own + /// `x-guidance`. llguidance's default is flexible whitespace, and that is the known runaway: + /// a constrained model emits whitespace until `max_tokens`. This module's own case table + /// measured it, with `{` followed by 63 bytes of whitespace. + pub fn json_schema( + &self, + schema: &serde_json::Value, + ) -> Result { + // A boolean schema carries no x-guidance, so it would compile with flexible whitespace: + // `true` (any JSON value) becomes `{}`, which means the same and compiles compact, and + // `false` admits no document at all, which could only ever be a dead end. + let mut schema = match schema { + serde_json::Value::Bool(true) => serde_json::json!({}), + serde_json::Value::Bool(false) => { + return Err(ConstraintError::SchemaInvalid( + "the schema `false` admits no document, so nothing could ever be generated" + .to_string(), + )) + }, + other => other.clone(), + }; + if schema.is_object() && schema.get("x-guidance").is_none() { + llguidance::JsonCompileOptions { + whitespace_flexible: false, + ..Default::default() + } + .apply_to(&mut schema); + } + self.compile(TopLevelGrammar::from_json_schema(schema), "JSON schema") + } + + /// One request's constraint for a Lark grammar. + pub fn lark(&self, grammar: &str) -> Result { + self.compile( + TopLevelGrammar::from_lark(grammar.to_string()), + "Lark grammar", + ) + } + + fn compile( + &self, + grammar: TopLevelGrammar, + what: &str, + ) -> Result { + let parser = self.factory.create_parser(grammar).map_err(|e| { + ConstraintError::SchemaUnsupported(format!( + "the {what} does not compile for constrained decoding: {e}" + )) + })?; + let mut matcher = Matcher::new(Ok(parser)); + if let Some(e) = matcher.get_error() { + return Err(ConstraintError::SchemaUnsupported(format!( + "the {what} does not start: {e}" + ))); + } + // An approximation is a weaker constraint than the caller wrote: refused, never used quietly. + let warnings = matcher.grammar_warnings(); + if !warnings.is_empty() { + return Err(ConstraintError::SchemaUnsupported(format!( + "the {what} compiles only approximately: {}", + warnings.join("; ") + ))); + } + Ok(LlgConstraint { + matcher, + position: 0, + }) + } +} + +/// One request's constraint: an llguidance matcher and how far it has advanced. +pub struct LlgConstraint { + matcher: Matcher, + position: usize, +} + +impl TokenConstraint for LlgConstraint { + fn mask(&mut self, logits: &mut [f32]) -> Result<(), ConstraintError> { + // llguidance reports "nothing can extend this" as an error (`NoExtensionBias`), not as + // an empty mask, so both are a dead end: the loop stops, never continues unconstrained. + let allowed = self + .matcher + .compute_mask_or_eos() + .map_err(|e| ConstraintError::DeadEnd { + position: self.position, + reason: e + .to_string() + .lines() + .next() + .unwrap_or("no reason given") + .to_string(), + })?; + let mut any = false; + for (id, logit) in logits.iter_mut().enumerate() { + // A logit row past the tokenizer's vocabulary (padded embeddings) is never allowed. + let ok = id < allowed.len() && u32::try_from(id).is_ok_and(|t| allowed.is_allowed(t)); + if ok { + any = true; + } else { + *logit = f32::NEG_INFINITY; + } + } + if any { + Ok(()) + } else { + Err(ConstraintError::DeadEnd { + position: self.position, + reason: "the mask allows no token".to_string(), + }) + } + } + + fn accept(&mut self, token: u32) -> Result<(), ConstraintError> { + self.matcher + .consume_token(token) + .map_err(|e| ConstraintError::Rejected { + token, + position: self.position, + reason: e.to_string(), + })?; + self.position += 1; + Ok(()) + } + + fn is_complete(&mut self) -> bool { + self.matcher.is_stopped() || self.matcher.is_accepting().unwrap_or(false) + } +} diff --git a/crates/aprender-serve/src/constrain/mod.rs b/crates/aprender-serve/src/constrain/mod.rs new file mode 100644 index 0000000000..a2cdf98376 --- /dev/null +++ b/crates/aprender-serve/src/constrain/mod.rs @@ -0,0 +1,229 @@ +//! Schema-constrained decoding (#3568): the one hook a generation loop calls. +//! +//! A constrained step is `mask` (every token the constraint forbids goes to +//! `-inf`), then the loop's OWN sampler, unchanged, then `accept`. The mask +//! only removes candidates, so greedy / temperature-0 decoding stays +//! deterministic. The answer is produced right rather than repaired after the +//! fact. +//! +//! The engine sits behind [`TokenConstraint`] so it can be swapped. Today it +//! is `llguidance` (MIT), behind the `structured-output` feature, which apr-cli's +//! `inference` feature turns on (cop ruling on #3568, 2026-09-21). Without that +//! feature, building a constraint refuses by name ([`ConstraintError::NotCompiled`]), +//! and the flag is never silently ignored. +//! +//! What a schema CANNOT say stays the consumer's to check. A grammar enforces +//! what JSON Schema expresses: types, required keys, `additionalProperties`, +//! bounds, `enum`/`const`, `pattern`. Cross-field facts are outside it, for +//! example `correct_index < options.length`, feedback that lines up with its +//! options, or a contiguous `n` across records. A passing constraint says +//! nothing about them (rmedia, #3716). +//! +//! This module has no generation loop in it. It is the trait, the refusals, +//! the schema source, and the vocabulary the engine needs (each token's RAW +//! BYTES, from the same per-token decoding `GGUFModel::decode` uses, so the mask +//! and the decoded text can never disagree about what a token is). + +use std::fmt; + +#[cfg(feature = "structured-output")] +mod llg; +#[cfg(feature = "structured-output")] +pub use llg::{LlgConstraint, LlgEnv}; + +/// What every constrained generation loop calls, once per generated token. +pub trait TokenConstraint: Send { + /// Set every logit this constraint forbids at the current position to `-inf`. + /// + /// `Err(DeadEnd)` when nothing is allowed. The loop must then stop: it never + /// continues unconstrained. + fn mask(&mut self, logits: &mut [f32]) -> Result<(), ConstraintError>; + + /// Advance past the token the sampler chose. A token the mask forbade is `Rejected`. + fn accept(&mut self, token: u32) -> Result<(), ConstraintError>; + + /// The output so far is a complete document: end-of-sequence is allowed here. + fn is_complete(&mut self) -> bool; +} + +/// Every way a constrained generation refuses, by name (#3568: "refused in one line"). +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum ConstraintError { + /// The schema argument is not a readable JSON Schema: bad JSON, an unreadable + /// `@path`, or a document no engine can compile. + SchemaInvalid(String), + /// A well-formed schema that uses something this engine cannot enforce. + SchemaUnsupported(String), + /// No token is allowed at this output position. + DeadEnd { + /// Tokens accepted before the dead end. + position: usize, + /// The engine's reason. + reason: String, + }, + /// The sampler chose a token the constraint does not allow. + Rejected { + /// The token that was refused. + token: u32, + /// Tokens accepted before it. + position: usize, + /// The engine's reason. + reason: String, + }, + /// The vocabulary could not be turned into the engine's token table. + Vocab(String), + /// This build has no `structured-output` feature. + NotCompiled, +} + +impl fmt::Display for ConstraintError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::SchemaInvalid(why) => write!(f, "SchemaInvalid: {why}"), + Self::SchemaUnsupported(why) => write!(f, "SchemaUnsupported: {why}"), + Self::DeadEnd { position, reason } => write!( + f, + "ConstraintDeadEnd: no token is allowed at output position {position}: {reason}" + ), + Self::Rejected { + token, + position, + reason, + } => write!( + f, + "ConstraintRejected: token {token} is not allowed at output position {position}: {reason}" + ), + Self::Vocab(why) => write!(f, "ConstraintVocab: {why}"), + Self::NotCompiled => write!( + f, + "StructuredOutputNotCompiled: this build has no `structured-output` feature, so a \ + schema cannot be enforced (refused rather than ignored)" + ), + } + } +} + +impl std::error::Error for ConstraintError {} + +/// Read a schema argument: the schema inline, or `@path` to a file holding it. +/// +/// Both forms are accepted on purpose (#3568 design input). A caller once passed +/// a path where a tool expected the schema, the process died on its arguments, +/// and from outside that looked like a lane with nothing to say. Validated here, +/// before the first token. +pub fn load_schema(arg: &str) -> Result { + let (text, from) = match arg.strip_prefix('@') { + Some(path) => ( + std::fs::read_to_string(path).map_err(|e| { + ConstraintError::SchemaInvalid(format!("cannot read the schema file {path}: {e}")) + })?, + format!("the schema file {path}"), + ), + None => (arg.to_string(), "the inline schema".to_string()), + }; + let value: serde_json::Value = serde_json::from_str(&text) + .map_err(|e| ConstraintError::SchemaInvalid(format!("{from} is not JSON: {e}")))?; + if !(value.is_object() || value.is_boolean()) { + return Err(ConstraintError::SchemaInvalid(format!( + "{from} is JSON but not a schema: a JSON Schema is an object or a boolean" + ))); + } + Ok(value) +} + +/// A vocabulary as a constraint sees it: each token's raw bytes, which tokens +/// are special (control tokens, which never appear in constrained text), and +/// the end-of-sequence id. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ConstraintVocab { + /// The bytes each token id decodes to, indexed by token id. + pub token_bytes: Vec>, + /// Whether each token id is a special (control) token. + pub special: Vec, + /// The end-of-sequence token id. + pub eos: u32, +} + +impl ConstraintVocab { + /// Check the three parts agree, before any engine sees them. + pub fn validate(&self) -> Result<(), ConstraintError> { + if self.token_bytes.is_empty() { + return Err(ConstraintError::Vocab( + "the vocabulary is empty".to_string(), + )); + } + if self.special.len() != self.token_bytes.len() { + return Err(ConstraintError::Vocab(format!( + "{} token byte strings but {} special flags", + self.token_bytes.len(), + self.special.len() + ))); + } + if self.eos as usize >= self.token_bytes.len() { + return Err(ConstraintError::Vocab(format!( + "eos {} is outside a vocabulary of {}", + self.eos, + self.token_bytes.len() + ))); + } + Ok(()) + } +} + +/// The engine built once per model, from which each request's constraint is made. +/// +/// Building it indexes the whole vocabulary (248,320 tokens for Qwen3.5), so a +/// caller builds it once and keeps it. +pub struct ConstraintEnv { + #[cfg(feature = "structured-output")] + inner: LlgEnv, +} + +impl ConstraintEnv { + /// Index `vocab` for constrained decoding. + pub fn new(vocab: &ConstraintVocab) -> Result { + vocab.validate()?; + #[cfg(feature = "structured-output")] + { + Ok(Self { + inner: LlgEnv::new(vocab)?, + }) + } + #[cfg(not(feature = "structured-output"))] + { + Err(ConstraintError::NotCompiled) + } + } + + /// A constraint that admits exactly the documents `schema` accepts. + pub fn json_schema( + &self, + schema: &serde_json::Value, + ) -> Result, ConstraintError> { + #[cfg(feature = "structured-output")] + { + Ok(Box::new(self.inner.json_schema(schema)?)) + } + #[cfg(not(feature = "structured-output"))] + { + let _ = schema; + Err(ConstraintError::NotCompiled) + } + } + + /// A constraint that admits exactly the strings a Lark grammar accepts. + pub fn lark(&self, grammar: &str) -> Result, ConstraintError> { + #[cfg(feature = "structured-output")] + { + Ok(Box::new(self.inner.lark(grammar)?)) + } + #[cfg(not(feature = "structured-output"))] + { + let _ = grammar; + Err(ConstraintError::NotCompiled) + } + } +} + +#[cfg(test)] +mod tests; diff --git a/crates/aprender-serve/src/constrain/tests.rs b/crates/aprender-serve/src/constrain/tests.rs new file mode 100644 index 0000000000..df80b3f09c --- /dev/null +++ b/crates/aprender-serve/src/constrain/tests.rs @@ -0,0 +1,721 @@ +//! #3568 case rows. The schema source and the refusals run in every build. The +//! engine rows need `structured-output`, which a workspace build turns on (apr-cli's +//! `inference`), and which `-p aprender-serve --features structured-output` selects. + +use super::*; + +#[test] +fn load_schema_takes_the_schema_inline() { + let v = load_schema(r#"{"type":"object"}"#).expect("an inline schema"); + assert_eq!(v["type"], "object"); +} + +#[test] +fn load_schema_takes_an_at_path() { + let dir = std::env::temp_dir().join(format!("apr-3568-{}", std::process::id())); + std::fs::create_dir_all(&dir).expect("temp dir"); + let file = dir.join("schema.json"); + std::fs::write(&file, r#"{"type":"string"}"#).expect("write the schema"); + let v = load_schema(&format!("@{}", file.display())).expect("an @path schema"); + assert_eq!(v["type"], "string"); + std::fs::remove_file(&file).expect("remove the schema"); +} + +#[test] +fn load_schema_refuses_an_unreadable_path_by_name() { + let e = load_schema("@/no/such/dir/schema.json").expect_err("an unreadable path must refuse"); + assert!( + matches!(&e, ConstraintError::SchemaInvalid(why) if why.contains("/no/such/dir/schema.json")), + "{e}" + ); +} + +#[test] +fn load_schema_refuses_text_that_is_not_json() { + // the mistake the design input names: a path passed where the schema was expected + let e = load_schema("schema.json").expect_err("a bare path is not a schema"); + assert!( + matches!(&e, ConstraintError::SchemaInvalid(why) if why.contains("not JSON")), + "{e}" + ); + assert!(e.to_string().starts_with("SchemaInvalid: "), "{e}"); +} + +#[test] +fn load_schema_refuses_json_that_is_not_a_schema() { + for not_a_schema in ["[1, 2]", "42", r#""object""#, "null"] { + let e = load_schema(not_a_schema).expect_err(not_a_schema); + assert!( + matches!(&e, ConstraintError::SchemaInvalid(why) if why.contains("not a schema")), + "{e}" + ); + } +} + +#[test] +fn a_vocabulary_whose_parts_disagree_is_refused_before_any_engine_sees_it() { + let good = ConstraintVocab { + token_bytes: vec![b"a".to_vec(), b"".to_vec()], + special: vec![false, true], + eos: 1, + }; + assert_eq!(good.validate(), Ok(())); + let short_flags = ConstraintVocab { + special: vec![false], + ..good.clone() + }; + assert!(matches!( + short_flags.validate(), + Err(ConstraintError::Vocab(_)) + )); + let eos_outside = ConstraintVocab { + eos: 2, + ..good.clone() + }; + assert!(matches!( + eos_outside.validate(), + Err(ConstraintError::Vocab(_)) + )); + let empty = ConstraintVocab { + token_bytes: vec![], + special: vec![], + eos: 0, + }; + assert!(matches!(empty.validate(), Err(ConstraintError::Vocab(_)))); +} + +#[cfg(not(feature = "structured-output"))] +#[test] +fn without_the_feature_a_constraint_is_refused_never_ignored() { + let vocab = ConstraintVocab { + token_bytes: vec![b"a".to_vec(), b"".to_vec()], + special: vec![false, true], + eos: 1, + }; + let e = ConstraintEnv::new(&vocab) + .err() + .expect("no engine compiled in: refuse"); + assert_eq!(e, ConstraintError::NotCompiled); + assert!( + e.to_string().starts_with("StructuredOutputNotCompiled: "), + "{e}" + ); +} + +#[cfg(feature = "structured-output")] +mod engine { + use super::*; + + const EOS: u32 = 0; + const IM_END: u32 = 1; + + /// A byte-level vocabulary: EOS and one more special token, then every single byte (so + /// every string is reachable), then multi-byte tokens a real BPE vocabulary would carry. + fn vocab() -> ConstraintVocab { + let mut token_bytes = vec![b"<|endoftext|>".to_vec(), b"<|im_end|>".to_vec()]; + let mut special = vec![true, true]; + for b in 0..=255u8 { + token_bytes.push(vec![b]); + special.push(false); + } + for multi in [ + "{\"", + "\":", + "\",", + "\"}", + "\"verdict\"", + "PASS", + "FAIL", + "xx", + "\"findings\"", + "summary", + " ", + ] { + token_bytes.push(multi.as_bytes().to_vec()); + special.push(false); + } + ConstraintVocab { + token_bytes, + special, + eos: EOS, + } + } + + fn id_of(v: &ConstraintVocab, bytes: &[u8]) -> u32 { + let pos = v + .token_bytes + .iter() + .position(|t| t == bytes) + .expect("token in the vocabulary"); + u32::try_from(pos).expect("id fits u32") + } + + /// Greedy pick: the highest logit's id (a masked token is -inf and never wins). + fn greedy(logits: &[f32]) -> u32 { + let (next, _) = + logits + .iter() + .enumerate() + .fold((0usize, f32::NEG_INFINITY), |best, (i, &l)| { + if l > best.1 { + (i, l) + } else { + best + } + }); + u32::try_from(next).expect("id fits u32") + } + + /// One decoding step, the way a constrained loop runs it: mask, the loop's own greedy + /// sampler, then accept. `None` when the model chose end-of-sequence. + fn step( + constraint: &mut Option<&mut dyn TokenConstraint>, + mut logits: Vec, + ) -> Result, ConstraintError> { + if let Some(c) = constraint.as_deref_mut() { + c.mask(&mut logits)?; + } + let next = greedy(&logits); + if next == EOS { + return Ok(None); + } + if let Some(c) = constraint.as_deref_mut() { + c.accept(next)?; + } + Ok(Some(next)) + } + + /// A fake model that WANTS to violate any schema: `x` scores highest, then `xx`, then + /// every token by a fixed arbitrary order. Greedy, like `--temperature 0`. + fn generate( + v: &ConstraintVocab, + mut constraint: Option<&mut dyn TokenConstraint>, + max: usize, + ) -> Result, ConstraintError> { + let x = id_of(v, b"x"); + let xx = id_of(v, b"xx"); + let score = |i: usize| { + let t = u32::try_from(i).expect("id fits u32"); + if t == x { + 100.0 + } else if t == xx { + 90.0 + } else if t == EOS { + 80.0 // a finished model wants to stop; masked until the document is complete + } else { + ((i * 7919) % 1000) as f32 / 100.0 + } + }; + let mut out = Vec::new(); + for _ in 0..max { + let logits = (0..v.token_bytes.len()).map(score).collect(); + let Some(next) = step(&mut constraint, logits)? else { + return Ok(out); + }; + out.extend_from_slice(&v.token_bytes[next as usize]); + } + Ok(out) + } + + /// How the intent model scores token `i` when `rest` is what it still means to write: + /// end-of-sequence once the intent is written; else the longest allowed prefix of `rest`; + /// else `fallback`'s fixed order, which closes structures. + fn intent_score(v: &ConstraintVocab, fallback: &[u32], rest: &[u8], i: usize) -> f32 { + let t = u32::try_from(i).expect("id fits u32"); + let bytes = &v.token_bytes[i]; + if t == EOS { + return if rest.is_empty() { 2000.0 } else { -1.0 }; + } + if !v.special[i] && !bytes.is_empty() && rest.starts_with(bytes) { + return 1000.0 + bytes.len() as f32; + } + fallback + .iter() + .position(|&f| f == t) + .map_or(0.0, |rank| 500.0 - rank as f32) + } + + /// A fake model with an INTENT: it prefers the allowed token that spells the longest prefix + /// of what it still means to write; when the constraint forbids its next intended byte it + /// falls back to a fixed order that closes structures. It stops once its intent is written. + fn generate_intent( + v: &ConstraintVocab, + mut constraint: Option<&mut dyn TokenConstraint>, + intended: &[u8], + max: usize, + ) -> Result, ConstraintError> { + let fallback: Vec = ["}", "]", ",", "\"findings\"", ":", "[", "\""] + .iter() + .map(|t| id_of(v, t.as_bytes())) + .collect(); + let mut out = Vec::new(); + let mut done = 0usize; // bytes of the intent already written + for _ in 0..max { + let rest = &intended[done..]; + let logits = (0..v.token_bytes.len()) + .map(|i| intent_score(v, &fallback, rest, i)) + .collect(); + let Some(next) = step(&mut constraint, logits)? else { + return Ok(out); + }; + let bytes = &v.token_bytes[next as usize]; + if rest.starts_with(bytes) { + done += bytes.len(); + } + out.extend_from_slice(bytes); + } + Ok(out) + } + + fn verdict_schema() -> serde_json::Value { + serde_json::json!({ + "type": "object", + "required": ["verdict"], + "additionalProperties": false, + "properties": {"verdict": {"type": "string", "enum": ["PASS", "FAIL", "do-not-implement-as-written"]}} + }) + } + + #[test] + fn a_model_that_wants_to_violate_the_schema_is_held_to_it() { + let v = vocab(); + let env = ConstraintEnv::new(&v).expect("index the vocabulary"); + + // the positive control: unconstrained, this model writes no JSON at all + let free = generate(&v, None, 16).expect("unconstrained run"); + assert!( + serde_json::from_slice::(&free).is_err(), + "{}", + String::from_utf8_lossy(&free) + ); + + let mut c = env + .json_schema(&verdict_schema()) + .expect("compile the schema"); + let out = generate(&v, Some(c.as_mut()), 64).expect("constrained run"); + let doc: serde_json::Value = serde_json::from_slice(&out) + .unwrap_or_else(|e| panic!("not JSON: {e}: {}", String::from_utf8_lossy(&out))); + let verdict = doc["verdict"] + .as_str() + .expect("verdict is required and a string"); + assert!( + ["PASS", "FAIL", "do-not-implement-as-written"].contains(&verdict), + "{doc}" + ); + } + + #[test] + fn the_rah_quorum_lane_schema_is_enforced_end_to_end() { + // paiml-implement agy/quorum-lane-schema.json, the first consumer (#3716) + let schema = serde_json::json!({ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "title": "paiml-implement quorum lane output", + "type": "object", + "required": ["verdict", "summary", "findings"], + "properties": { + "verdict": {"type": "string", "enum": ["PASS", "FAIL", "do-not-implement-as-written"]}, + "summary": {"type": "string"}, + "tree_witness": {"type": "string", "description": "copied verbatim"}, + "findings": {"type": "array", "items": { + "type": "object", + "required": ["file", "claim", "grounding"], + "properties": { + "file": {"type": "string"}, "line": {"type": "integer"}, "claim": {"type": "string"}, + "grounding": {"type": "string", "enum": ["cited", "measured", "asserted"]}, + "fix": {"type": "string"} + } + }} + } + }); + let v = vocab(); + let env = ConstraintEnv::new(&v).expect("index the vocabulary"); + let mut c = env.json_schema(&schema).expect("compile the lane schema"); + // the 2026-09-21 local-lane answer: right substance, `finding` for `findings`, a NULL vote + let intended = br#"{"verdict":"PASS","summary":"ok","finding":[]}"#; + let free = generate_intent(&v, None, intended, 128).expect("unconstrained run"); + assert_eq!( + free, + intended.to_vec(), + "unconstrained, the model writes the NULL vote" + ); + let out = generate_intent(&v, Some(c.as_mut()), intended, 128).expect("constrained run"); + let doc: serde_json::Value = serde_json::from_slice(&out) + .unwrap_or_else(|e| panic!("not JSON: {e}: {}", String::from_utf8_lossy(&out))); + for key in ["verdict", "summary", "findings"] { + assert!(doc.get(key).is_some(), "required {key} missing: {doc}"); + } + assert!(doc["findings"].is_array(), "{doc}"); + assert!( + doc.get("finding").is_none(), + "the 2026-09-21 NULL vote's key: {doc}" + ); + } + + #[test] + fn special_tokens_are_never_allowed_as_constrained_text() { + let v = vocab(); + let env = ConstraintEnv::new(&v).expect("index the vocabulary"); + let mut c = env + .json_schema(&serde_json::json!({"type": "string"})) + .expect("compile"); + // INSIDE an open string, where `<|im_end|>` would be legal TEXT if it were not special + c.accept(id_of(&v, b"\"")) + .expect("a string opens with a quote"); + let mut logits = vec![0.0f32; v.token_bytes.len()]; + c.mask(&mut logits).expect("string content can follow"); + assert!( + logits[id_of(&v, b"<") as usize].is_finite(), + "the bytes of `<|im_end|>` are legal string text" + ); + assert_eq!( + logits[IM_END as usize], + f32::NEG_INFINITY, + "a control token must never be string text" + ); + assert_eq!( + logits[EOS as usize], + f32::NEG_INFINITY, + "nor may end-of-sequence close an open string" + ); + } + + #[test] + fn a_complete_document_allows_end_of_sequence() { + let v = vocab(); + let env = ConstraintEnv::new(&v).expect("index the vocabulary"); + let mut c = env + .json_schema(&serde_json::json!({"type": "boolean"})) + .expect("compile"); + assert!(!c.is_complete(), "nothing generated yet"); + for b in b"true" { + c.accept(id_of(&v, &[*b])).expect("true is a boolean"); + } + assert!(c.is_complete(), "`true` is a complete boolean document"); + let mut logits = vec![0.0f32; v.token_bytes.len()]; + c.mask(&mut logits) + .expect("the mask after a complete document"); + assert!( + logits[EOS as usize].is_finite(), + "EOS must be allowed after a complete document" + ); + } + + #[test] + fn a_token_the_mask_forbade_is_rejected_by_name() { + let v = vocab(); + let env = ConstraintEnv::new(&v).expect("index the vocabulary"); + let mut c = env.json_schema(&verdict_schema()).expect("compile"); + let e = c + .accept(id_of(&v, b"x")) + .expect_err("an object cannot start with x"); + assert!( + matches!(e, ConstraintError::Rejected { position: 0, .. }), + "{e}" + ); + assert!(e.to_string().starts_with("ConstraintRejected: "), "{e}"); + } + + #[test] + fn a_vocabulary_that_cannot_spell_the_schema_is_a_dead_end_never_unconstrained() { + // only EOS and `a`: no digit exists, so an integer cannot even start + let v = ConstraintVocab { + token_bytes: vec![b"<|endoftext|>".to_vec(), b"a".to_vec()], + special: vec![true, false], + eos: EOS, + }; + let env = ConstraintEnv::new(&v).expect("index the vocabulary"); + let mut c = env + .json_schema(&serde_json::json!({"type": "integer"})) + .expect("compile"); + let mut logits = vec![1.0f32; 2]; + let e = c + .mask(&mut logits) + .expect_err("an integer cannot start from `a`"); + assert!( + matches!(e, ConstraintError::DeadEnd { position: 0, .. }), + "{e}" + ); + assert!(e.to_string().starts_with("ConstraintDeadEnd: "), "{e}"); + } + + #[test] + fn a_keyword_the_engine_cannot_enforce_is_refused_by_name_before_generation() { + let v = vocab(); + let env = ConstraintEnv::new(&v).expect("index the vocabulary"); + for (schema, keyword) in [ + ( + serde_json::json!({"type": "array", "uniqueItems": true}), + "uniqueItems", + ), + (serde_json::json!({"not": {"type": "string"}}), "not"), + ( + serde_json::json!({"type": "array", "contains": {"type": "integer"}}), + "contains", + ), + ] { + match env.json_schema(&schema) { + Ok(_) => panic!("{keyword}: compiled, but the engine cannot enforce it"), + Err(e) => assert!( + matches!(&e, ConstraintError::SchemaUnsupported(why) if why.contains(keyword)), + "{keyword}: {e}" + ), + } + } + } + + #[test] + fn json_is_compact_by_default_and_the_callers_x_guidance_wins() { + let v = vocab(); + let env = ConstraintEnv::new(&v).expect("index the vocabulary"); + let space = id_of(&v, b" "); + let brace = id_of(&v, b"{"); + let after_brace = |schema: serde_json::Value| { + let mut c = env.json_schema(&schema).expect("compile"); + c.accept(brace).expect("an object starts with {"); + let mut logits = vec![0.0f32; v.token_bytes.len()]; + c.mask(&mut logits).expect("the mask after {"); + logits[space as usize].is_finite() + }; + // default: no free whitespace, so no whitespace runaway + assert!( + !after_brace(serde_json::json!({"type": "object"})), + "compact JSON admits no space after {{" + ); + // a caller that chose flexible whitespace gets it + assert!(after_brace( + serde_json::json!({"type": "object", "x-guidance": {"whitespace_flexible": true}}) + )); + } + + /// Feed `doc` through the constraint one byte token at a time (the vocabulary carries every + /// byte at id 2 + byte). Ok(complete) when every byte is accepted, or the first refusal. + fn feed(c: &mut dyn TokenConstraint, doc: &[u8]) -> Result { + for (i, b) in doc.iter().enumerate() { + c.accept(2 + u32::from(*b)).map_err(|e| (i, e))?; + } + Ok(c.is_complete()) + } + + fn consumer_schemas() -> Vec<(&'static str, serde_json::Value)> { + vec![ + // paiml-implement agy/quorum-lane-schema.json (#3716, RAH) + ( + "rah-lane", + serde_json::json!({ + "$schema": "https://json-schema.org/draft/2020-12/schema", "type": "object", + "required": ["verdict", "summary", "findings"], + "properties": { + "verdict": {"type": "string", "enum": ["PASS", "FAIL", "do-not-implement-as-written"]}, + "summary": {"type": "string"}, "tree_witness": {"type": "string"}, + "findings": {"type": "array", "items": {"type": "object", "required": ["file", "claim", "grounding"], + "properties": {"file": {"type": "string"}, "line": {"type": "integer"}, "claim": {"type": "string"}, + "grounding": {"type": "string", "enum": ["cited", "measured", "asserted"]}, "fix": {"type": "string"}}}} + } + }), + ), + // rmedia #3716 issuecomment-5764061597: quiz + ( + "rmedia-quiz", + serde_json::json!({ + "$schema": "https://json-schema.org/draft/2020-12/schema", "type": "object", "additionalProperties": false, + "required": ["quiz", "graded", "n", "stem", "options", "correct_index", "grounded_in"], + "properties": { + "quiz": {"type": "string", "minLength": 1, "maxLength": 120}, "graded": {"type": "boolean"}, + "n": {"type": "integer", "minimum": 1}, "stem": {"type": "string", "minLength": 1, "maxLength": 300}, + "options": {"type": "array", "minItems": 2, "maxItems": 6, "items": {"type": "string", "minLength": 1, "maxLength": 200}}, + "correct_index": {"type": "integer", "minimum": 0, "maximum": 5}, + "grounded_in": {"type": "string", "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+-.*\\.srt$"}, + "option_feedback": {"type": "array", "maxItems": 6, "items": {"type": "string", "maxLength": 300}}, + "explanation": {"type": "string", "maxLength": 300} + } + }), + ), + // rmedia: outline + ( + "rmedia-outline", + serde_json::json!({ + "type": "object", "additionalProperties": false, + "required": ["kind", "course", "module", "index", "lesson", "description", "runtime", "seconds", "file", "rendered"], + "properties": { + "kind": {"const": "lesson"}, "course": {"type": "string", "minLength": 1}, + "module": {"type": "string", "pattern": "^Module [0-9]+: .+"}, "index": {"type": "integer", "minimum": 1}, + "lesson": {"type": "string", "minLength": 1}, "description": {"type": "string", "minLength": 1, "maxLength": 200}, + "runtime": {"type": "string", "pattern": "^[0-9]+:[0-9]{2}$"}, "seconds": {"type": "number", "exclusiveMinimum": 0}, + "file": {"type": "string", "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+-.*\\.mp4$"}, "rendered": {"type": "boolean"}, + "instructor": {"type": "string"}, "level": {"enum": ["Beginner", "Intermediate", "Advanced"]}, "repo": {"type": "string"} + } + }), + ), + // rmedia: key terms + reflection + ( + "rmedia-keyterms", + serde_json::json!({ + "type": "object", "additionalProperties": false, "required": ["lesson", "key_terms", "reflection"], + "properties": { + "lesson": {"type": "string", "pattern": "^[0-9]+\\.[0-9]+ — .+"}, + "key_terms": {"type": "array", "minItems": 3, "maxItems": 8, "items": {"type": "object", "additionalProperties": false, + "required": ["term", "definition"], "properties": {"term": {"type": "string", "maxLength": 60}, + "definition": {"type": "string", "minLength": 40, "maxLength": 600}}}}, + "reflection": {"type": "string", "minLength": 80, "maxLength": 1200} + } + }), + ), + ] + } + + #[test] + fn every_consumer_schema_compiles_exactly_and_admits_its_real_documents() { + let v = vocab(); + let env = ConstraintEnv::new(&v).expect("index the vocabulary"); + let def = "a definition that is comfortably longer than forty bytes"; + let keyterms = format!( + r#"{{"lesson":"1.1 — Opening","key_terms":[{{"term":"count","definition":"{def}"}},{{"term":"drawer","definition":"{def}"}},{{"term":"definition","definition":"{def}"}}],"reflection":"{}"}}"#, + "r".repeat(90) + ); + let accepted: Vec<(&str, String)> = vec![ + ("rah-lane", r#"{"verdict":"PASS","summary":"ok","findings":[{"file":"a.rs","line":3,"claim":"c","grounding":"cited"}]}"#.to_string()), + ("rmedia-quiz", r#"{"quiz":"Module 1: Counting honestly","graded":false,"n":1,"stem":"What is missing?","options":["A third count","A written definition"],"correct_index":1,"grounded_in":"1.1.1-first-count.srt"}"#.to_string()), + ("rmedia-outline", r#"{"kind":"lesson","course":"Counting Honestly","module":"Module 1: Counting honestly","index":1,"lesson":"Opening","description":"Why two counts disagree","runtime":"2:10","seconds":130.0,"file":"1.1.0-opening.mp4","rendered":true,"level":"Intermediate"}"#.to_string()), + ("rmedia-keyterms", keyterms), + ]; + for (name, schema) in consumer_schemas() { + let doc = &accepted + .iter() + .find(|(n, _)| *n == name) + .expect("a document per schema") + .1; + let mut c = env + .json_schema(&schema) + .unwrap_or_else(|e| panic!("{name}: {e}")); + match feed(c.as_mut(), doc.as_bytes()) { + Ok(true) => {}, + Ok(false) => panic!("{name}: the real document was accepted but is not complete"), + Err((at, e)) => panic!("{name}: byte {at} of a real document refused: {e}\n{doc}"), + } + // rmedia's framing request: ONE top-level value, so nothing may follow a complete one + let mut logits = vec![0.0f32; v.token_bytes.len()]; + c.mask(&mut logits) + .expect("the mask after a complete document"); + assert!( + logits[EOS as usize].is_finite(), + "{name}: EOS must be allowed" + ); + assert_eq!( + logits[2 + usize::from(b'{')], + f32::NEG_INFINITY, + "{name}: a second value must not start" + ); + } + } + + #[test] + fn every_consumer_schema_refuses_its_violations_at_the_byte() { + let v = vocab(); + let env = ConstraintEnv::new(&v).expect("index the vocabulary"); + let schema = |name: &str| { + consumer_schemas() + .into_iter() + .find(|(n, _)| *n == name) + .expect("schema") + .1 + }; + for (name, violating, why) in [ + ("rah-lane", r#"{"verdict":"MAYBE""#, "enum"), + ( + "rmedia-quiz", + r#"{"quiz":"q","graded":false,"n":0"#, + "minimum 1", + ), + ( + "rmedia-quiz", + r#"{"quiz":"q","graded":false,"n":1,"stem":"s","options":["a","b"],"correct_index":1,"grounded_in":"notes.txt""#, + "pattern", + ), + ( + "rmedia-quiz", + r#"{"quiz":"q","graded":false,"n":1,"stem":"s","options":["a","b"],"correct_index":1,"grounded_in":"1.1.1-a.srt","difficulty":"#, + "additionalProperties false", + ), + ("rmedia-outline", r#"{"kind":"lecture""#, "const"), + ( + "rmedia-outline", + r#"{"kind":"lesson","course":"c","module":"Chapter 1""#, + "pattern", + ), + ( + "rmedia-keyterms", + r#"{"lesson":"1.1 — O","key_terms":[{"term":"t","definition":"too short"}"#, + "minLength 40", + ), + ] { + let mut c = env + .json_schema(&schema(name)) + .unwrap_or_else(|e| panic!("{name}: {e}")); + match feed(c.as_mut(), violating.as_bytes()) { + Err((_, ConstraintError::Rejected { .. })) => {}, + other => panic!("{name} ({why}): the violation was not refused: {other:?}"), + } + } + } + + #[test] + fn a_caller_cannot_buy_an_approximation_through_x_guidance() { + let v = vocab(); + let env = ConstraintEnv::new(&v).expect("index the vocabulary"); + for schema in [ + serde_json::json!({"type": "string", "format": "not-a-format", "x-guidance": {"lenient": true}}), + serde_json::json!({"oneOf": [{"type": "integer"}, {"type": "number"}], "x-guidance": {"coerce_one_of": true}}), + ] { + match env.json_schema(&schema) { + Ok(_) => panic!("compiled approximately and was accepted: {schema}"), + Err(e) => assert!( + matches!(&e, ConstraintError::SchemaUnsupported(why) if why.contains("only approximately")), + "{e}" + ), + } + } + } + + #[test] + fn boolean_schemas_compile_compact_or_refuse() { + let v = vocab(); + let env = ConstraintEnv::new(&v).expect("index the vocabulary"); + // `true` is any JSON value, and it must still be compact: no whitespace after `{` + let mut c = env + .json_schema(&serde_json::json!(true)) + .expect("`true` compiles"); + c.accept(id_of(&v, b"{")) + .expect("any value may be an object"); + let mut logits = vec![0.0f32; v.token_bytes.len()]; + c.mask(&mut logits).expect("the mask after {"); + assert_eq!( + logits[id_of(&v, b" ") as usize], + f32::NEG_INFINITY, + "`true` must compile compact" + ); + // `false` admits nothing: refused before the first token, never a dead end at token 0 + let e = env + .json_schema(&serde_json::json!(false)) + .err() + .expect("`false` is refused"); + assert!( + matches!(&e, ConstraintError::SchemaInvalid(why) if why.contains("admits no document")), + "{e}" + ); + } + + #[test] + fn a_lark_grammar_is_enforced_too() { + let v = vocab(); + let env = ConstraintEnv::new(&v).expect("index the vocabulary"); + let mut c = env + .lark(r#"start: "PASS" | "FAIL""#) + .expect("compile the grammar"); + let out = generate(&v, Some(c.as_mut()), 16).expect("constrained run"); + assert!( + out == b"PASS" || out == b"FAIL", + "{}", + String::from_utf8_lossy(&out) + ); + } +} diff --git a/crates/aprender-serve/src/gguf/byte_level_bpe.rs b/crates/aprender-serve/src/gguf/byte_level_bpe.rs new file mode 100644 index 0000000000..93758fc37b --- /dev/null +++ b/crates/aprender-serve/src/gguf/byte_level_bpe.rs @@ -0,0 +1,593 @@ +//! Canonical byte-level BPE encoding for GGUF `tokenizer.ggml.model = "gpt2"` vocabularies +//! (#3726). +//! +//! `GGUFModel::encode` used to run greedy longest-match over each whole segment: no +//! pre-tokenizer, no merge ranks. On a byte-level BPE vocabulary that produces ids the model +//! never saw in training. `" quorum"` became `Ġquo|rum` where the merges give `Ġqu|orum`; +//! `" Title"` became `ĠĠĠ|Title` where the pre-tokenizer gives `ĠĠ|ĠTitle`; every byte +//! outside the printable glyph set became id 0. The model then copies the odd pieces out of +//! the prompt and joins canonical continuations onto them, so it quoted `quorum` back as +//! "quoorum" (#3693). +//! +//! This module ports the reference implementation, llama.cpp (the pinned comparator, +//! `scripts/llama_pin.toml`): +//! - the pre-tokenizer is `unicode_regex_split_custom_qwen2` / `_qwen35` (`src/unicode.cpp`), +//! the hand-written form of the tokenizer.json regex, selected by `tokenizer.ggml.pre`; +//! - each piece's bytes map through GPT-2's `bytes_to_unicode` glyphs; +//! - merges apply lowest rank first, leftmost on ties (`llm_tokenizer_bpe_session`); +//! - special tokens (token type CONTROL, USER_DEFINED or UNKNOWN) are split out of the text +//! first, longest first, as `tokenizer_st_partition` does with `parse_special`. + +use std::collections::hash_map::DefaultHasher; +use std::collections::{BinaryHeap, HashMap}; +use std::hash::{Hash, Hasher}; +use std::sync::{Arc, Mutex, OnceLock}; + +use unicode_properties::{GeneralCategoryGroup, UnicodeGeneralCategory}; + +use super::types::GGUFValue; + +/// GGUF token types that `tokenizer_st_partition` treats as special with `parse_special`: +/// UNKNOWN (2), CONTROL (3), USER_DEFINED (4). +const SPECIAL_TOKEN_TYPES: [i32; 3] = [2, 3, 4]; + +/// The pre-tokenizers this module implements, by their `tokenizer.ggml.pre` name. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum PreTokenizer { + /// `qwen2`: Qwen2, Qwen2.5, Qwen3 and Qwen3-MoE files. + Qwen2, + /// `qwen35`: Qwen3.5 dense and MoE. Letter runs also take combining marks (`\p{M}`). + Qwen35, +} + +impl PreTokenizer { + /// The pre-tokenizer named by a GGUF's `tokenizer.ggml.pre`, or `None` when this module + /// does not implement it. + #[must_use] + pub fn from_gguf_pre(pre: &str) -> Option { + match pre { + "qwen2" => Some(Self::Qwen2), + "qwen35" => Some(Self::Qwen35), + _ => None, + } + } + + /// Split `text` into pre-tokens: a line-for-line port of llama.cpp's + /// `unicode_regex_split_custom_qwen2` (and `_qwen35`, which differs only in treating + /// `\p{M}` as part of a letter run). Its regex, from tokenizer.json: + /// + /// ```text + /// (?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\r\n\p{L}\p{N}]?\p{L}+|\p{N}| ?[^\s\p{L}\p{N}]+[\r\n]*|\s*[\r\n]+|\s+(?!\S)|\s+ + /// ``` + #[must_use] + pub fn split(self, text: &str) -> Vec<&str> { + let sp = Splitter { + chars: text.char_indices().collect(), + marks: self == Self::Qwen35, + }; + let byte_at = |i: usize| sp.chars.get(i).map_or(text.len(), |&(b, _)| b); + let mut pieces = Vec::new(); + let mut pos = 0usize; + while pos < sp.chars.len() { + let end = sp.next_end(pos); + pieces.push(&text[byte_at(pos)..byte_at(end)]); + pos = end; + } + pieces + } +} + +/// One text's codepoints, and the regex alternatives of llama.cpp's Qwen splitter as methods +/// tried in the splitter's order. Each returns where its match ends, or `None`. +struct Splitter { + chars: Vec<(usize, char)>, + /// `qwen35`: `\p{M}` belongs to letter runs. + marks: bool, +} + +impl Splitter { + fn cpt(&self, i: usize) -> Option { + self.chars.get(i).map(|&(_, c)| c) + } + + /// `None` is llama.cpp's out-of-range `unicode_cpt_flags{}`: every flag false and + /// `as_uint() == 0`. Every in-range codepoint has a nonzero flag word. + fn flags(&self, i: usize) -> Option { + self.cpt(i).map(CptFlags::of) + } + + fn is_word(&self, f: CptFlags) -> bool { + f.letter || (self.marks && f.mark) + } + + fn is_other(&self, f: CptFlags) -> bool { + !(f.whitespace || f.letter || f.number || (self.marks && f.mark)) + } + + fn word_at(&self, i: usize) -> bool { + self.flags(i).is_some_and(|f| self.is_word(f)) + } + + fn other_at(&self, i: usize) -> bool { + self.flags(i).is_some_and(|f| self.is_other(f)) + } + + /// The end of the piece starting at `pos` (always past `pos`). + fn next_end(&self, pos: usize) -> usize { + self.contraction(pos) + .or_else(|| self.letters(pos)) + .or_else(|| self.number(pos)) + .or_else(|| self.punctuation(pos)) + .unwrap_or_else(|| self.whitespace(pos)) + } + + /// `(?i:'s|'t|'re|'ve|'m|'ll|'d)` + fn contraction(&self, pos: usize) -> Option { + if self.cpt(pos) != Some('\'') { + return None; + } + let c1 = self.cpt(pos + 1)?.to_ascii_lowercase(); + if matches!(c1, 's' | 't' | 'm' | 'd') { + return Some(pos + 2); + } + let c2 = self.cpt(pos + 2)?.to_ascii_lowercase(); + matches!((c1, c2), ('r' | 'v', 'e') | ('l', 'l')).then_some(pos + 3) + } + + /// `[^\r\n\p{L}\p{N}]?\p{L}+` (qwen35: `[\p{L}\p{M}]+`) + fn letters(&self, pos: usize) -> Option { + let c = self.cpt(pos)?; + let f = CptFlags::of(c); + if c == '\r' || c == '\n' || f.number || !(self.is_word(f) || self.word_at(pos + 1)) { + return None; + } + let mut end = pos + 1; + while self.word_at(end) { + end += 1; + } + Some(end) + } + + /// `\p{N}`: one digit per piece. + fn number(&self, pos: usize) -> Option { + self.flags(pos)?.number.then_some(pos + 1) + } + + /// `?[^\s\p{L}\p{N}]+[\r\n]*` (qwen35 also excludes `\p{M}`). llama.cpp tests the + /// NEXT codepoint when this one is a space, and an out-of-range next codepoint passes. + fn punctuation(&self, pos: usize) -> Option { + let space = self.cpt(pos)? == ' '; + let probe = if space { pos + 1 } else { pos }; + if self.flags(probe).is_some_and(|f| !self.is_other(f)) { + return None; + } + let mut end = probe; + while self.other_at(end) { + end += 1; + } + while matches!(self.cpt(end), Some('\r' | '\n')) { + end += 1; + } + Some(end) + } + + /// `\s*[\r\n]+`, then `\s+(?!\S)`, then `\s+`, then one unmatched codepoint. + fn whitespace(&self, pos: usize) -> usize { + let mut run = 0usize; + let mut last_newline_end = None; + while self.flags(pos + run).is_some_and(|f| f.whitespace) { + if matches!(self.cpt(pos + run), Some('\r' | '\n')) { + last_newline_end = Some(pos + run + 1); + } + run += 1; + } + match last_newline_end { + Some(end) => end, + // a run followed by more text gives its last codepoint to that text + None if run > 1 && pos + run < self.chars.len() => pos + run - 1, + None => pos + run.max(1), + } + } +} + +/// The subset of llama.cpp's `unicode_cpt_flags` the Qwen splitters read. +#[derive(Debug, Clone, Copy)] +struct CptFlags { + /// `\p{L}` + letter: bool, + /// `\p{N}` + number: bool, + /// `\p{M}` + mark: bool, + /// `\s`: llama.cpp's `unicode_set_whitespace` is exactly Unicode `White_Space`, which is + /// what `char::is_whitespace` tests. + whitespace: bool, +} + +impl CptFlags { + fn of(c: char) -> Self { + let group = c.general_category_group(); + Self { + letter: group == GeneralCategoryGroup::Letter, + number: group == GeneralCategoryGroup::Number, + mark: group == GeneralCategoryGroup::Mark, + whitespace: c.is_whitespace(), + } + } +} + +/// GPT-2's `bytes_to_unicode`: the glyph each byte is spelled with in a byte-level vocabulary. +/// Printable Latin-1 (`!`..`~`, `¡`..`¬`, `®`..`ÿ`) maps to itself; the other 68 bytes map to +/// U+0100 onward, in byte order. +fn byte_glyphs() -> &'static [char; 256] { + static GLYPHS: OnceLock<[char; 256]> = OnceLock::new(); + GLYPHS.get_or_init(|| { + let printable = |b: u8| matches!(b, b'!'..=b'~' | 0xA1..=0xAC | 0xAE..=0xFF); + let mut glyphs = ['\0'; 256]; + let mut next = 0u32; + for b in 0..=255u8 { + glyphs[usize::from(b)] = if printable(b) { + char::from(b) + } else { + next += 1; + char::from_u32(255 + next).expect("U+0100..U+0143 are valid scalars") + }; + } + glyphs + }) +} + +/// A byte-level BPE tokenizer built from a GGUF's vocabulary, merges, token types and +/// pre-tokenizer name. +#[derive(Debug)] +pub struct ByteLevelBpe { + pre: PreTokenizer, + token_to_id: HashMap, + /// `(left, right) -> (rank, merged)`, from `tokenizer.ggml.merges` in file order. + merges: HashMap<(u32, u32), (u32, u32)>, + /// The 256 byte glyphs' ids, by byte. + byte_ids: [Option; 256], + /// Special tokens, longest text first. + specials: Vec<(String, u32)>, +} + +/// Why a GGUF vocabulary cannot be encoded by [`ByteLevelBpe`]. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum ByteLevelBpeRefusal { + /// `tokenizer.ggml.pre` names a pre-tokenizer this module does not implement. + UnknownPreTokenizer(String), + /// The file has no `tokenizer.ggml.pre`. + MissingPreTokenizer, + /// The file has no `tokenizer.ggml.merges`. + MissingMerges, +} + +impl std::fmt::Display for ByteLevelBpeRefusal { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::UnknownPreTokenizer(p) => write!(f, "pre-tokenizer '{p}' is not implemented"), + Self::MissingPreTokenizer => write!(f, "the file has no tokenizer.ggml.pre"), + Self::MissingMerges => write!(f, "the file has no tokenizer.ggml.merges"), + } + } +} + +fn string_array<'a>( + metadata: &'a HashMap, + key: &str, +) -> Option> { + match metadata.get(key) { + Some(GGUFValue::Array(values)) => Some(values.iter().filter_map(|v| match v { + GGUFValue::String(s) => Some(s.as_str()), + _ => None, + })), + _ => None, + } +} + +impl ByteLevelBpe { + /// Build the tokenizer for a `gpt2`-model GGUF. The build is cached per distinct + /// (vocabulary, merges, token types, pre-tokenizer), so repeated calls on one model pay + /// for one hash of the tables, not for rebuilding them. + /// + /// # Errors + /// [`ByteLevelBpeRefusal`] when the file lacks merges or a pre-tokenizer this module + /// implements; the caller decides what to do instead. + pub fn from_gguf( + metadata: &HashMap, + vocab: &[String], + ) -> Result, ByteLevelBpeRefusal> { + let pre_name = match metadata.get("tokenizer.ggml.pre") { + Some(GGUFValue::String(s)) => s.as_str(), + _ => return Err(ByteLevelBpeRefusal::MissingPreTokenizer), + }; + let pre = PreTokenizer::from_gguf_pre(pre_name) + .ok_or_else(|| ByteLevelBpeRefusal::UnknownPreTokenizer(pre_name.to_string()))?; + let merges: Vec<&str> = string_array(metadata, "tokenizer.ggml.merges") + .ok_or(ByteLevelBpeRefusal::MissingMerges)? + .collect(); + let token_types: Vec = match metadata.get("tokenizer.ggml.token_type") { + Some(GGUFValue::Array(values)) => values + .iter() + .map(|v| match v { + GGUFValue::Int32(t) => *t, + _ => 1, + }) + .collect(), + _ => Vec::new(), + }; + + let mut hasher = DefaultHasher::new(); + pre.hash(&mut hasher); + vocab.hash(&mut hasher); + merges.hash(&mut hasher); + token_types.hash(&mut hasher); + let key = hasher.finish(); + + static CACHE: OnceLock)>>> = OnceLock::new(); + let cache = CACHE.get_or_init(|| Mutex::new(Vec::new())); + if let Some(hit) = cache.lock().ok().and_then(|c| { + c.iter() + .find(|(k, _)| *k == key) + .map(|(_, v)| Arc::clone(v)) + }) { + return Ok(hit); + } + let built = Arc::new(Self::build(pre, vocab, &merges, &token_types)); + if let Ok(mut c) = cache.lock() { + if c.len() >= 8 { + c.remove(0); + } + c.push((key, Arc::clone(&built))); + } + Ok(built) + } + + /// Build from explicit tables (no cache). + #[must_use] + pub fn build( + pre: PreTokenizer, + vocab: &[String], + merges: &[&str], + token_types: &[i32], + ) -> Self { + let token_to_id: HashMap = vocab + .iter() + .enumerate() + .map(|(id, t)| (t.clone(), id as u32)) + .collect(); + + let mut merge_map = HashMap::with_capacity(merges.len()); + for (rank, rule) in merges.iter().enumerate() { + let Some((left, right)) = rule.split_once(' ') else { + continue; + }; + let (Some(&l), Some(&r), Some(&m)) = ( + token_to_id.get(left), + token_to_id.get(right), + token_to_id.get(format!("{left}{right}").as_str()), + ) else { + continue; + }; + // The first (lowest-rank) occurrence of a pair wins, as a rank lookup would. + merge_map.entry((l, r)).or_insert((rank as u32, m)); + } + + let glyphs = byte_glyphs(); + let mut byte_ids = [None; 256]; + for (b, id) in byte_ids.iter_mut().enumerate() { + *id = token_to_id.get(glyphs[b].to_string().as_str()).copied(); + } + + let mut specials: Vec<(String, u32)> = if token_types.len() == vocab.len() { + vocab + .iter() + .zip(token_types) + .enumerate() + .filter(|(_, (_, t))| SPECIAL_TOKEN_TYPES.contains(t)) + .map(|(id, (text, _))| (text.clone(), id as u32)) + .collect() + } else { + // No usable token types: the `<|...|>` convention this module replaced (GH-320). + vocab + .iter() + .enumerate() + .filter(|(_, t)| t.starts_with("<|") && t.ends_with("|>")) + .map(|(id, t)| (t.clone(), id as u32)) + .collect() + }; + specials.retain(|(t, _)| !t.is_empty()); + specials.sort_by(|a, b| b.0.len().cmp(&a.0.len()).then(a.1.cmp(&b.1))); + + Self { + pre, + token_to_id, + merges: merge_map, + byte_ids, + specials, + } + } + + /// Encode `text` into token ids. Special tokens in the text are matched first, as + /// llama.cpp does with `parse_special = true`; no BOS is added (the caller owns that). + #[must_use] + pub fn encode(&self, text: &str) -> Vec { + let mut ids = Vec::new(); + for fragment in self.partition_specials(text) { + match fragment { + Fragment::Special(id) => ids.push(id), + Fragment::Raw(raw) => { + for piece in self.pre.split(raw) { + self.encode_piece(piece, &mut ids); + } + }, + } + } + ids + } + + /// `tokenizer_st_partition`: each special token, longest first, claims its occurrences + /// in the raw fragments that remain. + fn partition_specials<'t>(&self, text: &'t str) -> Vec> { + let mut fragments = vec![Fragment::Raw(text)]; + for (special, id) in &self.specials { + if !text.contains(special.as_str()) { + continue; + } + let mut next = Vec::with_capacity(fragments.len()); + for fragment in fragments { + let Fragment::Raw(mut raw) = fragment else { + next.push(fragment); + continue; + }; + while let Some(at) = raw.find(special.as_str()) { + if at > 0 { + next.push(Fragment::Raw(&raw[..at])); + } + next.push(Fragment::Special(*id)); + raw = &raw[at + special.len()..]; + } + if !raw.is_empty() { + next.push(Fragment::Raw(raw)); + } + } + fragments = next; + } + fragments + } + + /// BPE over one pre-token's byte glyphs: a port of `llm_tokenizer_bpe_session`'s merge + /// loop. Lowest rank first; among equal ranks, the leftmost pair. + fn encode_piece(&self, piece: &str, out: &mut Vec) { + let mut syms = self.seed_symbols(piece); + if syms.is_empty() { + return; + } + self.merge_all(&mut syms); + let mut at = Some(0); + while let Some(i) = at { + if syms[i].id != u32::MAX { + out.push(syms[i].id); + } + at = syms[i].next; + } + } + + /// One symbol per byte, spelled by its glyph. + fn seed_symbols(&self, piece: &str) -> Vec { + let glyphs = byte_glyphs(); + piece + .bytes() + .enumerate() + .map(|(i, b)| Sym { + // A byte-level vocabulary carries all 256 glyphs; one that does not is + // looked up by glyph text, and a miss stays visible as u32::MAX (never 0). + id: self.byte_ids[usize::from(b)].unwrap_or_else(|| { + self.token_to_id + .get(glyphs[usize::from(b)].to_string().as_str()) + .copied() + .unwrap_or(u32::MAX) + }), + prev: i.checked_sub(1), + next: Some(i + 1).filter(|&j| j < piece.len()), + alive: true, + }) + .collect() + } + + /// The merge loop: pop the best pending pair, apply it if it is still current, and queue + /// the two pairs it creates with its neighbours. + fn merge_all(&self, syms: &mut [Sym]) { + let mut queue = BinaryHeap::new(); + for left in 0..syms.len() - 1 { + self.enqueue(syms, left, left + 1, &mut queue); + } + while let Some(pending) = queue.pop() { + if !Self::apply(syms, &pending) { + continue; + } + if let Some(p) = syms[pending.left].prev { + self.enqueue(syms, p, pending.left, &mut queue); + } + if let Some(nn) = syms[pending.left].next { + self.enqueue(syms, pending.left, nn, &mut queue); + } + } + } + + /// Merge `pending.right` into `pending.left` unless either side changed since the pair + /// was queued (llama.cpp's stale-bigram check). Returns whether it merged. + fn apply(syms: &mut [Sym], pending: &Pending) -> bool { + let (l, r) = (syms[pending.left], syms[pending.right]); + let current = l.alive + && r.alive + && l.next == Some(pending.right) + && l.id == pending.left_id + && r.id == pending.right_id; + if !current { + return false; + } + syms[pending.left].id = pending.merged; + syms[pending.left].next = r.next; + syms[pending.right].alive = false; + if let Some(nn) = r.next { + syms[nn].prev = Some(pending.left); + } + true + } + + fn enqueue(&self, syms: &[Sym], left: usize, right: usize, queue: &mut BinaryHeap) { + let (left_id, right_id) = (syms[left].id, syms[right].id); + if let Some(&(rank, merged)) = self.merges.get(&(left_id, right_id)) { + queue.push(Pending { + rank, + left, + right, + left_id, + right_id, + merged, + }); + } + } +} + +enum Fragment<'t> { + Raw(&'t str), + Special(u32), +} + +#[derive(Debug, Clone, Copy)] +struct Sym { + id: u32, + prev: Option, + next: Option, + alive: bool, +} + +/// A candidate merge. Ordered so that `BinaryHeap` (a max-heap) pops the LOWEST rank first +/// and, among equal ranks, the LEFTMOST pair: llama.cpp's `llm_bigram_bpe::comparator`. +#[derive(Debug, PartialEq, Eq)] +struct Pending { + rank: u32, + left: usize, + right: usize, + left_id: u32, + right_id: u32, + merged: u32, +} + +impl Ord for Pending { + fn cmp(&self, other: &Self) -> std::cmp::Ordering { + other.rank.cmp(&self.rank).then(other.left.cmp(&self.left)) + } +} + +impl PartialOrd for Pending { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} + +#[cfg(test)] +#[path = "byte_level_bpe_tests.rs"] +mod tests; diff --git a/crates/aprender-serve/src/gguf/byte_level_bpe_tests.rs b/crates/aprender-serve/src/gguf/byte_level_bpe_tests.rs new file mode 100644 index 0000000000..e23c8214f6 --- /dev/null +++ b/crates/aprender-serve/src/gguf/byte_level_bpe_tests.rs @@ -0,0 +1,206 @@ +//! #3726: case tables for the byte-level BPE port. The splitter rows are worked through +//! llama.cpp's `unicode_regex_split_custom_qwen2` by hand; the end-to-end id parity against +//! the pinned llama.cpp on real GGUF files is `scripts/tokenizer_parity.sh`. + +use super::*; + +fn split(pre: PreTokenizer, s: &str) -> Vec<&str> { + pre.split(s) +} + +#[test] +fn byte_glyphs_are_gpt2_bytes_to_unicode() { + let g = byte_glyphs(); + let distinct: std::collections::HashSet = g.iter().copied().collect(); + assert_eq!(distinct.len(), 256, "every byte needs its own glyph"); + assert_eq!(g[usize::from(b'A')], 'A'); + assert_eq!(g[usize::from(b'~')], '~'); + assert_eq!(g[usize::from(b' ')], '\u{0120}', "space is Ġ"); + assert_eq!(g[usize::from(b'\n')], '\u{010A}', "newline is Ċ"); + assert_eq!( + g[0x00], '\u{0100}', + "the first non-printable byte is U+0100" + ); + assert_eq!( + g[0xAD], '\u{0143}', + "soft hyphen, the last non-printable byte, is U+0143" + ); + assert_eq!(g[0xE9], '\u{00E9}', "printable Latin-1 maps to itself"); +} + +#[test] +fn qwen2_split_case_table() { + let q = PreTokenizer::Qwen2; + let rows: &[(&str, &[&str])] = &[ + ("Hello world", &["Hello", " world"]), + // \s+(?!\S): a run of spaces gives its last one to the word after it + (" Title", &[" ", " Title"]), + ( + "a/docs/roadmaps/roadmap.yaml", + &["a", "/docs", "/roadmaps", "/roadmap", ".yaml"], + ), + // \p{N} is one digit per piece + ("100644", &["1", "0", "0", "6", "4", "4"]), + ("bed4ce6b", &["bed", "4", "ce", "6", "b"]), + ("it's", &["it", "'s"]), + ("WE'RE", &["WE", "'RE"]), + // punctuation run with its trailing newlines + ("!!!\n\nx", &["!!!\n\n", "x"]), + (" !!!", &[" !!!"]), + // \s*[\r\n]+ then \s+(?!\S) then a word with its space + ("x \n y", &["x", " \n", " ", " y"]), + // trailing whitespace at the end of the text is one piece + ("a ", &["a", " "]), + ("a ", &["a", " "]), + // a lone space before a digit stays alone + ("x 1", &["x", " ", "1"]), + // non-ASCII letters are letters; U+200B (a format char) is "other" + ("naïve café", &["naïve", " café"]), + ("a\u{200B}b", &["a", "\u{200B}b"]), + ("────────", &["────────"]), + ("日本語テキスト", &["日本語テキスト"]), + ]; + for (input, want) in rows { + assert_eq!(&split(q, input), want, "qwen2 split of {input:?}"); + } +} + +#[test] +fn qwen35_takes_combining_marks_into_letter_runs_and_qwen2_does_not() { + let decomposed = "e\u{301}t\u{e9}"; // e + COMBINING ACUTE, t, é + assert_eq!(split(PreTokenizer::Qwen35, decomposed), vec![decomposed]); + assert_eq!( + split(PreTokenizer::Qwen2, decomposed), + vec!["e", "\u{301}t\u{e9}"] + ); +} + +#[test] +fn split_pieces_concatenate_to_the_input() { + let text = "fn main() {\n let x = \"quorum\"; // it's 100% ok\r\n\t\tdone \n}\n "; + for pre in [PreTokenizer::Qwen2, PreTokenizer::Qwen35] { + assert_eq!( + pre.split(text).concat(), + text, + "{pre:?} must not drop or add text" + ); + } +} + +/// A synthetic byte-level vocabulary: the 256 glyphs (ids 0..256), then `extra`. +fn vocab_with(extra: &[&str]) -> Vec { + let mut v: Vec = byte_glyphs().iter().map(char::to_string).collect(); + v.extend(extra.iter().map(|s| (*s).to_string())); + v +} + +fn id_of(vocab: &[String], t: &str) -> u32 { + vocab.iter().position(|v| v == t).expect("in vocab") as u32 +} + +#[test] +fn merges_apply_lowest_rank_first() { + let vocab = vocab_with(&["ab", "bc", "abc"]); + // "b c" outranks "a b": abc -> a|bc -> abc + let bpe = ByteLevelBpe::build(PreTokenizer::Qwen2, &vocab, &["b c", "a b", "a bc"], &[]); + assert_eq!(bpe.encode("abc"), vec![id_of(&vocab, "abc")]); + // "a b" outranks "b c", and there is no "ab c": abc -> ab|c + let bpe = ByteLevelBpe::build(PreTokenizer::Qwen2, &vocab, &["a b", "b c"], &[]); + assert_eq!( + bpe.encode("abc"), + vec![id_of(&vocab, "ab"), id_of(&vocab, "c")] + ); +} + +#[test] +fn equal_ranks_merge_leftmost_first() { + let vocab = vocab_with(&["aa"]); + let bpe = ByteLevelBpe::build(PreTokenizer::Qwen2, &vocab, &["a a"], &[]); + assert_eq!( + bpe.encode("aaa"), + vec![id_of(&vocab, "aa"), id_of(&vocab, "a")] + ); +} + +#[test] +fn merges_never_cross_pre_token_boundaries() { + // " b" exists as a merge, but "a" and " b" are different pieces: no "a b" token forms. + let vocab = vocab_with(&["\u{0120}b", "a\u{0120}b"]); + let bpe = ByteLevelBpe::build( + PreTokenizer::Qwen2, + &vocab, + &["\u{0120} b", "a \u{0120}b"], + &[], + ); + assert_eq!( + bpe.encode("a b"), + vec![id_of(&vocab, "a"), id_of(&vocab, "\u{0120}b")] + ); +} + +#[test] +fn non_ascii_bytes_are_their_glyphs_never_id_zero() { + let vocab = vocab_with(&[]); + let bpe = ByteLevelBpe::build(PreTokenizer::Qwen35, &vocab, &[], &[]); + let glyphs = byte_glyphs(); + // é = C3 A9; ─ = E2 94 80 + let want: Vec = "é─" + .bytes() + .map(|b| id_of(&vocab, &glyphs[usize::from(b)].to_string())) + .collect(); + let got = bpe.encode("é─"); + assert_eq!(got, want); + assert!( + !got.contains(&0) || want.contains(&0), + "the old encoder's id-0 fallback is gone" + ); +} + +#[test] +fn special_tokens_are_partitioned_longest_first_by_token_type() { + // types: 1 normal, 3 control, 4 user-defined + let vocab = vocab_with(&["<|im_start|>", "", "<|im", "user"]); + let mut types = vec![1i32; vocab.len()]; + let im = id_of(&vocab, "<|im_start|>"); + let think = id_of(&vocab, ""); + types[im as usize] = 3; + types[think as usize] = 4; + types[id_of(&vocab, "<|im") as usize] = 3; + let bpe = ByteLevelBpe::build( + PreTokenizer::Qwen35, + &vocab, + &["u s", "us e", "use r"], + &types, + ); + let got = bpe.encode("<|im_start|>user"); + assert_eq!(got.first(), Some(&im), "the longer special wins over <|im"); + assert_eq!( + got.last(), + Some(&think), + "a USER_DEFINED token is special too (the old <|..|> rule missed )" + ); +} + +#[test] +fn unknown_pre_tokenizer_is_refused_by_name() { + let mut md = HashMap::new(); + md.insert( + "tokenizer.ggml.pre".to_string(), + GGUFValue::String("llama-bpe".to_string()), + ); + md.insert( + "tokenizer.ggml.merges".to_string(), + GGUFValue::Array(vec![]), + ); + let err = + ByteLevelBpe::from_gguf(&md, &vocab_with(&[])).expect_err("llama-bpe is not implemented"); + assert_eq!( + err, + ByteLevelBpeRefusal::UnknownPreTokenizer("llama-bpe".to_string()) + ); + md.remove("tokenizer.ggml.pre"); + assert_eq!( + ByteLevelBpe::from_gguf(&md, &vocab_with(&[])).expect_err("no pre"), + ByteLevelBpeRefusal::MissingPreTokenizer + ); +} diff --git a/crates/aprender-serve/src/gguf/inference/generate_quantized.rs b/crates/aprender-serve/src/gguf/inference/generate_quantized.rs index 059878fb6b..ab7d1da0fa 100644 --- a/crates/aprender-serve/src/gguf/inference/generate_quantized.rs +++ b/crates/aprender-serve/src/gguf/inference/generate_quantized.rs @@ -19,7 +19,7 @@ impl OwnedQuantizedModel { /// [`RealizarError::ContextLimitExceeded`] when the PROMPT alone exceeds the /// context window (GH-167) — that one is unsatisfiable, and callers map it to /// HTTP 400 because it is determined entirely by the request. - fn effective_max_tokens(&self, prompt_len: usize, requested: usize) -> Result { + pub(crate) fn effective_max_tokens(&self, prompt_len: usize, requested: usize) -> Result { if prompt_len > self.config.context_length { return Err(RealizarError::ContextLimitExceeded { provided: prompt_len, diff --git a/crates/aprender-serve/src/gguf/mod.rs b/crates/aprender-serve/src/gguf/mod.rs index 7c6743e34f..f56b666bbf 100644 --- a/crates/aprender-serve/src/gguf/mod.rs +++ b/crates/aprender-serve/src/gguf/mod.rs @@ -66,6 +66,8 @@ mod wgpu_model; // Pure math operations (shared between CPU and GPU paths) // UCBD §4: pub for re-export of rms_norm at crate root +/// #3726: canonical byte-level BPE (pre-tokenizer + ranked merges) for `gpt2` vocabularies. +pub mod byte_level_bpe; pub mod ops; // Test helpers module - shared utilities for GGUF tests diff --git a/crates/aprender-serve/src/gguf/test_factory.rs b/crates/aprender-serve/src/gguf/test_factory.rs index 7077e3f1ca..6d68755918 100644 --- a/crates/aprender-serve/src/gguf/test_factory.rs +++ b/crates/aprender-serve/src/gguf/test_factory.rs @@ -412,6 +412,19 @@ impl GGUFBuilder { self } + /// Add an i32 array metadata value (type 9, element_type 5), e.g. `tokenizer.ggml.token_type` + #[must_use] + pub fn add_i32_array(mut self, key: &str, values: &[i32]) -> Self { + let mut bytes = Vec::new(); + bytes.extend_from_slice(&5u32.to_le_bytes()); // element type = int32 + bytes.extend_from_slice(&(values.len() as u64).to_le_bytes()); + for &val in values { + bytes.extend_from_slice(&val.to_le_bytes()); + } + self.metadata.push((key.to_string(), 9, bytes)); + self + } + // ========================================================================= // Build // ========================================================================= diff --git a/crates/aprender-serve/src/gguf/test_factory_create.rs b/crates/aprender-serve/src/gguf/test_factory_create.rs index 3c335635f4..e2a7578943 100644 --- a/crates/aprender-serve/src/gguf/test_factory_create.rs +++ b/crates/aprender-serve/src/gguf/test_factory_create.rs @@ -298,6 +298,15 @@ pub fn build_minimal_llama_gguf( /// - num_layers: 1 #[must_use] pub fn build_executable_pygmy_gguf() -> Vec { + build_executable_pygmy_gguf_with(|builder| builder) +} + +/// [`build_executable_pygmy_gguf`] with extra metadata (a tokenizer, a chat +/// template) added by `metadata` before the tensors. +#[must_use] +pub fn build_executable_pygmy_gguf_with( + metadata: impl FnOnce(GGUFBuilder) -> GGUFBuilder, +) -> Vec { // Active Pygmy dimensions (T-COV-95) // All dimensions chosen to align with Q4_0's 32-element block size const VOCAB_SIZE: usize = 32; @@ -335,17 +344,19 @@ pub fn build_executable_pygmy_gguf() -> Vec { // This is separate from token_embd to avoid the "type 0" fallback let lm_head_data = create_q4_0_data(HIDDEN_DIM * VOCAB_SIZE); // 32x32 = 1024 elements - GGUFBuilder::new() - // Metadata - full LLaMA config - .architecture("llama") - .hidden_dim("llama", HIDDEN_DIM as u32) - .num_layers("llama", 1) - .num_heads("llama", NUM_HEADS as u32) - .num_kv_heads("llama", NUM_KV_HEADS as u32) - .context_length("llama", CONTEXT_LENGTH as u32) - .rope_freq_base("llama", 10000.0) - .rms_epsilon("llama", 1e-5) - .ffn_hidden_dim("llama", INTERMEDIATE_DIM as u32) + metadata( + GGUFBuilder::new() + // Metadata - full LLaMA config + .architecture("llama") + .hidden_dim("llama", HIDDEN_DIM as u32) + .num_layers("llama", 1) + .num_heads("llama", NUM_HEADS as u32) + .num_kv_heads("llama", NUM_KV_HEADS as u32) + .context_length("llama", CONTEXT_LENGTH as u32) + .rope_freq_base("llama", 10000.0) + .rms_epsilon("llama", 1e-5) + .ffn_hidden_dim("llama", INTERMEDIATE_DIM as u32), + ) // Token embedding - F32 (lookup table, not matmul) .add_f32_tensor( "token_embd.weight", diff --git a/crates/aprender-serve/src/gguf/tests/constraint_vocab.rs b/crates/aprender-serve/src/gguf/tests/constraint_vocab.rs new file mode 100644 index 0000000000..995c802fdb --- /dev/null +++ b/crates/aprender-serve/src/gguf/tests/constraint_vocab.rs @@ -0,0 +1,93 @@ +// #3568: the constraint's vocabulary is built from the SAME per-token decoding `decode` uses. + +fn gpt2_vocab_model(token_types: Option<&[i32]>) -> GGUFModel { + // Qwen-style byte-level BPE: `Ġ` is a space, `Ċ` a newline, `<0x0A>` a byte token + let tokens = ["<|endoftext|>", "{\"", "Ġverdict", "Ċ", "<0x0A>", "<|im_end|>", "", "é"]; + let mut b = GGUFBuilder::new() + .architecture("qwen2") + .hidden_dim("qwen2", 32) + .num_layers("qwen2", 1) + .num_heads("qwen2", 1) + .add_string("tokenizer.ggml.model", "gpt2") + .add_string_array("tokenizer.ggml.tokens", &tokens) + .add_u32("tokenizer.ggml.eos_token_id", 0); + if let Some(t) = token_types { + b = b.add_i32_array("tokenizer.ggml.token_type", t); + } + GGUFModel::from_bytes(&b.build()).expect("test model") +} + +#[test] +fn constraint_vocab_bytes_are_exactly_what_decode_renders() { + let model = gpt2_vocab_model(None); + let v = model.constraint_vocab().expect("a vocabulary and an eos"); + assert_eq!(v.eos, 0); + for (id, special) in v.special.iter().enumerate() { + if *special { + continue; + } + let decoded = model.decode(&[id as u32]); + match std::str::from_utf8(&v.token_bytes[id]) { + // whole UTF-8: decode renders exactly these bytes + Ok(text) => assert_eq!(text, decoded, "token {id}"), + // a partial UTF-8 byte: decode's final lossy step shows U+FFFD, the RAW byte is kept + Err(_) => assert_eq!(decoded, "\u{FFFD}", "token {id}"), + } + } + assert_eq!(v.token_bytes[2], b" verdict"); + assert_eq!(v.token_bytes[3], b"\n"); + assert_eq!(v.token_bytes[4], b"\n"); + // `é` in a byte-level vocabulary is the single byte 0xE9, the first half of nothing + assert_eq!(v.token_bytes[7], vec![0xE9]); +} + +#[test] +fn constraint_vocab_marks_control_tokens_special_from_token_type() { + // 1 normal, 3 control, 4 user-defined, 5 unused + let model = gpt2_vocab_model(Some(&[3, 1, 1, 1, 6, 3, 4, 5])); + let v = model.constraint_vocab().expect("a vocabulary and an eos"); + assert_eq!(v.special, vec![true, false, false, false, false, true, false, true]); + // a user-defined token like is text; its bytes are its decoding + assert!(!v.special[6]); +} + +#[test] +fn constraint_vocab_falls_back_to_the_encode_rule_without_token_type() { + let model = gpt2_vocab_model(None); + let v = model.constraint_vocab().expect("a vocabulary and an eos"); + // GH-320's rule: `<|…|>` is special + assert!(v.special[0] && v.special[5]); + assert!(!v.special[6], " is not a <|…|> token"); +} + +#[test] +fn constraint_vocab_needs_an_eos() { + let data = GGUFBuilder::new() + .architecture("llama") + .hidden_dim("llama", 32) + .num_layers("llama", 1) + .num_heads("llama", 1) + .add_string("tokenizer.ggml.model", "llama") + .add_string_array("tokenizer.ggml.tokens", &["a", "b"]) + .build(); + let model = GGUFModel::from_bytes(&data).expect("test model"); + assert!(model.constraint_vocab().is_none(), "no eos id: no constraint vocabulary"); +} + +#[test] +fn constraint_vocab_maps_sentencepiece_boundaries_on_bytes_and_keeps_byte_tokens() { + let data = GGUFBuilder::new() + .architecture("llama") + .hidden_dim("llama", 32) + .num_layers("llama", 1) + .num_heads("llama", 1) + .add_string("tokenizer.ggml.model", "llama") + .add_string_array("tokenizer.ggml.tokens", &["", "▁hello", "<0xE6>"]) + .add_u32("tokenizer.ggml.eos_token_id", 0) + .build(); + let model = GGUFModel::from_bytes(&data).expect("test model"); + let v = model.constraint_vocab().expect("a vocabulary and an eos"); + assert_eq!(v.token_bytes[1], b" hello"); + // a lone byte of a multi-byte UTF-8 sequence must survive as that byte, never U+FFFD + assert_eq!(v.token_bytes[2], vec![0xE6]); +} diff --git a/crates/aprender-serve/src/gguf/tests/tests_28.rs b/crates/aprender-serve/src/gguf/tests/tests_28.rs index eae87fa2ce..27d01ceeb6 100644 --- a/crates/aprender-serve/src/gguf/tests/tests_28.rs +++ b/crates/aprender-serve/src/gguf/tests/tests_28.rs @@ -444,3 +444,4 @@ fn test_decode_gpt2_style() { } include!("decode_byte.rs"); +include!("constraint_vocab.rs"); diff --git a/crates/aprender-serve/src/gguf/token.rs b/crates/aprender-serve/src/gguf/token.rs index 457e46f0ca..10319a7657 100644 --- a/crates/aprender-serve/src/gguf/token.rs +++ b/crates/aprender-serve/src/gguf/token.rs @@ -136,27 +136,7 @@ impl GGUFModel { let token = vocab .get(id as usize) .map_or("�", std::string::String::as_str); - - // Check if this is a byte token like <0xE6> - if token.starts_with("<0x") && token.ends_with('>') && token.len() == 6 { - if let Ok(byte_val) = u8::from_str_radix(token.get(3..5).expect("byte token <0xNN> has len 6, indices 3..5 always valid"), 16) { - bytes.push(byte_val); - continue; - } - } - - // For GPT-2 style tokenizers, decode byte-level BPE properly - // Each unicode character in the token represents a raw byte - if is_gpt2_style { - for c in token.chars() { - if let Some(byte) = gpt2_unicode_to_byte(c) { - bytes.push(byte); - } - } - } else { - // SentencePiece style - tokens are regular strings - bytes.extend_from_slice(token.as_bytes()); - } + push_token_bytes(token, is_gpt2_style, &mut bytes); } // Decode bytes as UTF-8 (lossy for invalid sequences) @@ -177,9 +157,32 @@ impl GGUFModel { } } + /// #3726: the canonical encoding of a byte-level (`gpt2`) vocabulary, or `None` when the + /// file is not byte-level or its pre-tokenizer is not implemented (said once on stderr). + /// Greedy longest-match gave these vocabularies ids the model never saw in training + /// (" quorum" -> `Ġquo|rum`, where the merges give `Ġqu|orum`; every byte outside the + /// printable glyphs -> id 0), and the model quoted identifiers back corrupted (#3693). + fn encode_byte_level(&self, text: &str, vocab: &[String]) -> Option> { + let byte_level = self + .metadata + .get("tokenizer.ggml.model") + .is_some_and(|v| matches!(v, GGUFValue::String(s) if s == "gpt2" || s == "bpe")); + if !byte_level { + return None; + } + crate::gguf::byte_level_bpe::ByteLevelBpe::from_gguf(&self.metadata, vocab) + .map_err(|refusal| warn_greedy_tokenizer_fallback_once(&refusal)) + .ok() + .map(|bpe| bpe.encode(text)) + } + /// Encode text to token IDs using vocabulary /// - /// Uses greedy longest-match tokenization with special token priority. + /// A byte-level (`tokenizer.ggml.model = "gpt2"`) vocabulary whose pre-tokenizer is + /// implemented is encoded canonically by [`crate::gguf::byte_level_bpe`]: special tokens, + /// then the model's pre-tokenizer, then its ranked merges, identical to llama.cpp (#3726). + /// Everything else still takes the greedy longest-match below, which is NOT the model's + /// tokenization; that fallback says so once on stderr. /// Returns None if no vocabulary is available. /// /// Supports both tokenizer types: @@ -189,6 +192,10 @@ impl GGUFModel { pub fn encode(&self, text: &str) -> Option> { let vocab = self.vocabulary()?; + if let Some(ids) = self.encode_byte_level(text, &vocab) { + return Some(ids); + } + // Build reverse lookup: token string -> token ID let token_to_id: std::collections::HashMap<&str, u32> = vocab .iter() @@ -215,32 +222,7 @@ impl GGUFModel { let space_char = if is_gpt2_style { '\u{0120}' } else { '▁' }; // Split text on special tokens first, preserving them - let mut segments: Vec<(bool, &str)> = Vec::new(); // (is_special, text) - let mut text_remaining = text; - while !text_remaining.is_empty() { - // Find earliest special token match - let mut earliest_match: Option<(usize, &str, u32)> = None; - for &(special_tok, special_id) in &special_tokens { - if let Some(pos) = text_remaining.find(special_tok) { - if earliest_match.is_none() - || pos < earliest_match.as_ref().map_or(usize::MAX, |m| m.0) - { - earliest_match = Some((pos, special_tok, special_id)); - } - } - } - - if let Some((pos, special_tok, _)) = earliest_match { - if pos > 0 { - segments.push((false, &text_remaining[..pos])); - } - segments.push((true, special_tok)); - text_remaining = &text_remaining[pos + special_tok.len()..]; - } else { - segments.push((false, text_remaining)); - break; - } - } + let segments = split_on_special_tokens(text, &special_tokens); let mut tokens = Vec::new(); @@ -323,6 +305,105 @@ impl GGUFModel { Some(tokens) } + + /// The vocabulary as schema-constrained decoding sees it (#3568): each + /// token's raw bytes from [`push_token_bytes`], the same per-token decoding + /// [`Self::decode`] uses, so the mask and the decoded text agree about what a + /// token is. Control and unused tokens (`tokenizer.ggml.token_type` 3 and 5) + /// are special and can never be emitted as constrained text. Without that + /// metadata, a `<|…|>` token is special: the same rule `encode` uses (GH-320). + /// + /// `None` when the file has no vocabulary or no end-of-sequence id. + #[must_use] + pub fn constraint_vocab(&self) -> Option { + let vocab = self.vocabulary()?; + let eos = self.eos_token_id()?; + let is_gpt2_style = self + .metadata + .get(crate::gguf::keys::TOKENIZER_MODEL) + .is_some_and(|v| matches!(v, GGUFValue::String(s) if s == "gpt2" || s == "bpe")); + let token_types: Option> = match self.metadata.get(TOKENIZER_TOKEN_TYPE) { + Some(GGUFValue::Array(arr)) => arr + .iter() + .map(|v| match v { + GGUFValue::Int32(t) => Some(*t), + _ => None, + }) + .collect(), + _ => None, + }; + let mut token_bytes = Vec::with_capacity(vocab.len()); + let mut special = Vec::with_capacity(vocab.len()); + for (id, token) in vocab.iter().enumerate() { + let is_special = match token_types.as_ref().and_then(|t| t.get(id)) { + Some(&t) => t == 3 || t == 5, // CONTROL, UNUSED + None => token.starts_with("<|") && token.ends_with("|>"), + }; + let mut bytes = Vec::new(); + if is_special { + bytes.extend_from_slice(token.as_bytes()); + } else { + push_token_bytes(token, is_gpt2_style, &mut bytes); + if !is_gpt2_style { + // decode() maps SentencePiece's word boundary `▁` (E2 96 81) to a space after + // joining; done per token here, on BYTES, so a lone <0xNN> byte token survives + bytes = replace_sp_word_boundary(&bytes); + } + } + token_bytes.push(bytes); + special.push(is_special); + } + Some(crate::constrain::ConstraintVocab { + token_bytes, + special, + eos, + }) + } +} + +/// `tokenizer.ggml.token_type`: one llama.cpp token type per token (1 normal, +/// 3 control, 5 unused, 6 byte, …). +const TOKENIZER_TOKEN_TYPE: &str = "tokenizer.ggml.token_type"; + +/// Append one token's raw bytes, exactly as [`GGUFModel::decode`] renders it: a +/// `<0xNN>` byte token is that byte; a GPT-2 byte-level BPE token maps each char +/// back to its byte; a SentencePiece token is its UTF-8 text. `decode` and the +/// constraint vocabulary (#3568) both call this, so they cannot drift apart. +fn push_token_bytes(token: &str, is_gpt2_style: bool, bytes: &mut Vec) { + if token.starts_with("<0x") && token.ends_with('>') && token.len() == 6 { + if let Ok(byte_val) = u8::from_str_radix( + token + .get(3..5) + .expect("byte token <0xNN> has len 6, indices 3..5 always valid"), + 16, + ) { + bytes.push(byte_val); + return; + } + } + if is_gpt2_style { + // Each unicode character in a byte-level BPE token represents a raw byte + bytes.extend(token.chars().filter_map(gpt2_unicode_to_byte)); + } else { + bytes.extend_from_slice(token.as_bytes()); + } +} + +/// SentencePiece's word boundary `▁` (U+2581, bytes E2 96 81) as a space, on raw bytes. +fn replace_sp_word_boundary(bytes: &[u8]) -> Vec { + const BOUNDARY: [u8; 3] = [0xE2, 0x96, 0x81]; + let mut out = Vec::with_capacity(bytes.len()); + let mut i = 0; + while i < bytes.len() { + if bytes[i..].starts_with(&BOUNDARY) { + out.push(b' '); + i += BOUNDARY.len(); + } else { + out.push(bytes[i]); + i += 1; + } + } + out } use crate::gguf::{ @@ -332,3 +413,38 @@ use crate::gguf::{ include!("loader_parse.rs"); include!("metadata.rs"); + +/// The greedy longest-match fallback is not the model's tokenization; say so, once (#3726). +fn warn_greedy_tokenizer_fallback_once(refusal: &crate::gguf::byte_level_bpe::ByteLevelBpeRefusal) { + static WARNED: std::sync::Once = std::sync::Once::new(); + WARNED.call_once(|| { + eprintln!( + "warning: byte-level BPE tokenizer: {refusal}; falling back to greedy longest-match, \ + which does NOT reproduce the model's tokenization (#3726)" + ); + }); +} + +/// Split `text` at the earliest occurrence of any special token, repeatedly, keeping the +/// specials as their own `(true, token)` segments (the greedy path's special handling). +fn split_on_special_tokens<'t>(text: &'t str, special_tokens: &[(&'t str, u32)]) -> Vec<(bool, &'t str)> { + let mut segments = Vec::new(); + let mut rest = text; + while !rest.is_empty() { + let earliest = special_tokens + .iter() + .filter_map(|&(tok, _)| rest.find(tok).map(|pos| (pos, tok))) + .min_by_key(|&(pos, _)| pos); + let Some((pos, tok)) = earliest else { + segments.push((false, rest)); + break; + }; + if pos > 0 { + segments.push((false, &rest[..pos])); + } + segments.push((true, tok)); + rest = &rest[pos + tok.len()..]; + } + segments +} + diff --git a/crates/aprender-serve/src/infer/inference_result.rs b/crates/aprender-serve/src/infer/inference_result.rs index c5fbc35b94..1130ef03e3 100644 --- a/crates/aprender-serve/src/infer/inference_result.rs +++ b/crates/aprender-serve/src/infer/inference_result.rs @@ -108,9 +108,26 @@ pub(crate) fn validate_model_path(path: &std::path::Path) -> Result<()> { /// - Model format is unsupported /// - Generation fails pub fn run_inference(config: &InferenceConfig) -> Result { + run_inference_report(config).map(|(result, _)| result) +} + +/// [`run_inference`], plus what only the decode path knows (#3718): why +/// generation ended and the model's context window. +/// +/// A separate entry point rather than new `InferenceResult` fields, so the +/// dozens of places that build an `InferenceResult` literal are untouched. +/// +/// # Errors +/// +/// Exactly those of [`run_inference`]. +pub fn run_inference_report( + config: &InferenceConfig, +) -> Result<(InferenceResult, run_report::RunReport)> { // PMAT-COV-95: Mock backend for testing without disk I/O if config.use_mock_backend { - return run_mock_inference(config); + let result = run_mock_inference(config)?; + let report = mock_run_report(config, &result); + return Ok((result, report)); } // GH-213: Detect sharded SafeTensors index.json BEFORE reading the file. @@ -123,7 +140,8 @@ pub fn run_inference(config: &InferenceConfig) -> Result { let format = ModelFormat::SafeTensors; let prepared = prepare_tokens(config, &format)?; - return run_sharded_safetensors_inference(config, &prepared); + return run_sharded_safetensors_inference(config, &prepared) + .map(|r| (r, run_report::RunReport::default())); } // Validate path to prevent traversal attacks (F-SEC-222) @@ -163,8 +181,27 @@ pub fn run_inference(config: &InferenceConfig) -> Result { match format { ModelFormat::Gguf => run_gguf_inference(config, &prepared), - ModelFormat::Apr => run_apr_inference(config, &prepared), - ModelFormat::SafeTensors => run_safetensors_inference(config, &prepared), + // Not reported on these paths yet: `None`, never a guess. + ModelFormat::Apr => { + run_apr_inference(config, &prepared).map(|r| (r, run_report::RunReport::default())) + }, + ModelFormat::SafeTensors => run_safetensors_inference(config, &prepared) + .map(|r| (r, run_report::RunReport::default())), + } +} + +/// The report for the mock backend: it generates `min(max_tokens, 32)` tokens +/// and no stop token, so a budget of 32 or less is spent and anything larger +/// ended as if on a stop. +fn mock_run_report(config: &InferenceConfig, result: &InferenceResult) -> run_report::RunReport { + let generated = result.tokens.get(result.input_token_count..).unwrap_or(&[]); + run_report::RunReport { + finish_reason: Some(run_report::FinishReason::from_decode( + generated, + &config.stop_tokens, + config.max_tokens, + )), + context_length: None, } } @@ -174,7 +211,7 @@ pub fn run_inference(config: &InferenceConfig) -> Result { fn run_gguf_inference( config: &InferenceConfig, prepared: &PreparedTokens, -) -> Result { +) -> Result<(InferenceResult, run_report::RunReport)> { use crate::gguf::{MappedGGUFModel, OwnedQuantizedModel, QuantizedGenerateConfig}; if config.verbose { @@ -291,17 +328,38 @@ fn run_gguf_inference( used_gpu, ); - Ok(InferenceResult { - text, - tokens, + // #3718: only the dense CPU loop (`generate_with_cache`) shrinks the budget to + // the context room; the hybrid, MoE, CUDA and wgpu loops run `max_tokens`. + let clamps_to_context = !is_qwen35 && canonical_arch != "qwen3_moe" && !used_gpu; + let budget = run_report::decode_budget( + gen_config.max_tokens, input_token_count, - generated_token_count, - inference_ms, - tok_per_sec: tps, - load_ms, - format: "GGUF".to_string(), - used_gpu, - }) + model_config.context_length, + clamps_to_context, + ); + let report = run_report::RunReport { + finish_reason: Some(run_report::FinishReason::from_decode( + generated_tokens, + &gen_config.stop_tokens, + budget, + )), + context_length: Some(model_config.context_length), + }; + + Ok(( + InferenceResult { + text, + tokens, + input_token_count, + generated_token_count, + inference_ms, + tok_per_sec: tps, + load_ms, + format: "GGUF".to_string(), + used_gpu, + }, + report, + )) } /// Print verbose model info for GGUF inference diff --git a/crates/aprender-serve/src/infer/mod.rs b/crates/aprender-serve/src/infer/mod.rs index 699948c133..99b9bd3950 100644 --- a/crates/aprender-serve/src/infer/mod.rs +++ b/crates/aprender-serve/src/infer/mod.rs @@ -588,3 +588,4 @@ include!("mod_05.rs"); include!("batch.rs"); pub mod qwen3_moe_generate; +pub mod run_report; diff --git a/crates/aprender-serve/src/infer/run_report.rs b/crates/aprender-serve/src/infer/run_report.rs new file mode 100644 index 0000000000..0a87d790e5 --- /dev/null +++ b/crates/aprender-serve/src/infer/run_report.rs @@ -0,0 +1,343 @@ +//! What a run owes its caller beyond the text (#3718). +//! +//! `apr run --json` reported the generated tokens and nothing about the prompt, so a +//! consumer that needed "how many tokens did the model actually read" rebuilt the +//! model's tokenizer from the GGUF to count them (RAH, #3716: 13,495 on one lane). And +//! a reply cut at `max_tokens` looked exactly like a reply that finished, so a +//! consumer could not tell a truncated verdict from a malformed one. +//! +//! [`RunReport`] carries the two facts only the decode path knows: why generation +//! ended, and the context window the prompt was measured against. The prompt count +//! itself is `InferenceResult::input_token_count`, which is the post-chat-template +//! count `prepare_tokens` fed to the model. + +/// Why generation ended. +/// +/// Shared by `apr run --json` and the OpenAI-compatible server (`api::types` +/// re-exports it), so both surfaces spell the same outcome the same way. +/// +/// Dogfood 0.63.0 (#2375 finding 6): the STREAMING chat path emitted the string +/// literal `"stop"` in its terminal chunk no matter what happened, while the +/// non-streaming path on the identical request correctly reported `"length"` +/// when the generation hit `max_tokens`. A client that streams therefore cannot +/// tell a truncated answer from a finished one, and every "continue from where +/// you stopped" flow silently breaks. +/// +/// The type exists so that literal cannot come back: the terminal-chunk +/// constructor takes a `FinishReason`, which is only obtainable from +/// [`FinishReason::from_generation`] (or an explicit, named variant). There is +/// no `&str` parameter left to hardcode. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum FinishReason { + /// The model emitted a stop token or a stop string matched. + Stop, + /// Generation was cut off at its token budget: `max_tokens`, or the room left + /// in the context window when a path shrinks the budget to fit it. + Length, +} + +impl FinishReason { + /// Decide the reason from what the generation actually did. + /// + /// Mirrors `finalize_chat_text` / `completion_finish_reason`: a matched stop + /// string wins over the budget, and a model that terminated early is + /// `Stop`. Only "ran to the budget with no stop match" is `Length`. + /// + /// `max_tokens` must be the budget the loop RAN with, not the one requested: + /// a server that clamps the request to the context window and then compares + /// against the unclamped number reports every clamped cut as `Stop` (#3718). + #[must_use] + pub fn from_generation(stopped: bool, completion_tokens: usize, max_tokens: usize) -> Self { + if !stopped && completion_tokens >= max_tokens { + Self::Length + } else { + Self::Stop + } + } + + /// Decide the reason from the tokens a decode loop returned. + /// + /// Every generate loop `apr run` reaches ends, short of an error, in one of two + /// ways: it sampled a stop token, or it spent its budget. The loops disagree + /// about the stop token itself. The Qwen3.5 hybrid, MoE and wgpu loops push it + /// and then break; the dense CPU and CUDA loops break before pushing. So the + /// last token alone cannot separate "stopped" from "cut": + /// + /// - a last token in `stop_tokens` is `Stop` (a loop that pushes it); + /// - otherwise a run that filled `budget` is `Length`; + /// - a shorter run can only have ended on a stop token its loop consumed + /// without pushing, which is `Stop`. + #[must_use] + pub fn from_decode(generated: &[u32], stop_tokens: &[u32], budget: usize) -> Self { + let ended_on_stop = generated.last().is_some_and(|t| stop_tokens.contains(t)); + Self::from_generation(ended_on_stop, generated.len(), budget) + } + + /// The wire string OpenAI clients match on. + #[must_use] + pub fn as_str(self) -> &'static str { + match self { + Self::Stop => "stop", + Self::Length => "length", + } + } +} + +impl std::fmt::Display for FinishReason { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(self.as_str()) + } +} + +/// The token budget a decode loop actually ran with. +/// +/// Only the dense CPU loop shrinks `max_tokens` to the room left in the context +/// window (`effective_max_tokens`, aprender#2376), so `clamps_to_context` is true +/// for that loop alone. Every other loop runs the request's `max_tokens` as is. +/// A prompt that does not fit at all is refused before this is asked +/// (`ContextLimitExceeded`), so the subtraction saturates only on paths that do +/// not check. +#[must_use] +pub fn decode_budget( + max_tokens: usize, + prompt_len: usize, + context_length: usize, + clamps_to_context: bool, +) -> usize { + if clamps_to_context { + max_tokens.min(context_length.saturating_sub(prompt_len)) + } else { + max_tokens + } +} + +/// What a run reports beyond [`super::InferenceResult`]. +/// +/// Each field is `None` on a path that does not know it, never a default that +/// reads as a measurement. The APR and SafeTensors paths report neither today. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub struct RunReport { + /// Why generation ended. + pub finish_reason: Option, + /// The model's context window, from its metadata. + pub context_length: Option, +} + +#[cfg(test)] +mod tests { + use super::*; + + const EOS: u32 = 7; + + /// The case table. Each row names the loop shape it stands for, because the + /// rule is only correct if it is correct for every shape. + #[test] + fn from_decode_case_table() { + let rows: &[(&str, &[u32], usize, FinishReason)] = &[ + // Loops that PUSH the stop token (qwen35, MoE, wgpu). + ( + "pushed stop, under budget", + &[5, 6, EOS], + 8, + FinishReason::Stop, + ), + ( + "pushed stop as the last budgeted token", + &[5, 6, EOS], + 3, + FinishReason::Stop, + ), + // Loops that CONSUME it (dense CPU, dense CUDA). + ( + "consumed stop, under budget", + &[5, 6], + 8, + FinishReason::Stop, + ), + ( + "consumed stop, nothing generated", + &[], + 8, + FinishReason::Stop, + ), + // The cut, on either shape. + ("budget spent, no stop", &[5, 6, 9], 3, FinishReason::Length), + ("zero budget", &[], 0, FinishReason::Length), + // A clamped budget (dense CPU at the context edge): the cut is below + // max_tokens, and it is still a cut. + ("clamped budget spent", &[5, 6], 2, FinishReason::Length), + ]; + for (name, generated, budget, want) in rows { + assert_eq!( + FinishReason::from_decode(generated, &[EOS], *budget), + *want, + "{name}" + ); + } + } + + /// The defect this rule exists for: a clamped cut compared against the + /// REQUESTED budget reads as a stop. Pinned so the call sites cannot regress + /// to passing `max_tokens` where the loop's budget belongs. + #[test] + fn clamped_cut_is_length_only_against_the_budget_it_ran_with() { + let generated = [5, 6, 9, 10]; + let requested = 256; + let ran_with = decode_budget(requested, 60, 64, true); + assert_eq!(ran_with, 4); + assert_eq!( + FinishReason::from_decode(&generated, &[EOS], requested), + FinishReason::Stop + ); + assert_eq!( + FinishReason::from_decode(&generated, &[EOS], ran_with), + FinishReason::Length + ); + } + + #[test] + fn decode_budget_clamps_only_when_the_loop_does() { + assert_eq!(decode_budget(100, 10, 64, true), 54); + assert_eq!(decode_budget(100, 10, 64, false), 100); + assert_eq!(decode_budget(20, 10, 64, true), 20); + assert_eq!(decode_budget(100, 70, 64, true), 0); + } + + #[test] + fn wire_strings_are_the_openai_ones() { + assert_eq!(FinishReason::Stop.as_str(), "stop"); + assert_eq!(FinishReason::Length.to_string(), "length"); + } + + #[test] + fn an_unreported_run_is_none_not_a_default_reason() { + let report = RunReport::default(); + assert_eq!(report.finish_reason, None); + assert_eq!(report.context_length, None); + } + + // ----------------------------------------------------------------------- + // Through `run_inference_report` on a real GGUF file: the dense CPU loop, + // which clamps its budget to the context and consumes its stop token. + // ----------------------------------------------------------------------- + + /// The pygmy GGUF's context window (`build_executable_pygmy_gguf`). + const PYGMY_CONTEXT: usize = 32; + + fn pygmy_file() -> tempfile::NamedTempFile { + use std::io::Write; + let mut file = tempfile::NamedTempFile::with_suffix(".gguf").expect("temp gguf"); + file.write_all(&crate::gguf::test_factory::build_executable_pygmy_gguf()) + .expect("write gguf"); + file.flush().expect("flush gguf"); + file + } + + fn run( + file: &tempfile::NamedTempFile, + prompt: Vec, + max_tokens: usize, + stop: Vec, + ) -> crate::error::Result<(super::super::InferenceResult, RunReport)> { + let config = super::super::InferenceConfig::new(file.path()) + .with_input_tokens(prompt) + .with_max_tokens(max_tokens) + .with_stop_tokens(stop) + .without_gpu(); + super::super::run_inference_report(&config) + } + + /// The case #3718 exists for, at the `apr run` layer: a request whose budget + /// is larger than the room left in the context. The dense CPU loop stops at + /// the context edge, short of `max_tokens`, and that is a cut. + #[test] + fn context_clamped_cut_reports_length_with_the_counts() { + let file = pygmy_file(); + let prompt = vec![1, 2, 3, 4]; + let (result, report) = run(&file, prompt.clone(), 1000, vec![]).expect("run"); + assert_eq!(result.input_token_count, prompt.len()); + assert_eq!(report.context_length, Some(PYGMY_CONTEXT)); + assert_eq!( + result.generated_token_count, + PYGMY_CONTEXT - prompt.len(), + "control: the loop must run to the context edge, or this is not measuring a clamp" + ); + assert_eq!(report.finish_reason, Some(FinishReason::Length)); + } + + #[test] + fn max_tokens_cut_reports_length() { + let file = pygmy_file(); + let (result, report) = run(&file, vec![1, 2, 3, 4], 3, vec![]).expect("run"); + assert_eq!(result.generated_token_count, 3); + assert_eq!(report.finish_reason, Some(FinishReason::Length)); + } + + /// The converse: a stop token under budget is `Stop`. The dense CPU loop + /// consumes it without pushing, so this is the "shorter run" branch. + #[test] + fn a_stop_token_under_budget_reports_stop() { + let file = pygmy_file(); + let prompt = vec![1, 2, 3, 4]; + let (first, _) = run(&file, prompt.clone(), 1, vec![]).expect("probe run"); + let first_token = first.tokens[prompt.len()]; + let (result, report) = run(&file, prompt, 10, vec![first_token]).expect("run"); + assert_eq!( + result.generated_token_count, 0, + "control: greedy is deterministic, so the probed token must stop the run at once" + ); + assert_eq!(report.finish_reason, Some(FinishReason::Stop)); + } + + /// done_when 1's case row: a fixed TEXT prompt through the GGUF's own + /// tokenizer, the path `apr run --prompt` takes. The expected ids are derived + /// by hand from the vocabulary, not from apr's encoder: under a SentencePiece + /// vocabulary that holds `▁Hello`, `▁world` and `!` whole, "Hello world!" is + /// ` ▁Hello ▁world !`, so a conforming tokenizer feeds 4 tokens. + #[test] + fn a_fixed_text_prompt_counts_the_tokens_the_tokenizer_feeds() { + use std::io::Write; + let mut vocab: Vec = ["", "", "", "▁Hello", "▁world", "!"] + .map(String::from) + .to_vec(); + vocab.extend((vocab.len()..32).map(|i| format!(""))); + let pieces: Vec<&str> = vocab.iter().map(String::as_str).collect(); + let bytes = crate::gguf::test_factory::build_executable_pygmy_gguf_with(|b| { + b.add_string("tokenizer.ggml.model", "llama") + .add_string_array("tokenizer.ggml.tokens", &pieces) + .add_u32("tokenizer.ggml.bos_token_id", 1) + .add_u32("tokenizer.ggml.eos_token_id", 2) + }); + let mut file = tempfile::NamedTempFile::with_suffix(".gguf").expect("temp gguf"); + file.write_all(&bytes).expect("write gguf"); + file.flush().expect("flush gguf"); + + let config = super::super::InferenceConfig::new(file.path()) + .with_prompt("Hello world!") + .with_max_tokens(2) + .without_gpu(); + let (result, report) = super::super::run_inference_report(&config).expect("run"); + assert_eq!( + result.tokens.get(..4), + Some(&[1, 3, 4, 5][..]), + "the prompt must be fed as ▁Hello ▁world !" + ); + assert_eq!(result.input_token_count, 4, "prompt_tokens is what was fed"); + assert_eq!(report.context_length, Some(PYGMY_CONTEXT)); + } + + /// A prompt that does not fit is refused, never cut to fit: done_when 3's + /// "or an error". + #[test] + fn a_prompt_longer_than_the_context_is_refused() { + let file = pygmy_file(); + let prompt: Vec = (0..(PYGMY_CONTEXT as u32 + 4)).map(|t| t % 32).collect(); + let err = run(&file, prompt, 4, vec![]).expect_err("an over-long prompt must be refused"); + let msg = err.to_string(); + assert!( + msg.contains(&(PYGMY_CONTEXT + 4).to_string()) + && msg.contains(&PYGMY_CONTEXT.to_string()), + "the refusal must name the prompt length and the context: {msg}" + ); + } +} diff --git a/crates/aprender-serve/src/lib.rs b/crates/aprender-serve/src/lib.rs index beae18143b..ab2bd150dc 100644 --- a/crates/aprender-serve/src/lib.rs +++ b/crates/aprender-serve/src/lib.rs @@ -239,6 +239,9 @@ pub mod chat_template; /// CLI command implementations (extracted for testability) #[cfg(feature = "cli")] pub mod cli; +/// Schema-constrained decoding (#3568): the `TokenConstraint` hook every +/// generation loop calls, and the llguidance engine behind it (`structured-output`). +pub mod constrain; /// GGUF to APR Transformer converter /// /// Converts GGUF models to APR format for fair comparison. diff --git a/crates/aprender-train/src/finetune/instruct_trainer.rs b/crates/aprender-train/src/finetune/instruct_trainer.rs index b1b941d40c..7790657d37 100644 --- a/crates/aprender-train/src/finetune/instruct_trainer.rs +++ b/crates/aprender-train/src/finetune/instruct_trainer.rs @@ -561,7 +561,8 @@ mod tests { assert_eq!(result.epoch_metrics.len(), 2); assert!(result.best_val_loss >= 0.0); - assert!(result.total_time_ms > 0); + // #3703: `total_time_ms` is a truncated wall-clock `as_millis()`, 0 on a host that trains + // this tiny config in under a millisecond, so it is not asserted. The epoch count is. } #[test] diff --git a/crates/aprender-train/src/monitor/inference/tests/inference_monitor_tests.rs b/crates/aprender-train/src/monitor/inference/tests/inference_monitor_tests.rs index 24dff4064f..5f58532d46 100644 --- a/crates/aprender-train/src/monitor/inference/tests/inference_monitor_tests.rs +++ b/crates/aprender-train/src/monitor/inference/tests/inference_monitor_tests.rs @@ -22,10 +22,13 @@ fn test_monotonic_ns_increasing() { fn test_monotonic_ns_non_zero() { use crate::monitor::inference::monotonic_ns; - // The first call might be 0 if called at exactly initialization time - // but subsequent calls should not be 0 + // #3703: the first call in the process initializes the epoch and can read 0 (two back-to-back + // clock reads are equal on a coarse timer). Sleeping BEFORE that call guarded nothing when this + // test ran first. Initialize, then sleep: the sleep, not host speed, puts the second read ahead. + let first = monotonic_ns(); std::thread::sleep(std::time::Duration::from_micros(1)); let ts = monotonic_ns(); + assert!(ts > first, "monotonic_ns should advance across a sleep"); assert!(ts > 0, "monotonic_ns should return a positive value after initialization"); } diff --git a/crates/aprender-verify-ml/src/data/pipeline.rs b/crates/aprender-verify-ml/src/data/pipeline.rs index b69fcb867a..3b17d70b24 100644 --- a/crates/aprender-verify-ml/src/data/pipeline.rs +++ b/crates/aprender-verify-ml/src/data/pipeline.rs @@ -435,7 +435,11 @@ mod tests { assert!(!programs.is_empty()); assert!(stats.total_generated > 0); - assert!(stats.generation_time_ms > 0 || stats.total_generated < 10); + // #3703: what generate() promises, never elapsed wall time. `generation_time_ms` is + // `elapsed.as_millis()`, which reads 0 on a host that generates 10 programs in under a + // millisecond: `> 0` stopped the v0.69.0 clean-room on a fast gx10. The count is the + // invariant, and it holds by construction (`total_generated: all_programs.len()`). + assert_eq!(stats.total_generated, programs.len()); } #[test] diff --git a/docs/audits/impl-PMAT-3715-receipt.md b/docs/audits/impl-PMAT-3715-receipt.md new file mode 100644 index 0000000000..ef38811f66 --- /dev/null +++ b/docs/audits/impl-PMAT-3715-receipt.md @@ -0,0 +1,116 @@ +# PMAT-3715 implementation receipt: `release-readiness-v1` (aprender#3715) + +**Read this before judging scope.** `pmat work status PMAT-3715` prints only the ticket's title. The ticket's scope +is the ISSUE, and the issue's author (the cop, aprender-04, posting as `noahgift`) widened it on the issue itself. +Below, every criterion this diff implements is **quoted verbatim, with its URL**, so each part of the diff can be +checked against the words that asked for it. Nothing below is a paraphrase standing in for a criterion. + +## 1. The issue body's done_when (https://github.com/paiml/aprender/issues/3715) + +> - [ ] **the evidence graph**: `pv extract` ingests the release receipts (per-host model inventory, ladder and matrix receipts from #3712/#3708, dogfood receipt) as nodes: `:Model` (sha256, quant, arch), `:Host`, `:Verb` (run/chat/serve/code), `:ContextRung` (token counts DERIVED from the consumer-traffic records on #3710, never invented), `:Receipt` (verdict, backend, fallback, apr version plus sha) +> - [ ] **the shape** `release-readiness-v1`: focus = every `:Model` in the inventory of every required `:Host`. For each (verb, context rung) exactly one `:Receipt` with `verdict=Pass`, `backend=cuda`, `fallback=false`, `apr_sha = release commit`. A missing, skipped, stale-sha or fallback receipt is a violation naming the cell. +> - [ ] **armed with a red-turning falsifier**: fixtures where each single cell is missing, skipped, falls back, or is stale, and each one turns the gate RED (the ONT-001 rule: a shape that has never gone red isn't armed) +> - [ ] **wired where the decision is made**: autopilot's T-1 step and `check_publish_preflight.sh` both run `pv lint --gate shapes --shape release-readiness-v1` on the release commit's evidence, and a violation STOPs before the tag and refuses at T-4 +> - [ ] **proof**: on the v0.69.0 evidence the shape reports the actual leaks (qwen3-8b lambda, qwen2.5-coder lambda, qwen3moe both hosts, chat/serve/code cells absent). On 0.69.1 it passes with N/N cells named. + +## 2. Scope ADDED on the issue by its author, each implemented here + +**2a. Kernel differential**, https://github.com/paiml/aprender/issues/3715#issuecomment-5763396082: +> **Added cell (operator approved, verbatim "yes, to all", 2026-09-21): kernel differential per GPU architecture.** +> - [ ] The evidence graph gains `:KernelDiffReceipt` nodes: (kernel, sm arch ∈ {sm_89 lambda, sm_121 gx10}, reference ∈ {cpu, cuda-oxide twin}, max_err, bound, verdict). The bound is DERIVED from a measured known-good/known-bad pair (the C14 basis pattern), never picked. +> - [ ] `release-readiness-v1` requires a Pass receipt for every kernel on the release's dispatch path (the kernels `apr parity --per-op` reports for each inventory model) on EACH GPU arch. A kernel that behaves differently on sm_89 and sm_121 is a violation BEFORE any model-level cell is judged. +> - [ ] The falsifier fixture is a kernel receipt that passes on sm_121 and fails on sm_89, and it turns the gate RED. + +→ `release:KernelCell` / `release:KernelDiffReceipt`, shape `release-readiness-v1.kernel`, reported before any cell. The fixture is `a_kernel_green_on_sm_121_and_red_on_sm_89_is_named_on_lambda_before_any_model_cell`. The (kernel, quant) key and the discrete-kernel `index_mismatch`/`tie_margin` rule are the producer's amendments, agreed on #3712 (issuecomment-5763633743). + +**2b. Thinking × context dimensions**, https://github.com/paiml/aprender/issues/3715#issuecomment-5763490841: +> **Dimensions added (operator, verbatim):** "lets ensure this release has MORE than enough tokens for our needs (overdo it), and our verbs are dogfooded, i.e chat with and without thinking, ditto run, ditto code, etc,". Each cell is now (model, host, verb ∈ {run, chat, serve, code}, **thinking ∈ {on, off}**, **context rung ∈ {4k, 8k, 20k, 60k, 148k, the model's declared context_length}**). The declared length is READ from the GGUF, not picked. Thinking ON must close its block and answer correctly, or REPORT an unclosed block as an error, never an empty answer. Full statement on #3710. + +The full statement, https://github.com/paiml/aprender/issues/3710#issuecomment-5763489958, adds: +> Each Qwen3.5 model must be CORRECT … at every rung up to the **model's own declared context length** (GGUF `*.context_length`, read from the file, not picked here). + +→ the cell key (verb, `think-on|off`, rung); `thinkOk` / `answered` on every row; the `declared` rung sized by the model's own `context_length`; a rung above the model's length is not owed. + +**2c. Which thinking modes a model owes**, https://github.com/paiml/aprender/issues/3723#issuecomment-5763858250 (the cop accepted this derivation there): +> - the template honours `enable_thinking` → **both** modes are supported (ON and OFF cells owed) +> - else the generation prompt ALWAYS opens `` → **ON only** (an OFF request is refused by name) +> - else → **OFF only** (an ON request is refused by name) + +→ `inventory[].thinking_modes`, and a `thinkingContradiction` whenever the markers disprove the modes. + +**2d. Tokenizer parity**, #3726's own done_when, https://github.com/paiml/aprender/issues/3726: +> a **tokenizer-parity gate** … that fails on any id mismatch against the pinned llama.cpp … It becomes a required cell per model in the release shape (#3715). + +→ `release:TokenizerCell`, shape `release-readiness-v1.tokenizer`; one mismatching id turns it RED. + +**2e. The memory rule**, operator decision "a", https://github.com/paiml/aprender/issues/3710#issuecomment-5763753907: +> - A (model, host, context-rung) cell is **owed on a host only if it fits** by the declared memory arithmetic (weights + KV at the chosen precision + workspace ≤ that host's GPU memory, measured). +> - Every rung a model declares must be **owed and Pass on at least one required host.** +> - On a host where the cell doesn't fit, a DIFFERENT cell is owed: **an honest pre-load refusal** (non-zero, naming the arithmetic), never an OOM mid-run and never a silent truncation. + +→ `release:RefusalCell` (shape `.refusal`) and `release:RungCoverage` (shape `.coverage`). The rule that a refusal is honest only against TOTAL memory, never free, is the producer's amendment on #3712 (issuecomment-5763795158 thread). + +**2f. No waivers.** #3710 allows "a FAIL for the release unless the operator waives that cell by name", but every agent session posts as `noahgift`, so a comment-based waiver is forgeable by any of them. The cop's ruling, recorded in the body of https://github.com/paiml/aprender/issues/3722: *"(c). #3715 ships with no waiver path."* The diff therefore has no waiver input. + +**2g. The extractor's positive control: #3704's rule, which now applies to this new extractor.** origin/main was +merged in because #3706 (PMAT-3704, merged 52f43da71) ties `pc_extract` to Σ: the merged tree would have been RED +with `release-evidence` implemented and uncontrolled. The criteria of https://github.com/paiml/aprender/issues/3704, +quoted verbatim, as they now bind `release-evidence`: +> - A test that fails when an extractor the shapes gate runs has no `pc_extract` entry — so the next extractor cannot ship without its control. +> - A not-fired parity control makes the gate refuse (as the other four already do), mutation-proved. + +→ `release_evidence::positive_control()` is drawn on EVERY gate run, with no release subject and no file. It runs +a sample cell with one fresh Pass row and a planted cell with no row through the real `build()`. It fires iff the +sample's cell has exactly one fresh row AND the planted cell is still a `release:Cell` node with ZERO rows, which +is to say absence is still materialized for `minCount 1`. `extract_controls()` carries it as `release-evidence`. +#3706's own Σ-tie test (`every_implemented_entity_type_in_sigma_has_an_extract_control_and_it_fires`) passes. +**Mutant, measured:** an extractor that emits only the cells it has rows for → `pv lint contracts --gate shapes` +exits **2**, `shapes: positive control pc_extract.release-evidence did not fire`, `decline: PositiveControlFailed`. +Restored → Pass, with all eight controls `fired`. + +## 3. Points a reviewer may otherwise read as out of scope + +- **"All models, not just Q4_K."** The universe is the host's measured inventory, exactly as done_when 2 says ("focus = every `:Model` in the inventory"). The inventory is defined by the ladder contract's `inventory.patterns`: `*q4_k*.gguf`, `*q4k*.gguf`, `*q4_k*.apr`, `*q4k*.apr` (#3712). That makes it Q4_K by construction. pv does not re-filter by quant, because that would be a second declaration of the universe. Ladder rungs join the inventory only when they claim `cuda`. +- **`receipts.rs` now reads v2.** #3712's `apr-model-ladder-receipt/v2` is the receipt done_when 1 ingests. Without this change, the first committed v2 receipt turns `pv lint --gate shapes` into exit 3 on every PR. A v3 is still refused by name (`a_v2_receipt_joins_its_rungs_like_v1_…`). +- **`--shape` forces arming.** done_when 4's invocation is `pv lint --gate shapes --shape release-readiness-v1`. A named shape that was only "computed and reported" would print its violations and exit 0 at T-1. In ordinary PR lint the family is unarmed and has 0 focus nodes, because there is no release subject. +- **The ont4b shape ratchet goes 9 → 18.** That test hard-codes the count so that a new shape must be named; the nine are named in it. + +## 4. Measured at the PR head + +| check | result | +|---|---| +| `cargo test -p aprender-contracts --lib` (merged head) | 1701 passed, 0 failed | +| `cargo test -p aprender-contracts-cli` (every target) | all green; `ont_release_readiness` has 34 cases | +| `cargo clippy -p aprender-contracts --lib --tests -- -D warnings`, and the same for `-cli` | clean | +| `pv lint contracts` (full) | PASS, 0 errors; no warning from `release-readiness-v1.yaml` | +| `pv extract contracts --check` | fresh (`contracts.nt`, `shapes.ttl` regenerated) | +| `bash scripts/check_ont_ratchet.sh --check` | PASS, after the `--write` restamp | +| `bash scripts/check_explicit_test_commands.sh` | PASS (new `.cmd` 450) | +| mutant: `in: [pass, skip]` on the cell verdict | `a_skipped_cell_is_red_naming_it` FAILS, so it is caught | +| mutant: `fresh()` always true | 3 staleness cases FAIL, so it is caught | +| mutant: emit only cells that have rows | `pv lint contracts --gate shapes` exit 2, `pc_extract.release-evidence did not fire` | +| 0.69.0 proof (`evidence/release/proof-0.69.0/`) | exit 1, 795 findings, naming every leak in done_when 5 | + +## 5. What this PR does NOT do (so it carries `Refs #3715`, not `Closes`) + +- done_when 4's call sites: autopilot T-1 and `check_publish_preflight.sh` T-4. Those are aprender-f0's (#3717/#3708). This PR provides the exact invocation and exit contract they call, agreed on #3712 (issuecomment-5763344947): 0 Pass · 1 Fail · 2 decline · 3 caller error, and every non-zero STOPs. +- done_when 5's second half, "on 0.69.1 it passes with N/N cells named". It needs the producer (#3712 `cells[]`, the memory arithmetic, and kernel and tokenizer receipts) and the model fixes. `release.cell_names` is how it will name the cells. + +## 6. Follow-up delta on release/0.69.1-batch-1 (base ded8a932a): a pure complexity refactor + +**What this delta is, and all it is.** It is not the feature above; the feature was already folded at ded8a932a. +The cop, aprender-04, asked for it on the 0.69.1 critical path. guard_tree's complexity ratchet was RED on the +batch against main a9502d992: +`RED NEW crates/aprender-contracts/src/lint/shapes_gate.rs::run_shapes_gate_with cyclomatic 13 cognitive 28` +(the cognitive limit is 25). The fix is a **pure refactor with no behaviour change**: +- `run_shapes_gate_with` delegates to `prepare`, `order_by_family`, `needs_receipts`, `verdict_of`, `by_shape` and `by_entity_type`. +- `prepare` keeps the original order of answers: collect-shapes error → no shapes → `armed_shapes` error → the family selection error. +- The verdict rule (violations → Fail; warnings alone → Unknown{Warn}; else Pass) and both report maps are moved byte for byte. + +| measured on this delta | result | +|---|---| +| `bash scripts/check_complexity_ratchet.sh` | `run_shapes_gate_with` no longer listed. rc stays 1 **only** for `crates/aprender-serve/src/constrain/tests.rs::generate_intent` (cyclomatic 13, cognitive 27), which is not in this diff and not this ticket's | +| `cargo test -p aprender-contracts --lib` | 1701 passed, 0 failed | +| `pv lint contracts --gate shapes` | Pass, 8/8 `pc_extract` fired (release-evidence included) | +| `cargo test -p aprender-contracts-cli --test ont_release_readiness --test ont4b_shapes_gate --test ont4c1_model_receipts --test ont4c3_parity_receipts` | 34 + 11 + 12 + 10 passed | +| `cargo clippy -p aprender-contracts --lib --tests -- -D warnings` | clean | diff --git a/docs/audits/impl-PMAT-3792-receipt.md b/docs/audits/impl-PMAT-3792-receipt.md new file mode 100644 index 0000000000..be55f740a2 --- /dev/null +++ b/docs/audits/impl-PMAT-3792-receipt.md @@ -0,0 +1,24 @@ +# PMAT-3792 implementation receipt: a pure refactor of `run_shapes_gate_with` (#3792) + +**This ticket IS the refactor.** It is not #3715's feature: that was folded into release/0.69.1-batch-1 at +ded8a932a. The diff against ded8a932a is only this refactor, its fragment and this receipt. + +The cop's request, verbatim: "Please land a pure-refactor commit on a branch OFF release/0.69.1-batch-1 (not your old branch) that brings it under threshold with no behaviour change. Proof: `bash scripts/check_complexity_ratchet.sh` rc 0, `cargo test -p aprender-contracts --lib` green, and `pv lint contracts --gate shapes` 8/8 controls fired." + +## What changed (no behaviour change) +`crates/aprender-contracts/src/lint/shapes_gate.rs::run_shapes_gate_with` (cyclomatic 13, cognitive 28; the +limit is 25) now delegates to: +- **`prepare`**: collect shapes → none (NoShapes) → the `armed_shapes` baseline error → `--shape` family selection. The order of answers is the original one, and a named family is armed as before. +- **`order_by_family`**: the `--shape` report ordering, moved as is. +- **`needs_receipts`**: the `resolves: receipt` test, moved as is. +- **`verdict_of`**: violations → Fail, warnings alone → Unknown{Warn}, else Pass. This is the original if/else. +- **`by_shape`** and **`by_entity_type`**: the two report maps, moved as is (including the ONT-4c3 comment). + +## Measured at 2a40dda85 +| check | result | +|---|---| +| `bash scripts/check_complexity_ratchet.sh` | `run_shapes_gate_with` is no longer RED. rc stays 1 only for `crates/aprender-serve/src/constrain/tests.rs::generate_intent` (cyclomatic 13, cognitive 27), which is not in this diff and not this ticket's | +| `cargo test -p aprender-contracts --lib` | 1701 passed, 0 failed | +| `pv lint contracts --gate shapes` | Pass, 8/8 `pc_extract` fired | +| CLI targets driving this function (`ont_release_readiness`, `ont4b_shapes_gate`, `ont4c1_model_receipts`, `ont4c3_parity_receipts`) | 34 / 11 / 12 / 10 passed | +| `cargo clippy -p aprender-contracts --lib --tests -- -D warnings` | clean | diff --git a/docs/audits/pmat-3703-timing-sweep.md b/docs/audits/pmat-3703-timing-sweep.md new file mode 100644 index 0000000000..180899d28f --- /dev/null +++ b/docs/audits/pmat-3703-timing-sweep.md @@ -0,0 +1,164 @@ +# PMAT-3703: the timing-assertion sweep, every remaining hit listed with its reason + +This is the listing #3703 done_when 2 asks for ("each hit fixed or listed with its reason"), as the issue author scoped it on 2026-09-21: the slow-host `elapsed < N` class is the named remainder in #3705. It is a **snapshot** of the tree at `8c08dd8dd`, the head the quorum judged. Line numbers drift; regenerate it with the commands below. What is *enforced* is `scripts/check_no_timing_in_required.sh` PART 4 and its ledger `scripts/wallclock_assert_baseline.txt` (101 hits, 32 texts), not this file. + +Scope: `git ls-files crates/*/src/*.rs`, the `--lib` surface that every required lane and the clean-room B2 gate run. Comment lines are excluded. + +| Class | Where | Count | +|---|---|---| +| Integer duration asserted `> 0` / `>= 1` / `!= 0` (the #3703 shape) | 3 FIXED here + 101 in the PART 4 ledger with reasons | 104 | +| An upper bound on a duration (`elapsed < N`), the slow-host twin | §1 below, reason `#3705` | 58 | +| Any other lower bound on a duration (`elapsed >= N`, `> Duration::ZERO`, `as_nanos() > 0`, `> 0.0`) | §2 below, each with a reason | 45 | +| `monotonic_ns()` read at epoch initialization | FIXED here (inference_monitor_tests.rs) | 1 | + +## Regenerate + +```bash +UB='assert[a-z_]*!\(.*(\.elapsed\(\)|\belapsed[a-z_]*\b|\.as_millis\(\)|\.as_micros\(\)|\.as_nanos\(\)|\.as_secs(_f64|_f32)?\(\)|\b[a-z_]*(_ms|_us|_ns|_secs)\b)[[:space:]]*<=?[[:space:]]*[0-9A-Za-z(]' +LB='assert[a-z_]*!\(.*(\.elapsed\(\)|\belapsed[a-z_]*\b|\.as_millis\(\)|\.as_micros\(\)|\.as_nanos\(\)|\.as_secs(_f64|_f32)?\(\))[[:space:]]*>=?[[:space:]]*[0-9A-Za-z(]' +WRE=$(sed -n "s/^WALLCLOCK_RE='\(.*\)'$/\1/p" scripts/check_no_timing_in_required.sh) # PART 4's own pattern +git ls-files -z -- 'crates/*/src/*.rs' | xargs -0 grep -nHE "$UB" | grep -vE '^[^:]+:[0-9]+:[[:space:]]*//' # §1 +git ls-files -z -- 'crates/*/src/*.rs' | xargs -0 grep -nHE "$LB" | grep -vE '^[^:]+:[0-9]+:[[:space:]]*//' | grep -vE "$WRE" # §2 +``` + +## §1: upper bounds, the slow-host class (58). Reason for every line: `#3705` + +Each holds on a fast idle host and can fail on a slow or loaded one. Many are not wall-clock at all (a parsed value, a constructed Duration within a tolerance, a configured limit). Classifying them one by one is #3705's done_when, not this row's. + +| # | Site | Assertion | +|---|---|---| +| 1 | `crates/aprender-cbtop/src/incremental_snapshot/tests.rs:159` | `assert!(snapshot.timestamp_ns >= 1200 && snapshot.timestamp_ns <= 1700);` | +| 2 | `crates/aprender-cgp/src/analysis/compete.rs:171` | `assert!(result.wall_time_ms < 1000.0);` | +| 3 | `crates/aprender-cgp/src/doctor.rs:406` | `assert!(elapsed.as_secs() < 2, "doctor checks took {:?}", elapsed);` | +| 4 | `crates/aprender-compute/src/brick/tests/phases/phase11_profiling.rs:26` | `assert!(avg_ns < 50.0, "cpu_cycles() overhead should be < 50ns, got {:.1}ns", avg_ns);` | +| 5 | `crates/aprender-compute/src/brick/tests/phases/phase11_profiling.rs:104` | `assert!(avg_ns < 20.0, "cached_nanos() overhead should be < 20ns, got {:.1}ns", avg_ns);` | +| 6 | `crates/aprender-core/src/audio/mod.rs:171` | `assert!(audio.duration_ms < 1);` | +| 7 | `crates/aprender-core/src/automl/tuner_tests.rs:47` | `assert!(result.elapsed.as_secs() <= 3);` | +| 8 | `crates/aprender-core/src/cache/tests_eviction.rs:326` | `assert!(age.as_secs() < 1);` | +| 9 | `crates/aprender-core/src/citl/metrics_tests_tracker.rs:289` | `assert!(duration.as_secs() < 10);` | +| 10 | `crates/aprender-core/src/demo/demo_tests.rs:75` | `assert!(metrics.load_time_ms < 5000);` | +| 11 | `crates/aprender-core/src/demo/demo_tests.rs:92` | `assert!(metrics.first_token_ms < 2000);` | +| 12 | `crates/aprender-core/src/qa/robustness_tests.rs:123` | `assert!(start.elapsed().as_millis() < 1000);` | +| 13 | `crates/aprender-core/src/wasm/wasm_tests.rs:43` | `assert!(estimated_load_ms < 500);` | +| 14 | `crates/aprender-orchestrate/src/falsification/invariants_tests.rs:102` | `assert!(result.duration_ms < 60000); // Less than 1 minute` | +| 15 | `crates/aprender-orchestrate/src/oracle/cookbook/recipes.rs:1011` | `assert!(overlap_ms < chunk_size_ms);` | +| 16 | `crates/aprender-orchestrate/src/serve/banco/state_tests.rs:47` | `assert!(health.uptime_secs < 2);` | +| 17 | `crates/aprender-present-core/src/animation.rs:1405` | `assert!(eased.elapsed <= eased.duration);` | +| 18 | `crates/aprender-present-terminal/src/tools/bench.rs:1141` | `assert!(stats.max_us <= 199);` | +| 19 | `crates/aprender-profile/src/analysis/anti_pattern.rs:600` | `assert!(*interval_ms < 10, "Interval should be < 10ms, got {}", interval_ms);` | +| 20 | `crates/aprender-profile/src/brick_tracer.rs:965` | `assert!(result.duration_us < 1000000); // Should be fast` | +| 21 | `crates/aprender-profile/src/metrics/histogram.rs:555` | `assert!(h.created_at().elapsed().as_secs() < 1);` | +| 22 | `crates/aprender-qa-runner/src/dimensional_check_tests.rs:101` | `assert!(result.duration_ms < 5000, "should complete quickly");` | +| 23 | `crates/aprender-serve/src/api/tests/gpu_batch_02.rs:293` | `assert!(low_latency.window_ms < default_config.window_ms);` | +| 24 | `crates/aprender-serve/src/api/tests/imp_137a.rs:446` | `assert!(elapsed < 10.0, "IMP-140b: Elapsed should be small (< 10s)");` | +| 25 | `crates/aprender-serve/src/api/tests/partial_batch.rs:164` | `assert!(stats.avg_wait_ms < 50.0);` | +| 26 | `crates/aprender-serve/src/api/tests/tests_19.rs:34` | `assert!(config.window_ms <= 10); // Low latency = short window` | +| 27 | `crates/aprender-serve/src/api/tests/tests_23.rs:24` | `assert!(config.window_ms < 100);` | +| 28 | `crates/aprender-serve/src/api/tests/tests_25.rs:97` | `assert!(result.avg_latency_ms < 10.0);` | +| 29 | `crates/aprender-serve/src/bench/statistics_measurement_protocol.rs:110` | `assert!(stats.p50.as_millis() >= 49 && stats.p50.as_millis() <= 51);` | +| 30 | `crates/aprender-serve/src/bench/statistics_measurement_protocol.rs:112` | `assert!(stats.p90.as_millis() >= 89 && stats.p90.as_millis() <= 91);` | +| 31 | `crates/aprender-serve/src/bench/statistics_measurement_protocol.rs:114` | `assert!(stats.p99.as_millis() >= 98 && stats.p99.as_millis() <= 100);` | +| 32 | `crates/aprender-serve/src/bench/tests_distributed_bench.rs:30` | `assert!(gather_1kb.latency_us < reduce_1kb.latency_us);` | +| 33 | `crates/aprender-serve/src/bench/tests_distributed_bench_02.rs:208` | `assert!(gather_1kb.latency_us < reduce_1kb.latency_us);` | +| 34 | `crates/aprender-serve/src/bench/tests_dynamic_sampler.rs:185` | `assert!(metrics.median_ms > 11.0 && metrics.median_ms < 13.0);` | +| 35 | `crates/aprender-serve/src/bench/tests_dynamic_sampler.rs:187` | `assert!(metrics.std_dev_ms < 5.0);` | +| 36 | `crates/aprender-serve/src/bench/tests_dynamic_sampler.rs:425` | `assert!(result.max_hol_blocking_ms < 500.0);` | +| 37 | `crates/aprender-serve/src/brick/profiler_basic_profiling.rs:41` | `assert!(stats.min_us <= stats.avg_us);` | +| 38 | `crates/aprender-serve/src/brick/profiler_basic_profiling.rs:42` | `assert!(stats.avg_us <= stats.max_us);` | +| 39 | `crates/aprender-serve/src/brick/tests_f083_timing_f084.rs:12` | `assert!(elapsed < 500_000, "Timing should not drift too much");` | +| 40 | `crates/aprender-serve/src/gguf/batch_scheduler_tests.rs:180` | `assert!(config.timeout_ms < 50); // Shorter than default` | +| 41 | `crates/aprender-serve/src/gguf/inference_types_tests_dispatch_metrics.rs:144` | `assert!(elapsed < 1.0);` | +| 42 | `crates/aprender-serve/src/gguf/tests/parity035a_chunked.rs:215` | `assert!(stats.ttft_ms < 500.0, "TTFT should be < 500ms");` | +| 43 | `crates/aprender-serve/src/gpu/tests/quantized_dot_matvec.rs:342` | `assert!(delay.as_secs() <= 2);` | +| 44 | `crates/aprender-serve/src/scheduler/tests_chunked_prefill.rs:125` | `assert!(wait.as_millis() < 100);` | +| 45 | `crates/aprender-serve/src/serve_state_predict.rs:175` | `assert!(response.latency_ms < 10.0); // Should be sub-10ms` | +| 46 | `crates/aprender-serve/src/serve_state_predict.rs:224` | `assert!(response.total_latency_ms < 10.0);` | +| 47 | `crates/aprender-test-lib/src/audio_quality/silence.rs:123` | `assert!(report.total_silence_secs < f64::EPSILON);` | +| 48 | `crates/aprender-test-lib/src/av_sync/comparison.rs:216` | `assert!(report.max_delta_ms < f64::EPSILON);` | +| 49 | `crates/aprender-test-lib/src/av_sync/detection.rs:377` | `assert!(pair[0].time_secs <= pair[1].time_secs);` | +| 50 | `crates/aprender-test-lib/src/brick/deterministic.rs:1764` | `assert!(duration.as_secs_f64() > 0.99 && duration.as_secs_f64() < 1.01);` | +| 51 | `crates/aprender-test-lib/src/browser.rs:5096` | `assert!(report.timestamp_ms <= after);` | +| 52 | `crates/aprender-test-lib/src/driver.rs:1228` | `assert!(duration.as_secs() < 1);` | +| 53 | `crates/aprender-test-lib/src/tui/compute_block.rs:556` | `assert!(duration.as_micros() < 1_000_000);` | +| 54 | `crates/aprender-test-lib/src/websocket.rs:1098` | `assert!(elapsed < 1000); // Should be very small, just created` | +| 55 | `crates/aprender-train/src/dashboard/wasm/tests.rs:111` | `assert!(loading_time.as_millis() < 1000, "wasm_bench: {loading_time:?}");` | +| 56 | `crates/aprender-train/src/io/load.rs:497` | `assert!(loading_time.as_millis() < 5000, "load_bench: {loading_time:?}");` | +| 57 | `crates/aprender-train/src/monitor/tui/state.rs:739` | `assert!(rem_ms > 5000 && rem_ms < 30000);` | +| 58 | `crates/aprender-zram-core/src/samefill.rs:401` | `assert!(elapsed.as_millis() < 1000);` | + +## §2: other lower bounds on a duration (45), each with its reason + +### A. a real measurement, but a sleep of at least the bound runs inside the measured span, so a monotonic clock cannot read below it (10) + +| Site | Assertion | +|---|---| +| `crates/aprender-core/src/automl/tuner_tests.rs:150` | `assert!(budget.elapsed() > Duration::ZERO);` | +| `crates/aprender-core/src/demo/reliable/tests.rs:113` | `assert!(timer.elapsed() >= Duration::from_millis(20));` | +| `crates/aprender-profile/src/metrics/histogram.rs:652` | `assert!(elapsed > 0.0);` | +| `crates/aprender-test-lib/src/perf/span.rs:368` | `assert!(elapsed > 0);` | +| `crates/aprender-test-lib/src/perf/trace.rs:417` | `assert!(duration.unwrap().as_millis() >= 10);` | +| `crates/aprender-test-lib/src/performance.rs:1481` | `assert!(elapsed >= 10);` | +| `crates/aprender-test-lib/src/pixel_coverage/wasm_demo.rs:1650` | `assert!(elapsed.as_millis() >= 10);` | +| `crates/aprender-test-lib/src/wait.rs:1055` | `assert!(start.elapsed() >= Duration::from_millis(50));` | +| `crates/aprender-train/src/hf_pipeline/trainer/tests.rs:273` | `assert!(state.elapsed().as_millis() >= 4);` | +| `crates/aprender-verify-ml/src/ml/experiment.rs:665` | `assert!(experiment.total_duration.as_millis() >= 10);` | + +### B. a tautology (`>= 0`, `>= 0.0`, `>= Duration::ZERO`) that no clock can fail (8) + +| Site | Assertion | +|---|---| +| `crates/aprender-core/src/demo/reliable/tests.rs:263` | `assert!(timer.elapsed() >= Duration::ZERO);` | +| `crates/aprender-core/src/demo/reliable/tests.rs:349` | `assert!(timer.elapsed() >= Duration::ZERO);` | +| `crates/aprender-orchestrate/src/oracle/rag/profiling.rs:695` | `assert!(elapsed >= Duration::ZERO);` | +| `crates/aprender-serve/src/api/tests/imp_137a.rs:445` | `assert!(elapsed >= 0.0, "IMP-140b: Elapsed should be >= 0");` | +| `crates/aprender-serve/src/gguf/inference_types_config_default.rs:409` | `assert!(elapsed >= 0.0);` | +| `crates/aprender-serve/src/gguf/inference_types_tests_dispatch_metrics.rs:143` | `assert!(elapsed >= 0.0);` | +| `crates/aprender-train/src/hf_pipeline/trainer/tests.rs:302` | `assert!(eta.as_secs_f32() >= 0.0);` | +| `crates/aprender-train/src/train/trainer/train_loop/tests.rs:86` | `assert!(result.elapsed_secs >= 0.0);` | + +### C. not a clock reading: the Duration is constructed, computed (ETA, back-off), or comes from a synthetic clock (8) + +| Site | Assertion | +|---|---| +| `crates/aprender-serve/src/bench/statistics_measurement_protocol.rs:110` | `assert!(stats.p50.as_millis() >= 49 && stats.p50.as_millis() <= 51);` | +| `crates/aprender-serve/src/bench/statistics_measurement_protocol.rs:112` | `assert!(stats.p90.as_millis() >= 89 && stats.p90.as_millis() <= 91);` | +| `crates/aprender-serve/src/bench/statistics_measurement_protocol.rs:114` | `assert!(stats.p99.as_millis() >= 98 && stats.p99.as_millis() <= 100);` | +| `crates/aprender-serve/src/gpu/tests/quantized_dot_matvec.rs:331` | `assert!(delay.as_millis() >= 200);` | +| `crates/aprender-test-lib/src/brick/deterministic.rs:1764` | `assert!(duration.as_secs_f64() > 0.99 && duration.as_secs_f64() < 1.01);` | +| `crates/aprender-train/src/ecosystem/batuta/tests.rs:170` | `assert!(adjusted.as_secs() >= 4500);` | +| `crates/aprender-train/src/ecosystem/batuta/tests.rs:180` | `assert!(adjusted.as_secs() > base);` | +| `crates/aprender-train/src/ecosystem/batuta/tests.rs:189` | `assert!(adjusted.as_secs() >= 7200);` | + +### D. a nanosecond or float reading over real work. It does not truncate to milliseconds, and reads 0 only if the work finishes inside one tick of the monotonic clock. The closest to that is trueno-ublk adaptive_batch.rs:494, which times two `add_items` calls: tens of nanoseconds of work (17) + +| Site | Assertion | +|---|---| +| `crates/aprender-core/src/automl/tuner_tests_callbacks.rs:56` | `assert!(result.elapsed.as_nanos() > 0);` | +| `crates/aprender-core/src/optim/tests/advanced.rs:246` | `assert!(result.elapsed_time.as_nanos() > 0);` | +| `crates/aprender-core/src/optim/tests/advanced.rs:422` | `assert!(result.elapsed_time.as_nanos() > 0);` | +| `crates/aprender-core/src/optim/tests/advanced_coordinate_descent.rs:312` | `assert!(result.elapsed_time.as_nanos() > 0);` | +| `crates/aprender-core/src/optim/tests/advanced_interior_point.rs:141` | `assert!(result.elapsed_time.as_nanos() > 0);` | +| `crates/aprender-core/src/optim/tests/advanced_line_search.rs:394` | `assert!(result.elapsed_time.as_nanos() > 0);` | +| `crates/aprender-core/src/optim/tests/advanced_projected_gradient.rs:207` | `assert!(result.elapsed_time.as_nanos() > 0);` | +| `crates/aprender-core/src/optim/tests/advanced_projected_gradient.rs:393` | `assert!(result.elapsed_time.as_nanos() > 0);` | +| `crates/aprender-core/src/optim/tests/core_lbfgs.rs:147` | `assert!(result.elapsed_time.as_nanos() > 0);` | +| `crates/aprender-distribute/src/executor/simd.rs:937` | `assert!(result.duration().as_nanos() > 0);` | +| `crates/aprender-gpu/src/monitor/compute/tests.rs:187` | `assert!(kernel.elapsed_ms > 0.0);` | +| `crates/aprender-serve/src/quantize/tests/tests_25.rs:273` | `assert!(simd_time.as_nanos() > 0);` | +| `crates/aprender-test-lib/src/perf/metrics.rs:507` | `assert!(metrics.duration.as_nanos() > 0);` | +| `crates/aprender-train/src/train/trainer/train_loop/tests.rs:38` | `assert!(result.elapsed_secs > 0.0);` | +| `crates/aprender-zram-core/src/benchmark.rs:247` | `assert!(result.compress_time.as_nanos() > 0);` | +| `crates/aprender-zram-core/src/benchmark.rs:248` | `assert!(result.decompress_time.as_nanos() > 0);` | +| `crates/aprender-zram/bins/trueno-ublk/src/perf/tenx/adaptive_batch.rs:494` | `assert!(duration.as_nanos() > 0);` | + +### E. #3705 (a hidden UPPER bound: `remaining >= 999 s` of a 1000 s budget fails if the host stalls over 1 s between on_start and remaining) (1) + +| Site | Assertion | +|---|---| +| `crates/aprender-core/src/automl/tuner_tests_callbacks.rs:191` | `assert!(remaining.as_secs() >= 999);` | + +### F. `#[ignore]`d, so B2 does not run it (its `elapsed < 500_000` ns upper bound is #3705 material if it is ever re-enabled) (1) + +| Site | Assertion | +|---|---| +| `crates/aprender-serve/src/brick/tests_f083_timing_f084.rs:11` | `assert!(elapsed > 50_000, "Timing should measure > 50µs");` | diff --git a/docs/audits/quorum-PMAT-3543.json b/docs/audits/quorum-PMAT-3543.json new file mode 100644 index 0000000000..14816ddb49 --- /dev/null +++ b/docs/audits/quorum-PMAT-3543.json @@ -0,0 +1,207 @@ +{ + "ticket": "PMAT-3543", + "base": "225b2a9ab", + "base_resolved": "225b2a9ab", + "base_note": "no origin/225b2a9ab exists; judged against the local ref", + "head": "6c8c1e8abace067d33d3b66eb53dc22e5a9b1bd3", + "diff_sha256": "204b599f2860e4bd858ed22add03c6c2f83143cee426b12245c52901a0642a11", + "width": 3, + "executor": "agy", + "prompt_mode": "inline", + "prompt_bytes": 45093, + "author": { + "model": "claude-opus-5", + "family": "claude", + "source": "flag" + }, + "agreed": false, + "lanes": [ + { + "lane": 1, + "status": "SUCCESS", + "verdict": "PASS", + "summary": "The diff perfectly implements the requirements of PMAT-3543. It makes the version gate phase-aware by distinguishing between `pre-publish` (asserting `version-unpublished` PASS when absent) and `post-publish` (asserting `version-published` PASS when present AND installable via `cargo install --locked`). It properly resolves Defect 2 by fixing the banner to print the actual `DOGFOOD_PHASE`. Finally, it introduces a rigorous falsifier script that verifies both branches for both phases along with mutants to ensure the gates aren't vacuous.", + "findings": [], + "raw_bytes": 2334, + "err_bytes": 0, + "envelope_status": "SUCCESS", + "verdict_source": "structured_output", + "grounding_check": "parity", + "model": "gemini-3.1-pro-high", + "model_measured": "gemini-3.1-pro-high", + "model_source": "measured", + "family": "gemini", + "role": "independent", + "fallback": { + "judged_by": "gemini-3.1-pro-high", + "exhausted": false, + "attempts": [ + { + "model": "gemini-3.1-pro-high", + "family": "gemini", + "outcome": "answered" + } + ], + "skipped": [] + } + }, + { + "lane": 2, + "status": "NO-VERDICT", + "verdict": "NO-VERDICT", + "summary": "", + "findings": [], + "raw_bytes": 1143, + "err_bytes": 100, + "envelope_status": "SUCCESS", + "transport_status": "SUCCESS", + "transport_error": "", + "salvageable": "/home/noah/.gemini/antigravity-cli/brain/fa50f4d0-6d69-466c-947c-5477ecdd0f03 (234055 bytes)", + "carried_verdict": "", + "blind_reason": "", + "agy_exit": "", + "model": "gemini-3.1-pro-high", + "model_measured": "gemini-3.1-pro-high", + "model_source": "measured", + "family": "gemini", + "role": "independent", + "fallback": { + "judged_by": "gemini-3.1-pro-high", + "exhausted": false, + "attempts": [ + { + "model": "gemini-3.1-pro-high", + "family": "gemini", + "outcome": "answered" + } + ], + "skipped": [ + { + "model": "gpt-oss-120b-medium", + "why": "no capacity: 503 on two pre-check probes (Eligibility check failed: UNAVAILABLE (code 503): The service is currently unavailable.)" + } + ] + } + }, + { + "lane": 3, + "status": "SUCCESS", + "verdict": "PASS", + "summary": "I have reviewed the diff against the PMAT-3543 ticket requirements and found that all acceptance criteria are met. \n\n1. `version-unpublished` is now phase-aware. The diff implements `mark_version_row` which correctly asserts that the version is ABSENT during `pre-publish` and PRESENT during `post-publish`.\n2. The `cargo install` requirement for `post-publish` is fully implemented in `published_install_check` with the exact flags specified (`--version = --locked --root --target-dir `) and token unset. It also asserts that at least one binary is installed and that every binary reports the correct version.\n3. The banner logic is correctly updated via `phase_banner` to accurately output the `$DOGFOOD_PHASE`.\n4. The falsifier (`scripts/check_dogfood_version_row.sh`) is fully implemented with tests covering all expected mutant rows, verifying the new phase-aware behavior comprehensively.\n5. The `full` phase's existing dry-run functionality is correctly preserved through the `else` block and `elif grep ...` fallthrough.\n\nNo regressions or weakening of gates were found. The claims in the receipt accurately match the code changes.", + "findings": [], + "raw_bytes": 3668, + "err_bytes": 0, + "envelope_status": "SUCCESS", + "verdict_source": "structured_output", + "grounding_check": "parity", + "model": "gemini-3.1-pro-low", + "model_measured": "gemini-3.1-pro-low", + "model_source": "measured", + "family": "gemini", + "role": "independent", + "fallback": { + "judged_by": "gemini-3.1-pro-low", + "exhausted": false, + "attempts": [ + { + "model": "gemini-3.1-pro-low", + "family": "gemini", + "outcome": "answered" + } + ], + "skipped": [] + } + } + ], + "dissent": [ + { + "lane": 2, + "verdict": "NO-VERDICT", + "summary": "", + "findings": [] + } + ], + "dedup": [], + "uncovered": [], + "coverage_source": "lanes", + "partial": true, + "partial_reasons": [ + "lane models: only 2 distinct model ids across 3 lanes (gemini-3.1-pro-high, gemini-3.1-pro-low) — lanes sharing an id are resamples, not independent reviewers (PMAT-125)", + "lane 2: no verdict object found in lane file (status=SUCCESS) salvageable=/home/noah/.gemini/antigravity-cli/brain/fa50f4d0-6d69-466c-947c-5477ecdd0f03 (234055 bytes) [UNREVIEWED CLAIM MATERIAL — never a PASS]", + "lane 2: judged by gemini-3.1-pro-high after falling through gpt-oss-120b-medium (skipped: no capacity: 503 on two pre-check probes (Eligibility check failed: UNAVAILABLE (code 503): The service is currently unavailable.)) (PMAT-321)" + ], + "fallback": { + "same_family_width": 1, + "chain": [ + { + "model": "gemini-3.1-pro-high", + "family": "gemini", + "disposition": "configured" + }, + { + "model": "gpt-oss-120b-medium", + "family": "openai", + "disposition": "configured" + }, + { + "model": "gemini-3.1-pro-low", + "family": "gemini", + "disposition": "configured" + }, + { + "model": "qwen3.5", + "family": "qwen", + "disposition": "not-run", + "why": "no quorum.local_lane in the config — the aprender lane has no model to load" + }, + { + "model": "claude-opus-4-6-thinking", + "family": "claude", + "disposition": "width", + "why": "same family as the author: at most 1 lane, recorded role width, counted toward no floor (R-15a)" + }, + { + "model": "claude-sonnet-4-6", + "family": "claude", + "disposition": "width", + "why": "same family as the author: at most 1 lane, recorded role width, counted toward no floor (R-15a)" + } + ], + "precheck": [ + { + "family": "gemini", + "model": "gemini-3.1-pro-high", + "probe": 1, + "outcome": "live" + }, + { + "family": "openai", + "model": "gpt-oss-120b-medium", + "probe": 1, + "outcome": "capacity", + "reason": "UNAVAILABLE (code 503): No capacity available for model gpt-oss-120b-medium on the server" + }, + { + "family": "openai", + "model": "gpt-oss-120b-medium", + "probe": 2, + "outcome": "capacity", + "reason": "Eligibility check failed: UNAVAILABLE (code 503): The service is currently unavailable." + } + ], + "prah": { + "source": "install-receipt", + "path": "/home/noah/.claude/skills/paiml-implement/bin/prah" + } + }, + "auto_merge": { + "checked": false, + "was_armed": false, + "disarmed": false, + "note": "no --pr given: nothing to disarm" + }, + "lint": { + "ok": true, + "output": "receipt complete: kind=artifact lanes=3 author=claude-opus-5/claude" + } +} diff --git a/docs/audits/quorum-PMAT-3563.json b/docs/audits/quorum-PMAT-3563.json new file mode 100644 index 0000000000..c0b9e3f22c --- /dev/null +++ b/docs/audits/quorum-PMAT-3563.json @@ -0,0 +1,199 @@ +{ + "ticket": "PMAT-3563", + "base": "52f43da71745eedacde2f86e1903ea81d6fa34ad", + "base_resolved": "52f43da71745eedacde2f86e1903ea81d6fa34ad", + "base_note": "no origin/52f43da71745eedacde2f86e1903ea81d6fa34ad exists; judged against the local ref", + "head": "47658dbabadec0377de5c8829d022ea1a9c00aa3", + "diff_sha256": "01777306f86394e1dc8d9afaf3139409091d9a130093b81207dc0d3442244f75", + "width": 3, + "executor": "agy", + "prompt_mode": "inline", + "prompt_bytes": 28299, + "author": { + "model": "claude-opus-5", + "family": "claude", + "source": "flag" + }, + "agreed": false, + "lanes": [ + { + "lane": 1, + "status": "SUCCESS", + "verdict": "PASS", + "summary": "The diff perfectly implements the requested changes for PMAT-3563, clause (A). The `llama_bin.sh` script correctly resolves the canonical pinned path (`${LLAMA_PIN_SRC_ROOT:-$HOME/src}/llama.cpp-/build/bin/llama-bench`) when nothing is specified in `LLAMA_BENCH_PATH`. It also handles git hash abbreviation lengths accurately (>= 7 hex digits) to support different environments like `gx10` and `lambda`, and prints explicit failure reasons and resolution steps as requested. The test suite (`check_llama_pin.sh`) was meticulously updated with explicit tests and mutants verifying this exact behaviour, and evidence files proving real-world execution on both host machines are properly included. No claims were unbacked, no tests were weakened, and all ticket constraints are fully satisfied.", + "findings": [], + "raw_bytes": 2903, + "err_bytes": 0, + "envelope_status": "SUCCESS", + "verdict_source": "structured_output", + "grounding_check": "parity", + "model": "gemini-3.1-pro-high", + "model_measured": "gemini-3.1-pro-high", + "model_source": "measured", + "family": "gemini", + "role": "independent", + "fallback": { + "judged_by": "gemini-3.1-pro-high", + "exhausted": false, + "attempts": [ + { + "model": "gemini-3.1-pro-high", + "family": "gemini", + "outcome": "answered" + } + ], + "skipped": [] + } + }, + { + "lane": 2, + "status": "NO-VERDICT", + "verdict": "NO-VERDICT", + "summary": "", + "findings": [], + "raw_bytes": 8923, + "err_bytes": 100, + "envelope_status": "SUCCESS", + "transport_status": "SUCCESS", + "transport_error": "", + "salvageable": "/home/noah/.gemini/antigravity-cli/brain/899a9ace-057a-4b3c-910b-bead7b3fb9f5 (27681118 bytes)", + "carried_verdict": "", + "blind_reason": "", + "agy_exit": "", + "model": "gpt-oss-120b-medium", + "model_measured": "gpt-oss-120b-medium", + "model_source": "measured", + "family": "openai", + "role": "independent", + "fallback": { + "judged_by": "gpt-oss-120b-medium", + "exhausted": false, + "attempts": [ + { + "model": "gpt-oss-120b-medium", + "family": "openai", + "outcome": "capacity", + "reason": "Our servers are experiencing high traffic right now, please try again in a minute. (UNAVAILABLE (code 503): No capacity available for model gpt-oss-120b-medium on the server)" + }, + { + "model": "gpt-oss-120b-medium", + "family": "openai", + "outcome": "answered" + } + ], + "skipped": [] + } + }, + { + "lane": 3, + "status": "SUCCESS", + "verdict": "PASS", + "summary": "The diff accurately implements clause 2 (A) of the ticket, splitting the requirements into separate PRs as directed by the cop ruling. It updates `llama_bin.sh` to resolve to the canonical location `${LLAMA_PIN_SRC_ROOT:-$HOME/src}/llama.cpp-` when no binary is named, verifies the build correctly (accounting for different git abbreviation lengths), and includes robust testing of this functionality in `check_llama_pin.sh` with mutants to prevent regressions. The evidence files prove it operates correctly on both lambda and gx10. There is no evidence of it implementing forbidden logic.", + "findings": [], + "raw_bytes": 3152, + "err_bytes": 0, + "envelope_status": "SUCCESS", + "verdict_source": "structured_output", + "model": "gemini-3.1-pro-low", + "model_measured": "gemini-3.1-pro-low", + "model_source": "measured", + "family": "gemini", + "role": "independent", + "fallback": { + "judged_by": "gemini-3.1-pro-low", + "exhausted": false, + "attempts": [ + { + "model": "gemini-3.1-pro-low", + "family": "gemini", + "outcome": "answered" + } + ], + "skipped": [] + } + } + ], + "dissent": [ + { + "lane": 2, + "verdict": "NO-VERDICT", + "summary": "", + "findings": [] + } + ], + "dedup": [], + "uncovered": [], + "coverage_source": "lanes", + "partial": true, + "partial_reasons": [ + "lane 2: no verdict object found in lane file (status=SUCCESS) salvageable=/home/noah/.gemini/antigravity-cli/brain/899a9ace-057a-4b3c-910b-bead7b3fb9f5 (27681118 bytes) [UNREVIEWED CLAIM MATERIAL — never a PASS]", + "lane 3: grounding NOT COMPARED — structured_output holds a verdict but no single verdict object could be read out of .response (3152 raw bytes), so agy's labels stand unchecked for this lane (PMAT-082)", + "lane 2: judged by gpt-oss-120b-medium after falling through gpt-oss-120b-medium (capacity: Our servers are experiencing high traffic right now, please try again in a minute. (UNAVAILABLE (code 503): No capacity available for model gpt-oss-120b-medium on the server)) (PMAT-321)" + ], + "fallback": { + "same_family_width": 1, + "chain": [ + { + "model": "gemini-3.1-pro-high", + "family": "gemini", + "disposition": "configured" + }, + { + "model": "gpt-oss-120b-medium", + "family": "openai", + "disposition": "configured" + }, + { + "model": "gemini-3.1-pro-low", + "family": "gemini", + "disposition": "configured" + }, + { + "model": "qwen3.5", + "family": "qwen", + "disposition": "not-run", + "why": "no quorum.local_lane in the config — the aprender lane has no model to load" + }, + { + "model": "claude-opus-4-6-thinking", + "family": "claude", + "disposition": "width", + "why": "same family as the author: at most 1 lane, recorded role width, counted toward no floor (R-15a)" + }, + { + "model": "claude-sonnet-4-6", + "family": "claude", + "disposition": "width", + "why": "same family as the author: at most 1 lane, recorded role width, counted toward no floor (R-15a)" + } + ], + "precheck": [ + { + "family": "gemini", + "model": "gemini-3.1-pro-high", + "probe": 1, + "outcome": "live" + }, + { + "family": "openai", + "model": "gpt-oss-120b-medium", + "probe": 1, + "outcome": "live" + } + ], + "prah": { + "source": "install-receipt", + "path": "/home/noah/.claude/skills/paiml-implement/bin/prah" + } + }, + "auto_merge": { + "checked": false, + "was_armed": false, + "disarmed": false, + "note": "no --pr given: nothing to disarm" + }, + "lint": { + "ok": true, + "output": "receipt complete: kind=artifact lanes=3 author=claude-opus-5/claude" + } +} diff --git a/docs/audits/quorum-PMAT-3571.json b/docs/audits/quorum-PMAT-3571.json index f60bbc0b13..40b4df6893 100644 --- a/docs/audits/quorum-PMAT-3571.json +++ b/docs/audits/quorum-PMAT-3571.json @@ -3,39 +3,45 @@ "base": "origin/main", "base_resolved": "origin/main", "base_note": "no origin/origin/main exists; judged against the local ref", - "head": "24d49bb1ca3d2636b4997a1a92c2ff712b323686", - "diff_sha256": "c0034ef873e67740c5e67305d98842390edb4cae544c87212a5b4e5f0d4d54f2", + "head": "7e6bc1a50d3d314202d0eefb271e4777c7243a97", + "diff_sha256": "11ab75b50065e70855c0d0da4cca6e82a8e5527729c0c9ff2ad0017ec3b5cbc6", "width": 3, "executor": "agy", "prompt_mode": "inline", - "prompt_bytes": 7598, + "prompt_bytes": 22433, "author": { - "model": "Opus 5 (1M context)", + "model": "claude-opus-5", "family": "claude", - "source": "measured" + "source": "flag" }, - "agreed": true, + "agreed": false, "lanes": [ { "lane": 1, "status": "SUCCESS", "verdict": "PASS", - "summary": "I reviewed the diff against the acceptance criteria for PMAT-3571. The diff implements exactly what was asked: branching in `handler_gpu_completion.rs` to route Qwen3.5 to `Qwen35Model::create_base_model` while maintaining the existing dense path for others. Furthermore, the newly added LOADER evidence strictly obeys the ticket's instruction to not claim end-to-end success, correctly capturing the next tokenizer failure instead. I found no assertions that weaken gates, contradict the ticket, or claim unfounded successes.", + "summary": "The diff precisely implements Unit 1 of the ticket's acceptance criteria. It introduces `zero_layer_refusal` to reject 0-layer transformer stacks at load across all `apr serve` entry points, names the architecture and #3571 in the error, and provides the mandated case row and mutant tests. The implementation correctly limits its scope to Unit 1 (refusal) without prematurely modifying the Qwen35 routing logic reserved for Unit 2. No regressions, weakened gates, or forbidden modifications were found.", "findings": [ { - "claim": "The code correctly branches on `is_qwen35` and calls `Qwen35Model::create_base_model` instead of falling through to the dense `OwnedQuantizedModel` path.", + "claim": "The zero_layer_refusal function is correctly implemented to reject models with 0 transformer layers, explicitly naming the architecture and ticket #3571 in the CliError.", "file": "crates/apr-cli/src/commands/serve/handler_gpu_completion.rs", "grounding": "cited", - "line": 396 + "line": 492 }, { - "claim": "The receipt accurately documents the subsequent BPE tokenizer failure (`Unknown token ''`) and explicitly adheres to the requirement not to claim the serve path is end-to-end green.", - "file": "evidence/serve/3571/LOADER.md", + "claim": "The required case row test ('zero_layers_is_refused_by_name_and_any_layer_is_admitted') and mutant test ('serve_refuses_the_qwen35_base_at_load') are included.", + "file": "crates/apr-cli/src/commands/serve/handler_gpu_completion.rs", + "grounding": "cited", + "line": 463 + }, + { + "claim": "The refusal check is correctly integrated across all specified GGUF, APR, and Safetensors serve entry points, including pre-checking the declared layer count for APR Q4K pool GPU before spawning its thread.", + "file": "crates/apr-cli/src/commands/serve/handlers_include_01.rs", "grounding": "cited", - "line": 34 + "line": 146 } ], - "raw_bytes": 4324, + "raw_bytes": 4046, "err_bytes": 100, "envelope_status": "SUCCESS", "verdict_source": "structured_output", @@ -43,170 +49,193 @@ "model": "gemini-3.1-pro-high", "model_measured": "gemini-3.1-pro-high", "model_source": "measured", - "family": "gemini" + "family": "gemini", + "role": "independent", + "fallback": { + "judged_by": "gemini-3.1-pro-high", + "exhausted": false, + "attempts": [ + { + "model": "gemini-3.1-pro-high", + "family": "gemini", + "outcome": "answered" + } + ], + "skipped": [] + } }, { "lane": 2, "status": "SUCCESS", "verdict": "PASS", - "summary": "PASS: The diff implements PMAT-3571 exactly as specified. In crates/apr-cli/src/commands/serve/handler_gpu_completion.rs, start_gguf_server now checks if the GGUF model architecture is 'qwen35' and dispatches to Qwen35Model::create_base_model instead of falling through to the dense OwnedQuantizedModel::from_mapped path. No gates were weakened, no extra scope was added, and the change compiles and passes tests cleanly.", - "findings": [ - { - "claim": "The Qwen3.5 hybrid architecture check routes to Qwen35Model::create_base_model instead of falling through to OwnedQuantizedModel::from_mapped, matching the acceptance criteria of PMAT-3571.", - "file": "crates/apr-cli/src/commands/serve/handler_gpu_completion.rs", - "fix": "N/A", - "grounding": "cited", - "line": 396 - }, - { - "claim": "CARGO_TARGET_DIR=/tmp/target_rel3571 cargo check -p apr-cli --features inference,cuda compiles cleanly without errors.", - "command": "CARGO_TARGET_DIR=/tmp/target_rel3571 cargo check -p apr-cli --features inference,cuda", - "file": "crates/apr-cli/src/commands/serve/handler_gpu_completion.rs", - "fix": "N/A", - "grounding": "measured", - "line": 387, - "output": "Finished dev profile [unoptimized + debuginfo] target(s) in 1m 00s" - }, - { - "claim": "PMAT-3571 task entry and roadmap entries define the exact single-layer scope and acceptance criteria.", - "file": "docs/roadmaps/entries/PMAT-3571.yaml", - "fix": "N/A", - "grounding": "cited", - "line": 1 - }, - { - "claim": "Evidence file LOADER.md accurately documents before and after state for apr serve on Qwen3.5 without overclaiming end-to-end readiness.", - "file": "evidence/serve/3571/LOADER.md", - "fix": "N/A", - "grounding": "cited", - "line": 1 - } - ], - "raw_bytes": 5920, + "summary": "The diff accurately implements the PMAT-3571 requirement. It ensures that any model stack resolving to 0 transformer layers is refused at load on all serve paths, preventing the Qwen3.5 hybrid model from incorrectly proceeding through the generic serve path.\n\nSpecifically, it checks:\n1. `build_serve_model` catches the 0-layer architecture, covering `start_gguf_server`, `start_gguf_server_cuda` and `try_start_wgpu_backend`.\n2. SafeTensors & APR handlers manually perform `zero_layer_refusal` as requested before finalizing initialization, catching the problem early.\n3. It names the architecture and references #3571 in the refusal message, along with the required `apr chat` hint for `qwen35`.\n4. It implements the two requested test criteria: the case row table `zero_layers_is_refused_by_name_and_any_layer_is_admitted` and the mutant test `serve_refuses_the_qwen35_base_at_load`.\n5. All sibling endpoints specified in the issue are accounted for and adequately guarded.\n\nNo refute found.", + "findings": [], + "raw_bytes": 3385, "err_bytes": 100, "envelope_status": "SUCCESS", "verdict_source": "structured_output", "grounding_check": "parity", - "model": "gemini-3.7-flash-high", - "model_measured": "gemini-3.7-flash-high", + "model": "gemini-3.1-pro-low", + "model_measured": "gemini-3.1-pro-low", "model_source": "measured", - "family": "gemini" + "family": "gemini", + "role": "independent", + "fallback": { + "judged_by": "gemini-3.1-pro-low", + "exhausted": false, + "attempts": [ + { + "model": "gemini-3.1-pro-low", + "family": "gemini", + "outcome": "answered" + } + ], + "skipped": [] + } }, { "lane": 3, - "status": "SUCCESS", - "verdict": "PASS", - "summary": "Lane 3 judgment for PMAT-3571: PASS. The diff correctly routes Qwen3.5 hybrid models in `apr serve` to `Qwen35Model::create_base_model` in `handler_gpu_completion.rs` lines 396-408, fulfilling the ticket acceptance criteria. All repository gates, roadmap verification scripts, and `cargo check -p apr-cli` pass cleanly with no gate weakening or unrequested changes.", - "findings": [ - { - "claim": "`crates/apr-cli/src/commands/serve/handler_gpu_completion.rs` branches on `is_qwen35` and invokes `realizar::gguf::forward_qwen35::Qwen35Model::create_base_model`, resolving the loader defect where Qwen3.5 hybrid models failed to load in `apr serve`.", - "command": "cargo check -p apr-cli", - "file": "crates/apr-cli/src/commands/serve/handler_gpu_completion.rs", - "fix": "Added conditional check for architecture 'qwen35' to build Qwen35Model base model instead of falling through to OwnedQuantizedModel::from_mapped.", - "grounding": "measured", - "line": 396, - "output": "Finished `dev` profile [unoptimized + debuginfo] target(s) in 1m 10s" - }, - { - "claim": "`docs/roadmaps/entries/PMAT-3571.yaml` and `docs/roadmaps/roadmap.yaml` correctly document task PMAT-3571 acceptance criteria and scope boundary (Layer 1 loading fix).", - "command": "bash scripts/check_roadmap_diff_additive.sh && bash scripts/check_roadmap_ids_unique.sh && bash scripts/check_roadmap_sorted.sh && bash scripts/check_roadmap_fragment_required.sh", - "file": "docs/roadmaps/entries/PMAT-3571.yaml", - "fix": "Added entry fragment PMAT-3571.yaml and aggregated into roadmap.yaml cleanly.", - "grounding": "measured", - "line": 1, - "output": "PASS check_roadmap_fragment_required.sh: 1 changed entry/entries, each with its fragment; the aggregate is regenerated" - }, - { - "claim": "`evidence/serve/3571/LOADER.md` records manual before/after verification logs of `apr serve` with Qwen3.5.", - "file": "evidence/serve/3571/LOADER.md", - "grounding": "cited", - "line": 1 - } - ], - "raw_bytes": 6020, + "status": "NO-VERDICT", + "verdict": "NO-VERDICT", + "summary": "", + "findings": [], + "raw_bytes": 1754, "err_bytes": 100, "envelope_status": "SUCCESS", - "verdict_source": "structured_output", - "grounding_check": "parity", - "model": "gemini-3.6-flash-high", - "model_measured": "gemini-3.6-flash-high", + "transport_status": "SUCCESS", + "transport_error": "", + "salvageable": "/home/noah/.gemini/antigravity-cli/brain/612e5b75-9884-4d4b-8f4c-af249538f9d7 (1280489 bytes)", + "carried_verdict": "", + "blind_reason": "", + "agy_exit": "", + "model": "gemini-3.8-flash-high", + "model_measured": "gemini-3.8-flash-high", "model_source": "measured", - "family": "gemini" + "family": "gemini", + "role": "independent", + "fallback": { + "judged_by": "gemini-3.8-flash-high", + "exhausted": false, + "attempts": [ + { + "model": "gemini-3.8-flash-high", + "family": "gemini", + "outcome": "answered" + } + ], + "skipped": [] + } + } + ], + "dissent": [ + { + "lane": 3, + "verdict": "NO-VERDICT", + "summary": "", + "findings": [] } ], - "dissent": [], "dedup": [ { "file": "crates/apr-cli/src/commands/serve/handler_gpu_completion.rs", - "line": 387, + "line": 463, "lanes_agreeing": [ - 2 + 1 ], "claims": [ - "CARGO_TARGET_DIR=/tmp/target_rel3571 cargo check -p apr-cli --features inference,cuda compiles cleanly without errors." + "The required case row test ('zero_layers_is_refused_by_name_and_any_layer_is_admitted') and mutant test ('serve_refuses_the_qwen35_base_at_load') are included." ] }, { "file": "crates/apr-cli/src/commands/serve/handler_gpu_completion.rs", - "line": 396, + "line": 492, "lanes_agreeing": [ - 1, - 2, - 3 - ], - "claims": [ - "The Qwen3.5 hybrid architecture check routes to Qwen35Model::create_base_model instead of falling through to OwnedQuantizedModel::from_mapped, matching the acceptance criteria of PMAT-3571.", - "The code correctly branches on `is_qwen35` and calls `Qwen35Model::create_base_model` instead of falling through to the dense `OwnedQuantizedModel` path.", - "`crates/apr-cli/src/commands/serve/handler_gpu_completion.rs` branches on `is_qwen35` and invokes `realizar::gguf::forward_qwen35::Qwen35Model::create_base_model`, resolving the loader defect where Qwen3.5 hybrid models failed to load in `apr serve`." - ] - }, - { - "file": "docs/roadmaps/entries/PMAT-3571.yaml", - "line": 1, - "lanes_agreeing": [ - 2, - 3 - ], - "claims": [ - "PMAT-3571 task entry and roadmap entries define the exact single-layer scope and acceptance criteria.", - "`docs/roadmaps/entries/PMAT-3571.yaml` and `docs/roadmaps/roadmap.yaml` correctly document task PMAT-3571 acceptance criteria and scope boundary (Layer 1 loading fix)." - ] - }, - { - "file": "evidence/serve/3571/LOADER.md", - "line": 1, - "lanes_agreeing": [ - 2, - 3 + 1 ], "claims": [ - "Evidence file LOADER.md accurately documents before and after state for apr serve on Qwen3.5 without overclaiming end-to-end readiness.", - "`evidence/serve/3571/LOADER.md` records manual before/after verification logs of `apr serve` with Qwen3.5." + "The zero_layer_refusal function is correctly implemented to reject models with 0 transformer layers, explicitly naming the architecture and ticket #3571 in the CliError." ] }, { - "file": "evidence/serve/3571/LOADER.md", - "line": 34, + "file": "crates/apr-cli/src/commands/serve/handlers_include_01.rs", + "line": 146, "lanes_agreeing": [ 1 ], "claims": [ - "The receipt accurately documents the subsequent BPE tokenizer failure (`Unknown token ''`) and explicitly adheres to the requirement not to claim the serve path is end-to-end green." + "The refusal check is correctly integrated across all specified GGUF, APR, and Safetensors serve entry points, including pre-checking the declared layer count for APR Q4K pool GPU before spawning its thread." ] } ], "uncovered": [], "coverage_source": "lanes", - "partial": false, - "partial_reasons": [], + "partial": true, + "partial_reasons": [ + "lane 3: no verdict object found in lane file (status=SUCCESS) salvageable=/home/noah/.gemini/antigravity-cli/brain/612e5b75-9884-4d4b-8f4c-af249538f9d7 (1280489 bytes) [UNREVIEWED CLAIM MATERIAL — never a PASS]" + ], + "fallback": { + "same_family_width": 1, + "chain": [ + { + "model": "gemini-3.1-pro-high", + "family": "gemini", + "disposition": "configured" + }, + { + "model": "gemini-3.1-pro-low", + "family": "gemini", + "disposition": "configured" + }, + { + "model": "gemini-3.8-flash-high", + "family": "gemini", + "disposition": "configured" + }, + { + "model": "gpt-oss-120b-medium", + "family": "openai", + "disposition": "fallback" + }, + { + "model": "qwen3.5", + "family": "qwen", + "disposition": "not-run", + "why": "no quorum.local_lane in the config — the aprender lane has no model to load" + }, + { + "model": "claude-opus-4-6-thinking", + "family": "claude", + "disposition": "width", + "why": "same family as the author: at most 1 lane, recorded role width, counted toward no floor (R-15a)" + }, + { + "model": "claude-sonnet-4-6", + "family": "claude", + "disposition": "width", + "why": "same family as the author: at most 1 lane, recorded role width, counted toward no floor (R-15a)" + } + ], + "precheck": [ + { + "family": "gemini", + "model": "gemini-3.1-pro-high", + "probe": 1, + "outcome": "live" + } + ], + "prah": { + "source": "install-receipt", + "path": "/home/noah/.claude/skills/paiml-implement/bin/prah" + } + }, "auto_merge": { - "checked": true, + "checked": false, "was_armed": false, "disarmed": false, - "note": "auto-merge not armed" + "note": "no --pr given: nothing to disarm" }, "lint": { "ok": true, - "output": "receipt complete: kind=artifact lanes=3 author=Opus 5 (1M context)/claude" + "output": "receipt complete: kind=artifact lanes=3 author=claude-opus-5/claude" } } diff --git a/docs/audits/quorum-PMAT-3691.json b/docs/audits/quorum-PMAT-3691.json new file mode 100644 index 0000000000..31236a4724 --- /dev/null +++ b/docs/audits/quorum-PMAT-3691.json @@ -0,0 +1,267 @@ +{ + "ticket": "PMAT-3691", + "base": "0517e75a7", + "base_resolved": "0517e75a7", + "base_note": "no origin/0517e75a7 exists; judged against the local ref", + "head": "fc54dd6091afc33dcf149e5ec4e20aa0671d8cc5", + "diff_sha256": "09ec169a578cd1b507f301310f7b22b4aa03a64c0b20d8a91217e989cd1d10cc", + "width": 3, + "executor": "agy", + "prompt_mode": "inline", + "prompt_bytes": 39401, + "author": { + "model": "claude-opus-5", + "family": "claude", + "source": "flag" + }, + "agreed": true, + "lanes": [ + { + "lane": 1, + "status": "SUCCESS", + "verdict": "PASS", + "summary": "The diff correctly replaces `CliError::InvalidFormat` with `CliError::invalid_model_file` (or the new variant `CliError::invalid_model_bytes`) at the six sites explicitly listed in the ticket, plus four more related sites in the SafeTensors header reader for completeness. A comprehensive test suite is added to verify that each error correctly identifies the file format (GGUF, SafeTensors, or unrecognised model) without incorrectly claiming it's an \"Invalid APR\" file, and confirms the exit code remains 4. Finally, the classification of the remaining sites is well-documented in the roadmap notes as required.", + "findings": [], + "raw_bytes": 2552, + "err_bytes": 0, + "envelope_status": "SUCCESS", + "verdict_source": "structured_output", + "grounding_check": "parity", + "model": "gemini-3.1-pro-high", + "model_measured": "gemini-3.1-pro-high", + "model_source": "measured", + "family": "gemini", + "role": "independent", + "fallback": { + "judged_by": "gemini-3.1-pro-high", + "exhausted": false, + "attempts": [ + { + "model": "gemini-3.1-pro-high", + "family": "gemini", + "outcome": "answered" + } + ], + "skipped": [] + } + }, + { + "lane": 2, + "status": "SUCCESS", + "verdict": "PASS", + "summary": "The diff correctly implements all criteria specified in PMAT-3691. The six identified `InvalidFormat` sites have been migrated to `CliError::invalid_model_file` or a new memory-based variant `invalid_model_bytes`. This ensures the errors appropriately name the underlying format (GGUF, SafeTensors, or \"model\" fallback) rather than erroneously stating \"Invalid APR format\". A comprehensive test file with 13 cases is added to assert that the formats are reported accurately and exit code 4 is maintained. No unauthorized changes were found.", + "findings": [ + { + "claim": "hex.rs sites 323 and 350 use `CliError::invalid_model_file(path, context, &e)` as requested.", + "file": "crates/apr-cli/src/commands/hex.rs", + "grounding": "cited", + "line": 327 + }, + { + "claim": "safe_tensors_header.rs sites 180 and 182 use the new `CliError::invalid_model_bytes` variant, avoiding \"Invalid APR format\" by reading the format from memory.", + "file": "crates/apr-cli/src/commands/safe_tensors_header.rs", + "grounding": "cited", + "line": 166 + }, + { + "claim": "trace_likely_has_repeated.rs sites 267 and 332 use `CliError::invalid_model_file` to accurately name the format on parse/inspect failures.", + "file": "crates/apr-cli/src/commands/trace_likely_has_repeated.rs", + "grounding": "cited", + "line": 267 + }, + { + "claim": "A comprehensive test row per command on a truncated GGUF and malformed SafeTensors is provided, asserting that \"Invalid APR\" is not emitted and exit codes remain 4.", + "file": "crates/apr-cli/src/commands/model_file_error_tests_3691.rs", + "grounding": "cited", + "line": 83 + } + ], + "raw_bytes": 4362, + "err_bytes": 0, + "envelope_status": "SUCCESS", + "verdict_source": "structured_output", + "grounding_check": "parity", + "model": "gemini-3.1-pro-high", + "model_measured": "gemini-3.1-pro-high", + "model_source": "measured", + "family": "gemini", + "role": "independent", + "fallback": { + "judged_by": "gemini-3.1-pro-high", + "exhausted": false, + "attempts": [ + { + "model": "gpt-oss-120b-medium", + "family": "openai", + "outcome": "capacity", + "reason": "UNAVAILABLE (code 503): No capacity available for model gpt-oss-120b-medium on the server" + }, + { + "model": "gpt-oss-120b-medium", + "family": "openai", + "outcome": "capacity", + "reason": "UNAVAILABLE (code 503): No capacity available for model gpt-oss-120b-medium on the server" + }, + { + "model": "gemini-3.1-pro-high", + "family": "gemini", + "outcome": "answered" + } + ], + "skipped": [] + } + }, + { + "lane": 3, + "status": "SUCCESS", + "verdict": "PASS", + "summary": "The diff implements all acceptance criteria of PMAT-3691 cleanly and completely:\n1. Replaces `CliError::InvalidFormat` with `CliError::invalid_model_file` (or `CliError::invalid_model_bytes` when bytes are already in memory) across all six specified sites in `hex.rs`, `safe_tensors_header.rs`, and `trace_likely_has_repeated.rs`, plus four directly related sites in `safe_tensors_header.rs`.\n2. Preserves exit code 4 via `CliError::InvalidModelFile`.\n3. Adds a comprehensive test suite in `model_file_error_tests_3691.rs` testing `hex`, `hex --slice`, `trace`, and direct SafeTensors header parsing on truncated/malformed GGUF and SafeTensors fixtures, verifying proper format identification, absence of misleading \"Invalid APR\" messages, no leakage of core error prefixes, and an APR control.\n4. Includes the comprehensive classification of all 126 `CliError::InvalidFormat` sites in the roadmap entry notes.\n5. No gates weakened, no out-of-scope modifications made, and all stop list rules are respected.", + "findings": [], + "raw_bytes": 3384, + "err_bytes": 100, + "envelope_status": "SUCCESS", + "verdict_source": "structured_output", + "grounding_check": "parity", + "model": "gemini-3.7-flash-high", + "model_measured": "gemini-3.7-flash-high", + "model_source": "measured", + "family": "gemini", + "role": "independent", + "fallback": { + "judged_by": "gemini-3.7-flash-high", + "exhausted": false, + "attempts": [ + { + "model": "gemini-3.7-flash-high", + "family": "gemini", + "outcome": "answered" + } + ], + "skipped": [] + } + } + ], + "dissent": [], + "dedup": [ + { + "file": "crates/apr-cli/src/commands/hex.rs", + "line": 327, + "lanes_agreeing": [ + 2 + ], + "claims": [ + "hex.rs sites 323 and 350 use `CliError::invalid_model_file(path, context, &e)` as requested." + ] + }, + { + "file": "crates/apr-cli/src/commands/model_file_error_tests_3691.rs", + "line": 83, + "lanes_agreeing": [ + 2 + ], + "claims": [ + "A comprehensive test row per command on a truncated GGUF and malformed SafeTensors is provided, asserting that \"Invalid APR\" is not emitted and exit codes remain 4." + ] + }, + { + "file": "crates/apr-cli/src/commands/safe_tensors_header.rs", + "line": 166, + "lanes_agreeing": [ + 2 + ], + "claims": [ + "safe_tensors_header.rs sites 180 and 182 use the new `CliError::invalid_model_bytes` variant, avoiding \"Invalid APR format\" by reading the format from memory." + ] + }, + { + "file": "crates/apr-cli/src/commands/trace_likely_has_repeated.rs", + "line": 267, + "lanes_agreeing": [ + 2 + ], + "claims": [ + "trace_likely_has_repeated.rs sites 267 and 332 use `CliError::invalid_model_file` to accurately name the format on parse/inspect failures." + ] + } + ], + "uncovered": [], + "coverage_source": "lanes", + "partial": false, + "partial_reasons": [ + "lane models: only 2 distinct model ids across 3 lanes (gemini-3.1-pro-high, gemini-3.7-flash-high) — lanes sharing an id are resamples, not independent reviewers (PMAT-125)", + "lane 2: judged by gemini-3.1-pro-high after falling through gpt-oss-120b-medium (capacity: UNAVAILABLE (code 503): No capacity available for model gpt-oss-120b-medium on the server); gpt-oss-120b-medium (capacity: UNAVAILABLE (code 503): No capacity available for model gpt-oss-120b-medium on the server) (PMAT-321)" + ], + "fallback": { + "same_family_width": 1, + "chain": [ + { + "model": "gemini-3.1-pro-high", + "family": "gemini", + "disposition": "configured" + }, + { + "model": "gpt-oss-120b-medium", + "family": "openai", + "disposition": "configured" + }, + { + "model": "gemini-3.7-flash-high", + "family": "gemini", + "disposition": "configured" + }, + { + "model": "qwen3.5", + "family": "qwen", + "disposition": "not-run", + "why": "no quorum.local_lane in the config — the aprender lane has no model to load" + }, + { + "model": "claude-opus-4-6-thinking", + "family": "claude", + "disposition": "width", + "why": "same family as the author: at most 1 lane, recorded role width, counted toward no floor (R-15a)" + }, + { + "model": "claude-sonnet-4-6", + "family": "claude", + "disposition": "width", + "why": "same family as the author: at most 1 lane, recorded role width, counted toward no floor (R-15a)" + } + ], + "precheck": [ + { + "family": "gemini", + "model": "gemini-3.1-pro-high", + "probe": 1, + "outcome": "live" + }, + { + "family": "openai", + "model": "gpt-oss-120b-medium", + "probe": 1, + "outcome": "capacity", + "reason": "UNAVAILABLE (code 503): No capacity available for model gpt-oss-120b-medium on the server" + }, + { + "family": "openai", + "model": "gpt-oss-120b-medium", + "probe": 2, + "outcome": "live" + } + ], + "prah": { + "source": "install-receipt", + "path": "/home/noah/.claude/skills/paiml-implement/bin/prah" + } + }, + "auto_merge": { + "checked": false, + "was_armed": false, + "disarmed": false, + "note": "no --pr given: nothing to disarm" + }, + "lint": { + "ok": true, + "output": "receipt complete: kind=artifact lanes=3 author=claude-opus-5/claude" + } +} diff --git a/docs/audits/quorum-PMAT-3694.json b/docs/audits/quorum-PMAT-3694.json new file mode 100644 index 0000000000..7e936b63fa --- /dev/null +++ b/docs/audits/quorum-PMAT-3694.json @@ -0,0 +1,249 @@ +{ + "ticket": "PMAT-3694", + "base": "225b2a9ab", + "base_resolved": "225b2a9ab", + "base_note": "no origin/225b2a9ab exists; judged against the local ref", + "head": "0aac053f21cee40962d38530ff4b9d702a6d9571", + "diff_sha256": "57fa4293b9139dd216c9c2c7ac1fa8941c1e030c666f3fdab9893fe250f198c9", + "width": 3, + "executor": "agy", + "prompt_mode": "inline", + "prompt_bytes": 36499, + "author": { + "model": "claude-opus-5", + "family": "claude", + "source": "flag" + }, + "agreed": true, + "lanes": [ + { + "lane": 1, + "status": "SUCCESS", + "verdict": "PASS", + "summary": "The diff flawlessly replaces the python3 JSON parser with a robust POSIX awk implementation that strictly adheres to RFC 8259 (including meticulous UTF-8 validation). It effectively removes the stop-gap UNMEASURED (exit 3) path across the shell script and the Rust test, successfully fulfilling all acceptance criteria. The test matrix comprehensively handles duplicate keys, escaped values, nesting, non-JSON numbers, and NUL byte edge cases. No negative effects or undocumented modifications were found.", + "findings": [], + "raw_bytes": 2334, + "err_bytes": 0, + "envelope_status": "SUCCESS", + "verdict_source": "structured_output", + "grounding_check": "parity", + "model": "gemini-3.1-pro-high", + "model_measured": "gemini-3.1-pro-high", + "model_source": "measured", + "family": "gemini", + "role": "independent", + "fallback": { + "judged_by": "gemini-3.1-pro-high", + "exhausted": false, + "attempts": [ + { + "model": "gemini-3.1-pro-high", + "family": "gemini", + "outcome": "answered" + } + ], + "skipped": [] + } + }, + { + "lane": 2, + "status": "SUCCESS", + "verdict": "PASS", + "summary": "The diff accurately and thoroughly addresses all PMAT-3694 ticket requirements. It successfully removes the reliance on Python and the UNMEASURED exit 3 path, replacing it with a robust, strict POSIX awk JSON parser that correctly implements UTF-8 validation and the specified edge cases (duplicate keys, NUL bytes). The self-test logic and Rust tests have been properly updated to verify these changes within a strict, python-free environment. No extraneous or forbidden behavior was introduced.", + "findings": [ + { + "claim": "The UNMEASURED exit 3 path for missing python interpreter has been fully removed.", + "file": "scripts/parity_receipt_denominator.sh", + "grounding": "cited", + "line": 1 + }, + { + "claim": "A POSIX awk script is embedded to iteratively parse JSON, validate UTF-8, and record root keys/values, functioning without python.", + "file": "scripts/parity_receipt_denominator.sh", + "grounding": "cited", + "line": 1 + }, + { + "claim": "The Rust test's acceptance of exit 3 (UNMEASURED) is removed and replaced with a strict assert!(out.status.success()).", + "file": "crates/aprender-contracts-cli/tests/ont4c3_parity_receipts.rs", + "grounding": "cited", + "line": 1 + }, + { + "claim": "A missing awk classifier results in an ENV exit 2, replacing the previous unmeasured fallback.", + "file": "scripts/parity_receipt_denominator.sh", + "grounding": "cited", + "line": 1 + }, + { + "claim": "The self-test successfully verifies the script using a python-free path environment and a comprehensive 35-row table test to validate the awk parser's edge cases.", + "file": "scripts/parity_receipt_denominator.sh", + "grounding": "cited", + "line": 1 + } + ], + "raw_bytes": 4502, + "err_bytes": 100, + "envelope_status": "SUCCESS", + "verdict_source": "structured_output", + "grounding_check": "parity", + "model": "gemini-3.1-pro-high", + "model_measured": "gemini-3.1-pro-high", + "model_source": "measured", + "family": "gemini", + "role": "independent", + "fallback": { + "judged_by": "gemini-3.1-pro-high", + "exhausted": false, + "attempts": [ + { + "model": "gpt-oss-120b-medium", + "family": "openai", + "outcome": "capacity", + "reason": "UNAVAILABLE (code 503): No capacity available for model gpt-oss-120b-medium on the server" + }, + { + "model": "gpt-oss-120b-medium", + "family": "openai", + "outcome": "capacity", + "reason": "Our servers are experiencing high traffic right now, please try again in a minute. (UNAVAILABLE (code 503): No capacity available for model gpt-oss-120b-medium on the server)" + }, + { + "model": "gemini-3.1-pro-high", + "family": "gemini", + "outcome": "answered" + } + ], + "skipped": [] + } + }, + { + "lane": 3, + "status": "SUCCESS", + "verdict": "PASS", + "summary": "Verified PMAT-3694 diff against ticket requirements, acceptance criteria, and notes:\n1. Python-free classify(): `scripts/parity_receipt_denominator.sh` replaces python3 `classify()` with an independent iterative RFC 8259 parser in POSIX awk under `LC_ALL=C`. It validates UTF-8 bytes (RFC 3629), decodes ASCII and `\\uXXXX` escapes, enforces numeric syntax, handles later duplicate keys winning, and guards against raw NUL byte truncation across C-string awks.\n2. Removal of UNMEASURED exit 3: Removed `PY_BIN`/`PARITY_PYTHON` and the exit 3 UNMEASURED paths from `scripts/parity_receipt_denominator.sh`. In `crates/aprender-contracts-cli/tests/ont4c3_parity_receipts.rs`, the exit 3 acceptance was removed, strictly enforcing `out.status.success()`.\n3. Corpus & Self-test: Real corpus passes with `PASS 7 receipt(s) under evidence/parity/**, and evidence/parity/EXPECTED_RECEIPTS says 7.` `--self-test` passes all 9 rows (including the 35-row classification table) across `gawk`, `mawk`, and `busybox awk`.\n4. Differential & Mutants: Differential test against python `json.load` over all 118 evidence files yielded 0 mismatches. All 6 candidate mutants (substring classifier, duplicate key ordering, UTF-8 validator bypass, trailing content, schema kind validation, and NUL precheck) were measured and verified RED.\n5. Cargo Tests & Lints: `cargo test -p aprender-contracts-cli --test ont4c3_parity_receipts` passes 9/9 tests on standard and python-free PATHs. `cargo fmt --all -- --check`, `bashrs lint`, and `check_shell_lint_ratchet.sh` pass cleanly. `docs/roadmaps/roadmap.yaml` is clean and generated via `make roadmap-aggregate`.", + "findings": [], + "raw_bytes": 4610, + "err_bytes": 100, + "envelope_status": "SUCCESS", + "verdict_source": "structured_output", + "grounding_check": "parity", + "model": "gemini-3.7-flash-high", + "model_measured": "gemini-3.7-flash-high", + "model_source": "measured", + "family": "gemini", + "role": "independent", + "fallback": { + "judged_by": "gemini-3.7-flash-high", + "exhausted": false, + "attempts": [ + { + "model": "gemini-3.7-flash-high", + "family": "gemini", + "outcome": "answered" + } + ], + "skipped": [] + } + } + ], + "dissent": [], + "dedup": [ + { + "file": "crates/aprender-contracts-cli/tests/ont4c3_parity_receipts.rs", + "line": 1, + "lanes_agreeing": [ + 2 + ], + "claims": [ + "The Rust test's acceptance of exit 3 (UNMEASURED) is removed and replaced with a strict assert!(out.status.success())." + ] + }, + { + "file": "scripts/parity_receipt_denominator.sh", + "line": 1, + "lanes_agreeing": [ + 2 + ], + "claims": [ + "A POSIX awk script is embedded to iteratively parse JSON, validate UTF-8, and record root keys/values, functioning without python.", + "A missing awk classifier results in an ENV exit 2, replacing the previous unmeasured fallback.", + "The UNMEASURED exit 3 path for missing python interpreter has been fully removed.", + "The self-test successfully verifies the script using a python-free path environment and a comprehensive 35-row table test to validate the awk parser's edge cases." + ] + } + ], + "uncovered": [], + "coverage_source": "lanes", + "partial": false, + "partial_reasons": [ + "lane models: only 2 distinct model ids across 3 lanes (gemini-3.1-pro-high, gemini-3.7-flash-high) — lanes sharing an id are resamples, not independent reviewers (PMAT-125)", + "lane 2: judged by gemini-3.1-pro-high after falling through gpt-oss-120b-medium (capacity: UNAVAILABLE (code 503): No capacity available for model gpt-oss-120b-medium on the server); gpt-oss-120b-medium (capacity: Our servers are experiencing high traffic right now, please try again in a minute. (UNAVAILABLE (code 503): No capacity available for model gpt-oss-120b-medium on the server)) (PMAT-321)" + ], + "fallback": { + "same_family_width": 1, + "chain": [ + { + "model": "gemini-3.1-pro-high", + "family": "gemini", + "disposition": "configured" + }, + { + "model": "gpt-oss-120b-medium", + "family": "openai", + "disposition": "configured" + }, + { + "model": "gemini-3.7-flash-high", + "family": "gemini", + "disposition": "configured" + }, + { + "model": "qwen3.5", + "family": "qwen", + "disposition": "not-run", + "why": "no quorum.local_lane in the config — the aprender lane has no model to load" + }, + { + "model": "claude-opus-4-6-thinking", + "family": "claude", + "disposition": "width", + "why": "same family as the author: at most 1 lane, recorded role width, counted toward no floor (R-15a)" + }, + { + "model": "claude-sonnet-4-6", + "family": "claude", + "disposition": "width", + "why": "same family as the author: at most 1 lane, recorded role width, counted toward no floor (R-15a)" + } + ], + "precheck": [ + { + "family": "gemini", + "model": "gemini-3.1-pro-high", + "probe": 1, + "outcome": "live" + }, + { + "family": "openai", + "model": "gpt-oss-120b-medium", + "probe": 1, + "outcome": "live" + } + ], + "prah": { + "source": "install-receipt", + "path": "/home/noah/.claude/skills/paiml-implement/bin/prah" + } + }, + "auto_merge": { + "checked": false, + "was_armed": false, + "disarmed": false, + "note": "no --pr given: nothing to disarm" + }, + "lint": { + "ok": true, + "output": "receipt complete: kind=artifact lanes=3 author=claude-opus-5/claude" + } +} diff --git a/docs/audits/quorum-PMAT-3697.json b/docs/audits/quorum-PMAT-3697.json new file mode 100644 index 0000000000..4822b97068 --- /dev/null +++ b/docs/audits/quorum-PMAT-3697.json @@ -0,0 +1,178 @@ +{ + "ticket": "PMAT-3697", + "base": "225b2a9ab", + "base_resolved": "225b2a9ab", + "base_note": "no origin/225b2a9ab exists; judged against the local ref", + "head": "78b03cbdd6f9e6c990ddc57fae87057c2baa6219", + "diff_sha256": "f0cdc35168ef76b75c3ddeaabc71afc6bcb1b906b0bbe14b569f082da1f06661", + "width": 3, + "executor": "agy", + "prompt_mode": "inline", + "prompt_bytes": 43954, + "author": { + "model": "claude-opus-5", + "family": "claude", + "source": "flag" + }, + "agreed": true, + "lanes": [ + { + "lane": 1, + "status": "SUCCESS", + "verdict": "PASS", + "summary": "I have reviewed the diff against the acceptance criteria of ticket PMAT-3697.\n\n- (a) `check_t2_preflight_verdict.sh` and `check_no_hosted_runners.sh` correctly replace their Python dependencies with `awk`. The `awk` equivalent regex handling of word boundaries in `check_no_hosted_runners.sh` is carefully implemented and explicitly tested for Unicode discrepancies, behaving as described in the PR notes. The `awk` line replacement in `check_t2_preflight_verdict.sh` functions flawlessly and handles missing target signatures appropriately.\n- (b) `check_nextest_ci_profile_no_fail_fast.sh` and `check_roadmap_ids_unique.sh` correctly retain their Python readers and utilize the new `scripts/lib/python_fleet_state.sh`. When the interpreter or required module (`PyYAML`) is missing, they exit 0 without printing a `PASS` line, instead correctly outputting the required `UNMEASURED` trace to `stderr`. Broken installations yield an `ENV` exit code (2).\n- The `lib/python_fleet_state.sh` logic aligns with prior art (`#3692 pathonly`) by emitting `reason=no-interpreter` and `reason=no-module`, staying true to the spirit of the acceptance criteria while avoiding fractured state spaces.\n- The `guard_tree.sh` changes explicitly excluded in the PR notes are correctly omitted from this diff.\n- All receipt claims (including adversarial corpus diffs and mutant behavior) are backed by the code.\n\nThe change perfectly fulfills the ticket requirements without introducing regressions or unrequested modifications.", + "findings": [], + "raw_bytes": 6043, + "err_bytes": 0, + "envelope_status": "SUCCESS", + "verdict_source": "structured_output", + "grounding_check": "parity", + "model": "gemini-3.1-pro-high", + "model_measured": "gemini-3.1-pro-high", + "model_source": "measured", + "family": "gemini", + "role": "independent", + "fallback": { + "judged_by": "gemini-3.1-pro-high", + "exhausted": false, + "attempts": [ + { + "model": "gemini-3.1-pro-high", + "family": "gemini", + "outcome": "answered" + } + ], + "skipped": [] + } + }, + { + "lane": 2, + "status": "SUCCESS", + "verdict": "PASS", + "summary": "Lane 2 audit of diff (225b2a9ab...HEAD) for ticket PMAT-3697 complete. The diff faithfully implements all ticket acceptance criteria without scope creep or weakened gates:\n1. `scripts/lib/python_fleet_state.sh` is an option-neutral, sourced library providing `py_fleet_state` and `py_fleet_state_self_test` probes for Python interpreter & module availability.\n2. `scripts/check_t2_preflight_verdict.sh` removes Python dependency by replacing the self-test mutant splice with `awk`.\n3. `scripts/check_no_hosted_runners.sh` removes Python dependency by replacing the workflow image scanner with `awk` using POSIX ERE ASCII word boundaries.\n4. `scripts/check_nextest_ci_profile_no_fail_fast.sh` and `scripts/check_roadmap_ids_unique.sh` use `py_fleet_state` to classify missing Python/PyYAML as fleet state `UNMEASURED` (exit 0), per ticket ruling (b).\n5. All self-tests were verified and pass cleanly. No refuting evidence found. Verdict: PASS.", + "findings": [], + "raw_bytes": 3307, + "err_bytes": 0, + "envelope_status": "SUCCESS", + "verdict_source": "structured_output", + "grounding_check": "parity", + "model": "gemini-3.6-flash-high", + "model_measured": "gemini-3.6-flash-high", + "model_source": "measured", + "family": "gemini", + "role": "independent", + "fallback": { + "judged_by": "gemini-3.6-flash-high", + "exhausted": false, + "attempts": [ + { + "model": "gemini-3.6-flash-high", + "family": "gemini", + "outcome": "answered" + } + ], + "skipped": [] + } + }, + { + "lane": 3, + "status": "SUCCESS", + "verdict": "PASS", + "summary": "The diff perfectly implements the requirements of PMAT-3697. It successfully removes the Python dependency for `check_t2_preflight_verdict.sh` and `check_no_hosted_runners.sh` by substituting them with equivalent, well-tested `awk` scripts. The deliberate edge-case differences in token boundaries for `no_hosted_runners` are explicitly documented in the ticket notes and accurately implemented. Furthermore, it cleanly integrates the new `py_fleet_state` mechanism into `check_nextest_ci_profile_no_fail_fast.sh` and `check_roadmap_ids_unique.sh` to classify the absence of a Python reader or its modules as `UNMEASURED` fleet state (exit 0) rather than a `FAIL`, fully aligning with the #3692 ruling described in the ticket. All receipts claimed in the PR notes are thoroughly backed by the diff.", + "findings": [], + "raw_bytes": 4377, + "err_bytes": 0, + "envelope_status": "SUCCESS", + "verdict_source": "structured_output", + "grounding_check": "parity", + "model": "gemini-3.1-pro-low", + "model_measured": "gemini-3.1-pro-low", + "model_source": "measured", + "family": "gemini", + "role": "independent", + "fallback": { + "judged_by": "gemini-3.1-pro-low", + "exhausted": false, + "attempts": [ + { + "model": "gemini-3.1-pro-low", + "family": "gemini", + "outcome": "answered" + } + ], + "skipped": [] + } + } + ], + "dissent": [], + "dedup": [], + "uncovered": [], + "coverage_source": "lanes", + "partial": false, + "partial_reasons": [], + "fallback": { + "same_family_width": 1, + "chain": [ + { + "model": "gemini-3.1-pro-high", + "family": "gemini", + "disposition": "configured" + }, + { + "model": "gemini-3.6-flash-high", + "family": "gemini", + "disposition": "configured" + }, + { + "model": "gemini-3.1-pro-low", + "family": "gemini", + "disposition": "configured" + }, + { + "model": "gpt-oss-120b-medium", + "family": "openai", + "disposition": "fallback" + }, + { + "model": "qwen3.5", + "family": "qwen", + "disposition": "not-run", + "why": "no quorum.local_lane in the config — the aprender lane has no model to load" + }, + { + "model": "claude-opus-4-6-thinking", + "family": "claude", + "disposition": "width", + "why": "same family as the author: at most 1 lane, recorded role width, counted toward no floor (R-15a)" + }, + { + "model": "claude-sonnet-4-6", + "family": "claude", + "disposition": "width", + "why": "same family as the author: at most 1 lane, recorded role width, counted toward no floor (R-15a)" + } + ], + "precheck": [ + { + "family": "gemini", + "model": "gemini-3.1-pro-high", + "probe": 1, + "outcome": "live" + } + ], + "prah": { + "source": "install-receipt", + "path": "/home/noah/.claude/skills/paiml-implement/bin/prah" + } + }, + "auto_merge": { + "checked": false, + "was_armed": false, + "disarmed": false, + "note": "no --pr given: nothing to disarm" + }, + "lint": { + "ok": true, + "output": "receipt complete: kind=artifact lanes=3 author=claude-opus-5/claude" + } +} diff --git a/docs/audits/quorum-PMAT-3699.json b/docs/audits/quorum-PMAT-3699.json new file mode 100644 index 0000000000..4d2256179c --- /dev/null +++ b/docs/audits/quorum-PMAT-3699.json @@ -0,0 +1,194 @@ +{ + "ticket": "PMAT-3699", + "base": "225b2a9ab", + "base_resolved": "225b2a9ab", + "base_note": "no origin/225b2a9ab exists; judged against the local ref", + "head": "030af09a2b563551e980d38f2ea0a6f217f703ec", + "diff_sha256": "131d2c92dd2af10f488e0a37d6f1b0ccb12c1011c762366bad517f157604bf3d", + "width": 3, + "executor": "agy", + "prompt_mode": "inline", + "prompt_bytes": 45012, + "author": { + "model": "claude-opus-5", + "family": "claude", + "source": "flag" + }, + "agreed": true, + "lanes": [ + { + "lane": 1, + "status": "SUCCESS", + "verdict": "PASS", + "summary": "The diff implements all acceptance criteria in PMAT-3699.\n1. `prepare_bump.sh` uses `check_pr_closes_issue.sh --list-owed` to correctly derive the owed `#N` list and appends exactly ONE `keep-open:` line using the verbatim reason requested.\n2. Before any commits, pushes, or PR creations, `prepare_bump.sh` runs `check_pr_closes_issue.sh --body` and correctly aborts (via `die`) if the return code is non-zero.\n3. `scripts/check_release_bump_pr_body.sh` provides the required case-table row validating that the EPIC ref along with a `refs #N` produces a PASSing PR body, and explicitly includes a test for a mutant that drops the `keep-open` line, verifying that it goes RED.\n4. No requirements or negative constraints from the ticket are violated.", + "findings": [], + "raw_bytes": 2781, + "err_bytes": 0, + "envelope_status": "SUCCESS", + "verdict_source": "structured_output", + "grounding_check": "parity", + "model": "gemini-3.1-pro-high", + "model_measured": "gemini-3.1-pro-high", + "model_source": "measured", + "family": "gemini", + "role": "independent", + "fallback": { + "judged_by": "gemini-3.1-pro-high", + "exhausted": false, + "attempts": [ + { + "model": "gemini-3.1-pro-high", + "family": "gemini", + "outcome": "answered" + } + ], + "skipped": [] + } + }, + { + "lane": 2, + "status": "SUCCESS", + "verdict": "PASS", + "summary": "The diff completely and correctly implements the acceptance criteria for PMAT-3699. It modifies `prepare_bump.sh` to determine owed references via the new `check_pr_closes_issue.sh --list-owed` seam, constructs exactly one `keep-open` line with the required reason string, and runs the guard on the PR body before any commit/push/PR creation, refusing to proceed if the body fails. It also includes the comprehensive `check_release_bump_pr_body.sh` case table verifying these behaviors and their corresponding mutant failures (e.g., epic-and-refs passing, drop-keep-open going RED, landmine blocking PR push). No violations or deviations were found.", + "findings": [], + "raw_bytes": 2670, + "err_bytes": 0, + "envelope_status": "SUCCESS", + "verdict_source": "structured_output", + "grounding_check": "parity", + "model": "gemini-3.1-pro-high", + "model_measured": "gemini-3.1-pro-high", + "model_source": "measured", + "family": "gemini", + "role": "independent", + "fallback": { + "judged_by": "gemini-3.1-pro-high", + "exhausted": false, + "attempts": [ + { + "model": "gpt-oss-120b-medium", + "family": "openai", + "outcome": "capacity", + "reason": "UNAVAILABLE (code 503): No capacity available for model gpt-oss-120b-medium on the server" + }, + { + "model": "gpt-oss-120b-medium", + "family": "openai", + "outcome": "capacity", + "reason": "Our servers are experiencing high traffic right now, please try again in a minute. (UNAVAILABLE (code 503): No capacity available for model gpt-oss-120b-medium on the server)" + }, + { + "model": "gemini-3.1-pro-high", + "family": "gemini", + "outcome": "answered" + } + ], + "skipped": [] + } + }, + { + "lane": 3, + "status": "SUCCESS", + "verdict": "PASS", + "summary": "The diff meets all acceptance criteria for PMAT-3699. The script now calculates the un-closed references using the guard's logic, appends exactly ONE keep-open line with the specified reason before opening the PR, and validates the generated body against the guard beforehand. The check script includes case table rows that test both positive and negative cases as specified.", + "findings": [], + "raw_bytes": 1789, + "err_bytes": 100, + "envelope_status": "SUCCESS", + "verdict_source": "response", + "model": "gemini-3.1-pro-low", + "model_measured": "gemini-3.1-pro-low", + "model_source": "measured", + "family": "gemini", + "role": "independent", + "fallback": { + "judged_by": "gemini-3.1-pro-low", + "exhausted": false, + "attempts": [ + { + "model": "gemini-3.1-pro-low", + "family": "gemini", + "outcome": "answered" + } + ], + "skipped": [] + } + } + ], + "dissent": [], + "dedup": [], + "uncovered": [], + "coverage_source": "lanes", + "partial": false, + "partial_reasons": [ + "lane models: only 2 distinct model ids across 3 lanes (gemini-3.1-pro-high, gemini-3.1-pro-low) — lanes sharing an id are resamples, not independent reviewers (PMAT-125)", + "lane 3: verdict counted from .response past a non-JSON residue of 269 bytes ('``` I am waiting for the background search to lo') — agy emitted no structured_output for this lane (PMAT-261)", + "lane 2: judged by gemini-3.1-pro-high after falling through gpt-oss-120b-medium (capacity: UNAVAILABLE (code 503): No capacity available for model gpt-oss-120b-medium on the server); gpt-oss-120b-medium (capacity: Our servers are experiencing high traffic right now, please try again in a minute. (UNAVAILABLE (code 503): No capacity available for model gpt-oss-120b-medium on the server)) (PMAT-321)" + ], + "fallback": { + "same_family_width": 1, + "chain": [ + { + "model": "gemini-3.1-pro-high", + "family": "gemini", + "disposition": "configured" + }, + { + "model": "gpt-oss-120b-medium", + "family": "openai", + "disposition": "configured" + }, + { + "model": "gemini-3.1-pro-low", + "family": "gemini", + "disposition": "configured" + }, + { + "model": "qwen3.5", + "family": "qwen", + "disposition": "not-run", + "why": "no quorum.local_lane in the config — the aprender lane has no model to load" + }, + { + "model": "claude-opus-4-6-thinking", + "family": "claude", + "disposition": "width", + "why": "same family as the author: at most 1 lane, recorded role width, counted toward no floor (R-15a)" + }, + { + "model": "claude-sonnet-4-6", + "family": "claude", + "disposition": "width", + "why": "same family as the author: at most 1 lane, recorded role width, counted toward no floor (R-15a)" + } + ], + "precheck": [ + { + "family": "gemini", + "model": "gemini-3.1-pro-high", + "probe": 1, + "outcome": "live" + }, + { + "family": "openai", + "model": "gpt-oss-120b-medium", + "probe": 1, + "outcome": "live" + } + ], + "prah": { + "source": "install-receipt", + "path": "/home/noah/.claude/skills/paiml-implement/bin/prah" + } + }, + "auto_merge": { + "checked": false, + "was_armed": false, + "disarmed": false, + "note": "no --pr given: nothing to disarm" + }, + "lint": { + "ok": true, + "output": "receipt complete: kind=artifact lanes=3 author=claude-opus-5/claude" + } +} diff --git a/docs/audits/quorum-PMAT-3703.json b/docs/audits/quorum-PMAT-3703.json new file mode 100644 index 0000000000..f995a4f68b --- /dev/null +++ b/docs/audits/quorum-PMAT-3703.json @@ -0,0 +1,196 @@ +{ + "ticket": "PMAT-3703", + "base": "origin/main", + "base_resolved": "origin/main", + "base_note": "no origin/origin/main exists; judged against the local ref", + "head": "8c08dd8ddfa0a553def60f9d582239258d5b097c", + "diff_sha256": "1a373a497fef965a9f5bc7fdd612121e571b439702fa4718efd4f7db6644ec76", + "width": 3, + "executor": "agy", + "prompt_mode": "inline", + "prompt_bytes": 36148, + "author": { + "model": "claude-opus-5", + "family": "claude", + "source": "flag" + }, + "agreed": true, + "lanes": [ + { + "lane": 1, + "status": "SUCCESS", + "verdict": "PASS", + "summary": "The diff successfully addresses PMAT-3703 without weakening tests. It replaces arbitrary integer wall-clock assertions with correct invariants (e.g., generated counts and guaranteed monotonic invariants over wait boundaries). The `check_no_timing_in_required.sh` Part 4 properly handles regex matching, parsing, and baseline verification logic for remaining cases in the code, enforcing a `keyed` test ratchet list that strictly avoids regressions. All acceptance criteria are fully met.", + "findings": [], + "raw_bytes": 2337, + "err_bytes": 0, + "envelope_status": "SUCCESS", + "verdict_source": "structured_output", + "grounding_check": "parity", + "model": "gemini-3.1-pro-high", + "model_measured": "gemini-3.1-pro-high", + "model_source": "measured", + "family": "gemini", + "role": "independent", + "fallback": { + "judged_by": "gemini-3.1-pro-high", + "exhausted": false, + "attempts": [ + { + "model": "gemini-3.1-pro-high", + "family": "gemini", + "outcome": "answered" + } + ], + "skipped": [] + } + }, + { + "lane": 2, + "status": "SUCCESS", + "verdict": "PASS", + "summary": "The diff perfectly matches the acceptance criteria for PMAT-3703. It fixes the `generation_time_ms` assertion in `pipeline.rs`, drops the integer time assertions in `bug_hunter` and `instruct_trainer`, and correctly updates the `monotonic_ns` check in `inference_monitor_tests`. It introduces a strict guard in `check_no_timing_in_required.sh` with a comprehensive case table and a text-keyed ledger to prevent future integer wall-clock assertions. No tests assert the opposite, no gates are weakened, and all ticket claims are fully backed by the diff.", + "findings": [], + "raw_bytes": 2477, + "err_bytes": 100, + "envelope_status": "SUCCESS", + "verdict_source": "structured_output", + "grounding_check": "parity", + "model": "gemini-3.1-pro-high", + "model_measured": "gemini-3.1-pro-high", + "model_source": "measured", + "family": "gemini", + "role": "independent", + "fallback": { + "judged_by": "gemini-3.1-pro-high", + "exhausted": false, + "attempts": [ + { + "model": "gemini-3.1-pro-high", + "family": "gemini", + "outcome": "answered" + } + ], + "skipped": [ + { + "model": "gpt-oss-120b-medium", + "why": "no capacity: 503 on two pre-check probes (UNAVAILABLE (code 503): No capacity available for model gpt-oss-120b-medium on the server)" + } + ] + } + }, + { + "lane": 3, + "status": "SUCCESS", + "verdict": "PASS", + "summary": "The diff perfectly implements the requirements of PMAT-3703. \n\n1. **AC1 (Fix test_pipeline_generate_small)**: The wall-clock assertion `stats.generation_time_ms > 0` was removed and replaced with a deterministic state verification `assert_eq!(stats.total_generated, programs.len())`.\n2. **AC2 (Sweep & Guard)**: A comprehensive sweep was performed. The exact integer duration shape (`.as_millis()`, `_ms`, etc. `> 0`) is now guarded against in `scripts/check_no_timing_in_required.sh` (Part 4), checking against a `keyed` baseline file `scripts/wallclock_assert_baseline.txt` that documents the 101 existing safe hits with justifications. Three unsafe hits were fixed. The `Instant`-based test (`test_monotonic_ns_non_zero`) was also fixed to sleep *after* initialization to guarantee a clock advance, exactly as described in the ticket notes.\n3. **AC3 (Mutation proof)**: Documented in the ticket notes, and mathematically verified by the diff since the new assertion in `test_pipeline_generate_small` has no dependency on the timing metrics.\n\nNo regressions or contradictions with the ticket were found.", + "findings": [], + "raw_bytes": 3561, + "err_bytes": 0, + "envelope_status": "SUCCESS", + "verdict_source": "structured_output", + "grounding_check": "parity", + "model": "gemini-3.1-pro-low", + "model_measured": "gemini-3.1-pro-low", + "model_source": "measured", + "family": "gemini", + "role": "independent", + "fallback": { + "judged_by": "gemini-3.1-pro-low", + "exhausted": false, + "attempts": [ + { + "model": "gemini-3.1-pro-low", + "family": "gemini", + "outcome": "answered" + } + ], + "skipped": [] + } + } + ], + "dissent": [], + "dedup": [], + "uncovered": [], + "coverage_source": "lanes", + "partial": false, + "partial_reasons": [ + "lane models: only 2 distinct model ids across 3 lanes (gemini-3.1-pro-high, gemini-3.1-pro-low) — lanes sharing an id are resamples, not independent reviewers (PMAT-125)", + "lane 2: judged by gemini-3.1-pro-high after falling through gpt-oss-120b-medium (skipped: no capacity: 503 on two pre-check probes (UNAVAILABLE (code 503): No capacity available for model gpt-oss-120b-medium on the server)) (PMAT-321)" + ], + "fallback": { + "same_family_width": 1, + "chain": [ + { + "model": "gemini-3.1-pro-high", + "family": "gemini", + "disposition": "configured" + }, + { + "model": "gpt-oss-120b-medium", + "family": "openai", + "disposition": "configured" + }, + { + "model": "gemini-3.1-pro-low", + "family": "gemini", + "disposition": "configured" + }, + { + "model": "qwen3.5", + "family": "qwen", + "disposition": "not-run", + "why": "no quorum.local_lane in the config — the aprender lane has no model to load" + }, + { + "model": "claude-opus-4-6-thinking", + "family": "claude", + "disposition": "width", + "why": "same family as the author: at most 1 lane, recorded role width, counted toward no floor (R-15a)" + }, + { + "model": "claude-sonnet-4-6", + "family": "claude", + "disposition": "width", + "why": "same family as the author: at most 1 lane, recorded role width, counted toward no floor (R-15a)" + } + ], + "precheck": [ + { + "family": "gemini", + "model": "gemini-3.1-pro-high", + "probe": 1, + "outcome": "live" + }, + { + "family": "openai", + "model": "gpt-oss-120b-medium", + "probe": 1, + "outcome": "capacity", + "reason": "UNAVAILABLE (code 503): No capacity available for model gpt-oss-120b-medium on the server" + }, + { + "family": "openai", + "model": "gpt-oss-120b-medium", + "probe": 2, + "outcome": "capacity", + "reason": "UNAVAILABLE (code 503): No capacity available for model gpt-oss-120b-medium on the server" + } + ], + "prah": { + "source": "none", + "path": null, + "why": "no --prah, none on PATH, and no install receipt at /home/noah/.claude/skills/paiml-implement/install-receipt.json" + } + }, + "auto_merge": { + "checked": false, + "was_armed": false, + "disarmed": false, + "note": "no --pr given: nothing to disarm" + }, + "lint": { + "ok": true, + "output": "receipt complete: kind=artifact lanes=3 author=claude-opus-5/claude" + } +} diff --git a/docs/audits/quorum-PMAT-3708.json b/docs/audits/quorum-PMAT-3708.json new file mode 100644 index 0000000000..08cf275925 --- /dev/null +++ b/docs/audits/quorum-PMAT-3708.json @@ -0,0 +1,195 @@ +{ + "ticket": "PMAT-3708", + "base": "0cb1e4bd6", + "base_resolved": "0cb1e4bd6", + "base_note": "no origin/0cb1e4bd6 exists; judged against the local ref", + "head": "91fc42d16540861fb4bdc4990f5b33475a034e53", + "diff_sha256": "90fb651f5c1cb819fa03510c97e967e591da1ea310663b968c4f91875ec003ac", + "width": 3, + "executor": "agy", + "prompt_mode": "inline", + "prompt_bytes": 78730, + "author": { + "model": "claude-opus-5", + "family": "claude", + "source": "flag" + }, + "agreed": true, + "lanes": [ + { + "lane": 1, + "status": "SUCCESS", + "verdict": "PASS", + "summary": "The diff strictly fulfills PMAT-3708. Dogfood inheritance is successfully removed in favor of running the real dogfood script at T-1. Both T-1 (via `--receipt-only`) and T-4 run the exact same R5 validation function (`rule_r5`) on the same receipt file, ensuring alignment and preventing NO-GOs from being tagged. The close step properly asserts the epic is the only open item in the milestone, closes it, and then closes the milestone; any other open item aborts the process. Additionally, `prepare_bump.sh` strictly verifies that model-ladder receipts are green and unignored before opening the bump PR. All case tables, including the required mutants for each acceptance criterion, are comprehensively implemented and completely backed by the diff.", + "findings": [], + "raw_bytes": 2802, + "err_bytes": 0, + "envelope_status": "SUCCESS", + "verdict_source": "structured_output", + "grounding_check": "parity", + "model": "gemini-3.1-pro-high", + "model_measured": "gemini-3.1-pro-high", + "model_source": "measured", + "family": "gemini", + "role": "independent", + "fallback": { + "judged_by": "gemini-3.1-pro-high", + "exhausted": false, + "attempts": [ + { + "model": "gemini-3.1-pro-high", + "family": "gemini", + "outcome": "answered" + } + ], + "skipped": [] + } + }, + { + "lane": 2, + "status": "SUCCESS", + "verdict": "PASS", + "summary": "The diff fully implements PMAT-3708 according to all acceptance criteria. It successfully eliminates the dogfood inheritance path ('NEVER inherit'), running the dogfood in all cases. It introduces `rule_r5` in `check_publish_preflight.sh` to ensure the T-1 test and the T-4 publish gate share the exact same evaluation logic on the same file. It also resolves the epic close sequence by explicitly closing the epic prior to the milestone, checking for any other open issues first. Finally, it satisfies the new obligation to ensure model-ladder receipts are present and non-ignored before executing the release bump. The new test file `check_release_autopilot_dogfood_close.sh` thoroughly verifies these exact behaviors with comprehensive row tables and mutant checks, matching the counts and outputs claimed in the PR body. No regression or contradictory behavior was found.", + "findings": [], + "raw_bytes": 3005, + "err_bytes": 0, + "envelope_status": "SUCCESS", + "verdict_source": "structured_output", + "grounding_check": "parity", + "model": "gemini-3.1-pro-high", + "model_measured": "gemini-3.1-pro-high", + "model_source": "measured", + "family": "gemini", + "role": "independent", + "fallback": { + "judged_by": "gemini-3.1-pro-high", + "exhausted": false, + "attempts": [ + { + "model": "gemini-3.1-pro-high", + "family": "gemini", + "outcome": "answered" + } + ], + "skipped": [ + { + "model": "gpt-oss-120b-medium", + "why": "no capacity: 503 on two pre-check probes (UNAVAILABLE (code 503): No capacity available for model gpt-oss-120b-medium on the server)" + } + ] + } + }, + { + "lane": 3, + "status": "SUCCESS", + "verdict": "PASS", + "summary": "The diff successfully implements all requirements outlined in PMAT-3708.\n- Done when 1 & 2: Autopilot now never inherits the dogfood GO. Instead, it always runs the dogfood step and enforces rule R5 at T-1 using `check_publish_preflight.sh --receipt-only`, reusing the exact same file and function (`rule_r5`) as the T-4 gate. A comprehensive case table (`check_release_autopilot_dogfood_close.sh`) was added to test this logic with all required rows and mutants.\n- Done when 4: The `close` step in `autopilot.sh` now correctly identifies the release epic, verifies no other milestone items are open, closes the epic if it's open, and then closes the milestone.\n- Obligation 5: `prepare_bump.sh --ship` now mandates green model-ladder receipts for the new version by running `scripts/check_model_ladder.sh --version \"$V\"`. It also refuses if they are present but ignored. The `check_release_bump_pr_body.sh` case table was expanded to include `ladder-missing-gx10` and `ladder-ignored` rows and their mutants.\n\nNo deviations from the ticket's instructions or prohibitions were found.", + "findings": [], + "raw_bytes": 3450, + "err_bytes": 0, + "envelope_status": "SUCCESS", + "verdict_source": "structured_output", + "grounding_check": "parity", + "model": "gemini-3.1-pro-low", + "model_measured": "gemini-3.1-pro-low", + "model_source": "measured", + "family": "gemini", + "role": "independent", + "fallback": { + "judged_by": "gemini-3.1-pro-low", + "exhausted": false, + "attempts": [ + { + "model": "gemini-3.1-pro-low", + "family": "gemini", + "outcome": "answered" + } + ], + "skipped": [] + } + } + ], + "dissent": [], + "dedup": [], + "uncovered": [], + "coverage_source": "lanes", + "partial": false, + "partial_reasons": [ + "lane models: only 2 distinct model ids across 3 lanes (gemini-3.1-pro-high, gemini-3.1-pro-low) — lanes sharing an id are resamples, not independent reviewers (PMAT-125)", + "lane 2: judged by gemini-3.1-pro-high after falling through gpt-oss-120b-medium (skipped: no capacity: 503 on two pre-check probes (UNAVAILABLE (code 503): No capacity available for model gpt-oss-120b-medium on the server)) (PMAT-321)" + ], + "fallback": { + "same_family_width": 1, + "chain": [ + { + "model": "gemini-3.1-pro-high", + "family": "gemini", + "disposition": "configured" + }, + { + "model": "gpt-oss-120b-medium", + "family": "openai", + "disposition": "configured" + }, + { + "model": "gemini-3.1-pro-low", + "family": "gemini", + "disposition": "configured" + }, + { + "model": "qwen3.5", + "family": "qwen", + "disposition": "not-run", + "why": "no quorum.local_lane in the config — the aprender lane has no model to load" + }, + { + "model": "claude-opus-4-6-thinking", + "family": "claude", + "disposition": "width", + "why": "same family as the author: at most 1 lane, recorded role width, counted toward no floor (R-15a)" + }, + { + "model": "claude-sonnet-4-6", + "family": "claude", + "disposition": "width", + "why": "same family as the author: at most 1 lane, recorded role width, counted toward no floor (R-15a)" + } + ], + "precheck": [ + { + "family": "gemini", + "model": "gemini-3.1-pro-high", + "probe": 1, + "outcome": "live" + }, + { + "family": "openai", + "model": "gpt-oss-120b-medium", + "probe": 1, + "outcome": "capacity", + "reason": "UNAVAILABLE (code 503): No capacity available for model gpt-oss-120b-medium on the server" + }, + { + "family": "openai", + "model": "gpt-oss-120b-medium", + "probe": 2, + "outcome": "capacity", + "reason": "UNAVAILABLE (code 503): No capacity available for model gpt-oss-120b-medium on the server" + } + ], + "prah": { + "source": "install-receipt", + "path": "/home/noah/.claude/skills/paiml-implement/bin/prah" + } + }, + "auto_merge": { + "checked": false, + "was_armed": false, + "disarmed": false, + "note": "no --pr given: nothing to disarm" + }, + "lint": { + "ok": true, + "output": "receipt complete: kind=artifact lanes=3 author=claude-opus-5/claude" + } +} diff --git a/docs/audits/quorum-PMAT-3711.json b/docs/audits/quorum-PMAT-3711.json new file mode 100644 index 0000000000..4529eb2e72 --- /dev/null +++ b/docs/audits/quorum-PMAT-3711.json @@ -0,0 +1,260 @@ +{ + "ticket": "PMAT-3711", + "base": "225b2a9ab", + "base_resolved": "225b2a9ab", + "base_note": "no origin/225b2a9ab exists; judged against the local ref", + "head": "8e1214835c8f8ec265c8625092a22b1595a4d172", + "diff_sha256": "f5ccce3821ff2ff9bd5f5f873c995959840f440c67c3143a4bced938e599c46a", + "width": 3, + "executor": "agy", + "prompt_mode": "inline", + "prompt_bytes": 45697, + "author": { + "model": "claude-opus-5", + "family": "claude", + "source": "flag" + }, + "agreed": true, + "lanes": [ + { + "lane": 1, + "status": "SUCCESS", + "verdict": "PASS", + "summary": "The diff implements exactly what is requested in PMAT-3711 and the Cop ruling. It introduces `GpuGoldenLeg` and updates the pipeline to classify CUDA initialization and GPU generation errors as gate failures (`FAIL`) instead of skipping. It successfully separates cases into actual errors vs intended skips (no device, no cuda feature, non-GGUF). The runtime gate is also updated to use the measured backend via `InferenceResult::used_gpu`, satisfying the Cop ruling to label the actual dispatch outcome instead of the intent. All required test cases and mutations are covered in the new tests. No gates were weakened; instead, the GPU validation gate has been strengthened.", + "findings": [], + "raw_bytes": 3329, + "err_bytes": 0, + "envelope_status": "SUCCESS", + "verdict_source": "structured_output", + "model": "gemini-3.1-pro-high", + "model_measured": "gemini-3.1-pro-high", + "model_source": "measured", + "family": "gemini", + "role": "independent", + "fallback": { + "judged_by": "gemini-3.1-pro-high", + "exhausted": false, + "attempts": [ + { + "model": "gemini-3.1-pro-high", + "family": "gemini", + "outcome": "answered" + } + ], + "skipped": [] + } + }, + { + "lane": 2, + "status": "SUCCESS", + "verdict": "PASS", + "summary": "The diff perfectly implements the requirements of PMAT-3711. It fixes the critical bug where GPU generation errors were incorrectly reported as a SKIP, allowing broken GPUs to pass the gate. Now, `validate_gpu_golden_output` uses `GpuGoldenLeg` to correctly distinguish between `Passed`, `WrongAnswer`, `Errored`, and `NotRun`. Only valid skipped states (\"no cuda feature\", \"no CUDA device on this host\", \"judges GGUF only\") result in a skip. Furthermore, it implements the Cop ruling for `run_golden_output_gate_runtime` by properly asserting that the GPU actually served the request (`used_gpu` signal from dispatch), failing if it silently falls back to CPU. Comprehensive tests have been added to verify these behaviors. No gates are weakened and no unexpected behaviors are introduced.", + "findings": [ + { + "claim": "GPU golden generation error correctly fails the gate naming the error, instead of skipping, through `GpuGoldenLeg::judge` mapping `Err` to `Errored` and `failure()` returning the error string.", + "file": "crates/apr-cli/src/commands/output_verification.rs", + "grounding": "cited", + "line": 606 + }, + { + "claim": "SKIP stays only for \"no GPU on this host / no cuda feature\" and names which, implemented cleanly in `gpu_golden_not_run`.", + "file": "crates/apr-cli/src/commands/output_verification.rs", + "grounding": "cited", + "line": 531 + }, + { + "claim": "`run_golden_output_gate_runtime` correctly names the backend that actually ran based on `used_gpu` and fails if it unexpectedly falls back, fulfilling the Cop ruling.", + "file": "crates/apr-cli/src/commands/output_verification.rs", + "grounding": "cited", + "line": 554 + }, + { + "claim": "Case rows (tests) added to verify GPU errors, skips, correct generation, and wrong answers.", + "file": "crates/apr-cli/src/commands/golden_output.rs", + "grounding": "cited", + "line": 673 + } + ], + "raw_bytes": 4894, + "err_bytes": 0, + "envelope_status": "SUCCESS", + "verdict_source": "structured_output", + "grounding_check": "parity", + "model": "gemini-3.1-pro-high", + "model_measured": "gemini-3.1-pro-high", + "model_source": "measured", + "family": "gemini", + "role": "independent", + "fallback": { + "judged_by": "gemini-3.1-pro-high", + "exhausted": false, + "attempts": [ + { + "model": "gpt-oss-120b-medium", + "family": "openai", + "outcome": "capacity", + "reason": "UNAVAILABLE (code 503): No capacity available for model gpt-oss-120b-medium on the server" + }, + { + "model": "gpt-oss-120b-medium", + "family": "openai", + "outcome": "capacity", + "reason": "UNAVAILABLE (code 503): No capacity available for model gpt-oss-120b-medium on the server" + }, + { + "model": "gemini-3.1-pro-high", + "family": "gemini", + "outcome": "answered" + } + ], + "skipped": [] + } + }, + { + "lane": 3, + "status": "SUCCESS", + "verdict": "PASS", + "summary": "PMAT-3711: The diff satisfies all acceptance criteria and the folded cop ruling without refutation. (1) In `output_verification.rs` and `golden_output.rs`, CUDA init and GPU generation errors in the dense golden gate now yield `GpuGoldenLeg::Errored`, which fails the gate naming the error rather than skipping/hiding behind CPU pass. (2) Skips are strictly restricted to cases where the leg never starts (`!cuda_feature`, `!cuda_device`, or dense-loader on non-GGUF/hybrid). (3) In the runtime gate (`run_golden_output_gate_runtime`), the reported backend is measured directly from `InferenceResult::used_gpu`; any fallback to CPU when the GPU was expected fails the gate, and runs without a device are accurately labeled CPU. (4) All 14 unit tests in `golden_output_tests` pass cleanly under both default features and `--features cuda`, and `check_roadmap_completion_is_cited.sh` and `cargo fmt` pass. Verdict: PASS.", + "findings": [], + "raw_bytes": 3225, + "err_bytes": 100, + "envelope_status": "SUCCESS", + "verdict_source": "structured_output", + "grounding_check": "parity", + "model": "gemini-3.7-flash-high", + "model_measured": "gemini-3.7-flash-high", + "model_source": "measured", + "family": "gemini", + "role": "independent", + "fallback": { + "judged_by": "gemini-3.7-flash-high", + "exhausted": false, + "attempts": [ + { + "model": "gemini-3.7-flash-high", + "family": "gemini", + "outcome": "answered" + } + ], + "skipped": [] + } + } + ], + "dissent": [], + "dedup": [ + { + "file": "crates/apr-cli/src/commands/golden_output.rs", + "line": 673, + "lanes_agreeing": [ + 2 + ], + "claims": [ + "Case rows (tests) added to verify GPU errors, skips, correct generation, and wrong answers." + ] + }, + { + "file": "crates/apr-cli/src/commands/output_verification.rs", + "line": 531, + "lanes_agreeing": [ + 2 + ], + "claims": [ + "SKIP stays only for \"no GPU on this host / no cuda feature\" and names which, implemented cleanly in `gpu_golden_not_run`." + ] + }, + { + "file": "crates/apr-cli/src/commands/output_verification.rs", + "line": 554, + "lanes_agreeing": [ + 2 + ], + "claims": [ + "`run_golden_output_gate_runtime` correctly names the backend that actually ran based on `used_gpu` and fails if it unexpectedly falls back, fulfilling the Cop ruling." + ] + }, + { + "file": "crates/apr-cli/src/commands/output_verification.rs", + "line": 606, + "lanes_agreeing": [ + 2 + ], + "claims": [ + "GPU golden generation error correctly fails the gate naming the error, instead of skipping, through `GpuGoldenLeg::judge` mapping `Err` to `Errored` and `failure()` returning the error string." + ] + } + ], + "uncovered": [], + "coverage_source": "lanes", + "partial": false, + "partial_reasons": [ + "lane models: only 2 distinct model ids across 3 lanes (gemini-3.1-pro-high, gemini-3.7-flash-high) — lanes sharing an id are resamples, not independent reviewers (PMAT-125)", + "lane 1: grounding NOT COMPARED — structured_output holds a verdict but no single verdict object could be read out of .response (3329 raw bytes), so agy's labels stand unchecked for this lane (PMAT-082)", + "lane 2: judged by gemini-3.1-pro-high after falling through gpt-oss-120b-medium (capacity: UNAVAILABLE (code 503): No capacity available for model gpt-oss-120b-medium on the server); gpt-oss-120b-medium (capacity: UNAVAILABLE (code 503): No capacity available for model gpt-oss-120b-medium on the server) (PMAT-321)" + ], + "fallback": { + "same_family_width": 1, + "chain": [ + { + "model": "gemini-3.1-pro-high", + "family": "gemini", + "disposition": "configured" + }, + { + "model": "gpt-oss-120b-medium", + "family": "openai", + "disposition": "configured" + }, + { + "model": "gemini-3.7-flash-high", + "family": "gemini", + "disposition": "configured" + }, + { + "model": "qwen3.5", + "family": "qwen", + "disposition": "not-run", + "why": "no quorum.local_lane in the config — the aprender lane has no model to load" + }, + { + "model": "claude-opus-4-6-thinking", + "family": "claude", + "disposition": "width", + "why": "same family as the author: at most 1 lane, recorded role width, counted toward no floor (R-15a)" + }, + { + "model": "claude-sonnet-4-6", + "family": "claude", + "disposition": "width", + "why": "same family as the author: at most 1 lane, recorded role width, counted toward no floor (R-15a)" + } + ], + "precheck": [ + { + "family": "gemini", + "model": "gemini-3.1-pro-high", + "probe": 1, + "outcome": "live" + }, + { + "family": "openai", + "model": "gpt-oss-120b-medium", + "probe": 1, + "outcome": "live" + } + ], + "prah": { + "source": "install-receipt", + "path": "/home/noah/.claude/skills/paiml-implement/bin/prah" + } + }, + "auto_merge": { + "checked": false, + "was_armed": false, + "disarmed": false, + "note": "no --pr given: nothing to disarm" + }, + "lint": { + "ok": true, + "output": "receipt complete: kind=artifact lanes=3 author=claude-opus-5/claude" + } +} diff --git a/docs/audits/quorum-PMAT-3715.json b/docs/audits/quorum-PMAT-3715.json new file mode 100644 index 0000000000..a916791f9d --- /dev/null +++ b/docs/audits/quorum-PMAT-3715.json @@ -0,0 +1,187 @@ +{ + "ticket": "PMAT-3715", + "base": "origin/main", + "base_resolved": "origin/main", + "base_note": "no origin/origin/main exists; judged against the local ref", + "head": "84289cb7ba38adf96c0a1c5f836eb84ccb8d0646", + "diff_sha256": "9df958d287befbf36a6b50be55984468291262bf8a3cf2cf9507b584e2f5676b", + "width": 3, + "executor": "agy", + "prompt_mode": "file", + "prompt_bytes": 291117, + "author": { + "model": "claude-opus-5", + "family": "claude", + "source": "flag" + }, + "agreed": true, + "lanes": [ + { + "lane": 1, + "status": "SUCCESS", + "verdict": "PASS", + "summary": "The PR accurately and completely implements the PMAT-3715 release-readiness shape and its requested scope additions. I verified the graph extraction logic, the generation of KernelDiffReceipt and TokenizerParityReceipt nodes, the parsing of the GGUF `context_length` for declared rungs, and the correct verification of thinking mode consistency against `enable_thinking` and `thinking_markers`. The release-readiness-v1 shape rigorously checks for missing, skipped, or stale receipts and requires `fallback=false` without providing any waiver path. Furthermore, the `positive_control()` has been successfully added to `extract_controls()` to ensure the gate cannot ship without its control, perfectly fulfilling the ticket requirements without introducing any forbidden behavior.", + "findings": [], + "raw_bytes": 2809, + "err_bytes": 100, + "envelope_status": "SUCCESS", + "verdict_source": "structured_output", + "grounding_check": "parity", + "model": "gemini-3.1-pro-high", + "model_measured": "gemini-3.1-pro-high", + "model_source": "measured", + "family": "gemini", + "role": "independent", + "fallback": { + "judged_by": "gemini-3.1-pro-high", + "exhausted": false, + "attempts": [ + { + "model": "gemini-3.1-pro-high", + "family": "gemini", + "outcome": "answered" + } + ], + "skipped": [] + } + }, + { + "lane": 2, + "status": "SUCCESS", + "verdict": "PASS", + "summary": "The diff perfectly implements the requirements of PMAT-3715 and the receipt claims. The SHACL shape enforces the required matrix of cells, and all edge cases (missing, skipped, fallback, stale receipts, kernel architecture mismatches, tokenizer parity) are verified in the test suite to turn the gate red and name the cell. There are no doctrinal violations, weakened gates, or unauthorized scopes. The gate works exactly as requested.", + "findings": [], + "raw_bytes": 3819, + "err_bytes": 100, + "envelope_status": "SUCCESS", + "verdict_source": "structured_output", + "grounding_check": "parity", + "model": "gemini-3.1-pro-low", + "model_measured": "gemini-3.1-pro-low", + "model_source": "measured", + "family": "gemini", + "role": "independent", + "fallback": { + "judged_by": "gemini-3.1-pro-low", + "exhausted": false, + "attempts": [ + { + "model": "gemini-3.1-pro-low", + "family": "gemini", + "outcome": "answered" + } + ], + "skipped": [] + } + }, + { + "lane": 3, + "status": "SUCCESS", + "verdict": "PASS", + "summary": "The audit prompt file was fully read, the repository builds, and all tests pass. No issues found.", + "findings": [], + "raw_bytes": 1862, + "err_bytes": 100, + "envelope_status": "SUCCESS", + "verdict_source": "structured_output", + "model": "gpt-oss-120b-medium", + "model_measured": "gpt-oss-120b-medium", + "model_source": "measured", + "family": "openai", + "role": "independent", + "fallback": { + "judged_by": "gpt-oss-120b-medium", + "exhausted": false, + "attempts": [ + { + "model": "gpt-oss-120b-medium", + "family": "openai", + "outcome": "capacity", + "reason": "Our servers are experiencing high traffic right now, please try again in a minute. (UNAVAILABLE (code 503): No capacity available for model gpt-oss-120b-medium on the server)" + }, + { + "model": "gpt-oss-120b-medium", + "family": "openai", + "outcome": "answered" + } + ], + "skipped": [] + } + } + ], + "dissent": [], + "dedup": [], + "uncovered": [], + "coverage_source": "lanes", + "partial": false, + "partial_reasons": [ + "lane 3: grounding NOT COMPARED — structured_output holds a verdict but no single verdict object could be read out of .response (1862 raw bytes), so agy's labels stand unchecked for this lane (PMAT-082)", + "lane 3: judged by gpt-oss-120b-medium after falling through gpt-oss-120b-medium (capacity: Our servers are experiencing high traffic right now, please try again in a minute. (UNAVAILABLE (code 503): No capacity available for model gpt-oss-120b-medium on the server)) (PMAT-321)" + ], + "fallback": { + "same_family_width": 1, + "chain": [ + { + "model": "gemini-3.1-pro-high", + "family": "gemini", + "disposition": "configured" + }, + { + "model": "gemini-3.1-pro-low", + "family": "gemini", + "disposition": "configured" + }, + { + "model": "gpt-oss-120b-medium", + "family": "openai", + "disposition": "configured" + }, + { + "model": "qwen3.5", + "family": "qwen", + "disposition": "not-run", + "why": "no quorum.local_lane in the config — the aprender lane has no model to load" + }, + { + "model": "claude-opus-4-6-thinking", + "family": "claude", + "disposition": "width", + "why": "same family as the author: at most 1 lane, recorded role width, counted toward no floor (R-15a)" + }, + { + "model": "claude-sonnet-4-6", + "family": "claude", + "disposition": "width", + "why": "same family as the author: at most 1 lane, recorded role width, counted toward no floor (R-15a)" + } + ], + "precheck": [ + { + "family": "gemini", + "model": "gemini-3.1-pro-high", + "probe": 1, + "outcome": "live" + }, + { + "family": "openai", + "model": "gpt-oss-120b-medium", + "probe": 1, + "outcome": "live" + } + ], + "prah": { + "source": "install-receipt", + "path": "/home/noah/.claude/skills/paiml-implement/bin/prah" + } + }, + "auto_merge": { + "checked": true, + "was_armed": false, + "disarmed": false, + "note": "auto-merge not armed" + }, + "lint": { + "ok": true, + "output": "receipt complete: kind=artifact lanes=3 author=claude-opus-5/claude" + } +} diff --git a/docs/audits/quorum-PMAT-3717.json b/docs/audits/quorum-PMAT-3717.json new file mode 100644 index 0000000000..c1af26b223 --- /dev/null +++ b/docs/audits/quorum-PMAT-3717.json @@ -0,0 +1,194 @@ +{ + "ticket": "PMAT-3717", + "base": "629143eeb", + "base_resolved": "629143eeb", + "base_note": "no origin/629143eeb exists; judged against the local ref", + "head": "8a51105dc232b0bbafa5a32f9fdf3b41012a82d3", + "diff_sha256": "939b60b45e689dfe720200f8337017ccf9d0204542baeaa94dc5de37e203a6cb", + "width": 3, + "executor": "agy", + "prompt_mode": "inline", + "prompt_bytes": 85882, + "author": { + "model": "claude-opus-5", + "family": "claude", + "source": "flag" + }, + "agreed": true, + "lanes": [ + { + "lane": 1, + "status": "SUCCESS", + "verdict": "PASS", + "summary": "The diff successfully implements the requirements of PMAT-3717 without any contradictions or gate weakenings. \n\nIt correctly introduces the `models` step at T-1 in `autopilot.sh` immediately before `tag`, enforcing NO-GO behavior on failure (e.g. unreachable host, failed build, missing/red receipts, or judge decline).\n\nThe gx10 remote execution leg observes instrument isolation strictly per the ticket amendment: it utilizes a derived per-release directory (`$HOME/.cache/aprender-release/rel-`), deletes previous release directories, and checks the remote free space prior to building, failing gracefully with an ENV error if insufficient. It never uses the shared `~/src/aprender/target`.\n\nIt satisfies the GPU-sharing requirement by executing the model ladder under `choom -n 1000 --` on both hosts, correctly avoiding a redundant wrapper `flock` since the ladder handles locking per `apr` call. \n\nIn `check_publish_preflight.sh`, the newly added `rule_r7` correctly re-reads the committed receipts at T-4 by executing the same T-1 judge.\n\nThe provided test suite (`scripts/check_release_models_t1.sh`) exhaustively drives the `models_t1.sh` and T-4 R7 via stubs, covering all failure rows and successfully failing against 16 mutants, verifying that every safeguard is actively enforced.", + "findings": [], + "raw_bytes": 3903, + "err_bytes": 0, + "envelope_status": "SUCCESS", + "verdict_source": "structured_output", + "grounding_check": "parity", + "model": "gemini-3.1-pro-high", + "model_measured": "gemini-3.1-pro-high", + "model_source": "measured", + "family": "gemini", + "role": "independent", + "fallback": { + "judged_by": "gemini-3.1-pro-high", + "exhausted": false, + "attempts": [ + { + "model": "gemini-3.1-pro-high", + "family": "gemini", + "outcome": "answered" + } + ], + "skipped": [] + } + }, + { + "lane": 2, + "status": "SUCCESS", + "verdict": "PASS", + "summary": "I have reviewed the diff against the PMAT-3717 acceptance criteria, amendments, and GPU-sharing ruling. I found no contradictions or missed requirements.\n\n1. **Before the tag:** `autopilot.sh` correctly runs `models` right before `tag` and `die`s if it returns a non-zero exit code.\n2. **Instrument isolation:** The gx10 leg builds into a dynamically derived `$HOME/.cache/aprender-release/rel-` directory, correctly checks disk space before attempting to build, explicitly defines `CARGO_TARGET_DIR`, and ensures older `rel-*` directories are removed. It completely avoids `~/src/aprender/target`.\n3. **Receipt apr_version proof:** `models_t1.sh` asserts `apr --version` produces `apr ()` directly from the newly built binary before executing `model_ladder.sh`. It then explicitly validates the returned receipt's `apr_version` field.\n4. **T-4 Re-read:** `check_publish_preflight.sh` properly implements R7 to invoke `check_model_ladder.sh --version ` at T-4 without `--receipts` so it defaults to the committed receipts in `evidence/dogfood/models/`. It correctly respects the frozen interface and fails the gate if the judge finds red or declines.\n5. **GPU-Sharing lock / choom:** `choom -n 1000 --` wraps the `model_ladder.sh` invocations on both lambda and gx10 legs. There is no `flock` in the wrapper.\n6. **Red-turning tests:** `scripts/check_release_models_t1.sh` covers all required scenarios: unreachable host, build failure, missing receipt, red receipt, judge decline, disk refusal, stale binary, and the choom/flock rules. The corresponding mutants verify all NO-GO rules and failure parsing paths are strictly enforced. R7 rules are tested via isolated T-4 rows extracting `rule_r7`.", + "findings": [], + "raw_bytes": 4834, + "err_bytes": 0, + "envelope_status": "SUCCESS", + "verdict_source": "structured_output", + "grounding_check": "parity", + "model": "gemini-3.1-pro-high", + "model_measured": "gemini-3.1-pro-high", + "model_source": "measured", + "family": "gemini", + "role": "independent", + "fallback": { + "judged_by": "gemini-3.1-pro-high", + "exhausted": false, + "attempts": [ + { + "model": "gpt-oss-120b-medium", + "family": "openai", + "outcome": "capacity", + "reason": "UNAVAILABLE (code 503): No capacity available for model gpt-oss-120b-medium on the server" + }, + { + "model": "gpt-oss-120b-medium", + "family": "openai", + "outcome": "capacity", + "reason": "UNAVAILABLE (code 503): No capacity available for model gpt-oss-120b-medium on the server" + }, + { + "model": "gemini-3.1-pro-high", + "family": "gemini", + "outcome": "answered" + } + ], + "skipped": [] + } + }, + { + "lane": 3, + "status": "SUCCESS", + "verdict": "PASS", + "summary": "I have reviewed the diff against the PMAT-3717 ticket requirements and found that it strictly adheres to all acceptance criteria.\n\n- **T-1 model matrix step before tag**: A `models` step has been appropriately added to `scripts/release/autopilot.sh` right before the `tag` step. It executes `scripts/release/models_t1.sh` and correctly terminates the release process without tagging upon any non-zero exit code.\n- **Instrument isolation on gx10**: `models_t1.sh` sets up a dedicated, per-release directory (`$HOME/.cache/aprender-release/rel-$ver`) on the remote host, explicitly avoiding the shared `~/src/aprender/target` directory. It clears previous `rel-*` directories and successfully validates available disk space upfront.\n- **T-4 receipt re-read**: `scripts/check_publish_preflight.sh` adds rule `R7` which utilizes `check_model_ladder.sh` to judge the committed receipts at T-4, handling exit code 2 (decline) accurately.\n- **Red-turning tests and mutations**: `scripts/check_release_models_t1.sh` provides exhaustive coverage for the specified scenarios (green-pair, gx10-unreachable, build-fails, etc.) including corresponding mutants that verify proper failure detection. \n- **GPU-sharing requirement**: The models script respects the one-owner lock policy by running the ladder under `choom -n 1000 --` without taking any extra `flock`.\n\nNo issues, weakening of gates, or missing requirements were found.", + "findings": [], + "raw_bytes": 4108, + "err_bytes": 0, + "envelope_status": "SUCCESS", + "verdict_source": "structured_output", + "grounding_check": "parity", + "model": "gemini-3.1-pro-low", + "model_measured": "gemini-3.1-pro-low", + "model_source": "measured", + "family": "gemini", + "role": "independent", + "fallback": { + "judged_by": "gemini-3.1-pro-low", + "exhausted": false, + "attempts": [ + { + "model": "gemini-3.1-pro-low", + "family": "gemini", + "outcome": "answered" + } + ], + "skipped": [] + } + } + ], + "dissent": [], + "dedup": [], + "uncovered": [], + "coverage_source": "lanes", + "partial": false, + "partial_reasons": [ + "lane models: only 2 distinct model ids across 3 lanes (gemini-3.1-pro-high, gemini-3.1-pro-low) — lanes sharing an id are resamples, not independent reviewers (PMAT-125)", + "lane 2: judged by gemini-3.1-pro-high after falling through gpt-oss-120b-medium (capacity: UNAVAILABLE (code 503): No capacity available for model gpt-oss-120b-medium on the server); gpt-oss-120b-medium (capacity: UNAVAILABLE (code 503): No capacity available for model gpt-oss-120b-medium on the server) (PMAT-321)" + ], + "fallback": { + "same_family_width": 1, + "chain": [ + { + "model": "gemini-3.1-pro-high", + "family": "gemini", + "disposition": "configured" + }, + { + "model": "gpt-oss-120b-medium", + "family": "openai", + "disposition": "configured" + }, + { + "model": "gemini-3.1-pro-low", + "family": "gemini", + "disposition": "configured" + }, + { + "model": "qwen3.5", + "family": "qwen", + "disposition": "not-run", + "why": "no quorum.local_lane in the config — the aprender lane has no model to load" + }, + { + "model": "claude-opus-4-6-thinking", + "family": "claude", + "disposition": "width", + "why": "same family as the author: at most 1 lane, recorded role width, counted toward no floor (R-15a)" + }, + { + "model": "claude-sonnet-4-6", + "family": "claude", + "disposition": "width", + "why": "same family as the author: at most 1 lane, recorded role width, counted toward no floor (R-15a)" + } + ], + "precheck": [ + { + "family": "gemini", + "model": "gemini-3.1-pro-high", + "probe": 1, + "outcome": "live" + }, + { + "family": "openai", + "model": "gpt-oss-120b-medium", + "probe": 1, + "outcome": "live" + } + ], + "prah": { + "source": "install-receipt", + "path": "/home/noah/.claude/skills/paiml-implement/bin/prah" + } + }, + "auto_merge": { + "checked": false, + "was_armed": false, + "disarmed": false, + "note": "no --pr given: nothing to disarm" + }, + "lint": { + "ok": true, + "output": "receipt complete: kind=artifact lanes=3 author=claude-opus-5/claude" + } +} diff --git a/docs/audits/quorum-PMAT-3718.json b/docs/audits/quorum-PMAT-3718.json new file mode 100644 index 0000000000..4c907d2ced --- /dev/null +++ b/docs/audits/quorum-PMAT-3718.json @@ -0,0 +1,241 @@ +{ + "ticket": "PMAT-3718", + "base": "225b2a9ab", + "base_resolved": "225b2a9ab", + "base_note": "no origin/225b2a9ab exists; judged against the local ref", + "head": "187b390e1dcfcd6c565d3f27aa779c06a60f8c82", + "diff_sha256": "20f9de301805687f8fb0e6c0853636df2b1aa308c0fbc03c28fe5ff343120b8f", + "width": 3, + "executor": "agy", + "prompt_mode": "inline", + "prompt_bytes": 65794, + "author": { + "model": "claude-opus-5", + "family": "claude", + "source": "flag" + }, + "agreed": true, + "lanes": [ + { + "lane": 1, + "status": "SUCCESS", + "verdict": "PASS", + "summary": "The diff successfully implements PMAT-3718, correctly adding `prompt_tokens`, `completion_tokens`, and `finish_reason` to `apr run --json` and fixing `apr serve` endpoints to accurately report truncations. It resolves the silent stream cuts for clamped requests by judging the finish against the actual ran budget, maps unfittable prompts to a proper 400 error, and stops the APR CPU handler from echoing the prompt when no tokens are generated. The implementation conforms to all acceptance criteria and the provided receipt claims.", + "findings": [ + { + "claim": "Adds accurate token counts and finish_reason to CLI usage output using the actual engine count.", + "file": "crates/apr-cli/src/commands/inference_output.rs", + "grounding": "cited", + "line": 184 + }, + { + "claim": "Shadows max_tokens with clamped budget for correct length evaluation, and maps unfittable prompt errors to a 400 response.", + "file": "crates/aprender-serve/src/api/cuda_chat_backend.rs", + "grounding": "cited", + "line": 322 + }, + { + "claim": "Implements budget-aware finish reason determination correctly handling clamped runs and intrinsic stop token consumption.", + "file": "crates/aprender-serve/src/infer/run_report.rs", + "grounding": "cited", + "line": 39 + }, + { + "claim": "Fixes APR CPU fallback to yield empty instead of the prompt when nothing is generated, and uses `from_decode` to determine stop reason accurately.", + "file": "crates/apr-cli/src/commands/serve/handlers.rs", + "grounding": "cited", + "line": 1129 + } + ], + "raw_bytes": 4150, + "err_bytes": 0, + "envelope_status": "SUCCESS", + "verdict_source": "structured_output", + "grounding_check": "parity", + "model": "gemini-3.1-pro-high", + "model_measured": "gemini-3.1-pro-high", + "model_source": "measured", + "family": "gemini", + "role": "independent", + "fallback": { + "judged_by": "gemini-3.1-pro-high", + "exhausted": false, + "attempts": [ + { + "model": "gemini-3.1-pro-high", + "family": "gemini", + "outcome": "answered" + } + ], + "skipped": [] + } + }, + { + "lane": 2, + "status": "SUCCESS", + "verdict": "PASS", + "summary": "The diff strictly conforms to the acceptance criteria of PMAT-3718 and makes no unauthorized changes. `RunUsage` is implemented and emitted by `apr run --json` with `null` fallbacks. `FinishReason` correctly distinguishes cuts from stops via `from_decode`, fixing silent cuts in both CPU non-streaming and streaming `serve` paths. The context-clamped budgets are correctly used to evaluate the finish reason for the dense CPU loop, and over-long prompts now return a 400 error. The `a_fixed_text_prompt_counts_the_tokens_the_tokenizer_feeds` test adequately verifies the token count via the newly exposed `build_executable_pygmy_gguf_with` metadata hook. All receipt claims are backed by the code and tests.", + "findings": [], + "raw_bytes": 2788, + "err_bytes": 100, + "envelope_status": "SUCCESS", + "verdict_source": "structured_output", + "grounding_check": "parity", + "model": "gemini-3.1-pro-low", + "model_measured": "gemini-3.1-pro-low", + "model_source": "measured", + "family": "gemini", + "role": "independent", + "fallback": { + "judged_by": "gemini-3.1-pro-low", + "exhausted": false, + "attempts": [ + { + "model": "gemini-3.1-pro-low", + "family": "gemini", + "outcome": "answered" + } + ], + "skipped": [] + } + }, + { + "lane": 3, + "status": "SUCCESS", + "verdict": "PASS", + "summary": "The diff perfectly implements the requested features from PMAT-3718. It adds `prompt_tokens`, `completion_tokens`, `finish_reason`, and `context_length` to `apr run --json` using the new `RunUsage` structure and the `run_inference_report` hook, returning `null` when a specific backend does not report these metrics. \nFor `apr serve`, it correctly calculates the `finish_reason` in `/v1/chat/completions` against the effective clamped budget rather than the requested one, preventing context-clamped responses from being falsely reported as `stop`. It correctly throws a 400 error (instead of truncating or throwing a 500 after the fact) when the prompt exceeds the context window. \nThe comprehensive test additions and updates enforce these behaviors properly, including the exact `a_fixed_text_prompt_counts_the_tokens_the_tokenizer_feeds` case row required by the ticket's done criteria.", + "findings": [], + "raw_bytes": 3040, + "err_bytes": 0, + "envelope_status": "SUCCESS", + "verdict_source": "structured_output", + "grounding_check": "parity", + "model": "gemini-3.1-pro-high", + "model_measured": "gemini-3.1-pro-high", + "model_source": "measured", + "family": "gemini", + "role": "independent", + "fallback": { + "judged_by": "gemini-3.1-pro-high", + "exhausted": false, + "attempts": [ + { + "model": "gemini-3.1-pro-high", + "family": "gemini", + "outcome": "answered" + } + ], + "skipped": [] + } + } + ], + "dissent": [], + "dedup": [ + { + "file": "crates/apr-cli/src/commands/inference_output.rs", + "line": 184, + "lanes_agreeing": [ + 1 + ], + "claims": [ + "Adds accurate token counts and finish_reason to CLI usage output using the actual engine count." + ] + }, + { + "file": "crates/apr-cli/src/commands/serve/handlers.rs", + "line": 1129, + "lanes_agreeing": [ + 1 + ], + "claims": [ + "Fixes APR CPU fallback to yield empty instead of the prompt when nothing is generated, and uses `from_decode` to determine stop reason accurately." + ] + }, + { + "file": "crates/aprender-serve/src/api/cuda_chat_backend.rs", + "line": 322, + "lanes_agreeing": [ + 1 + ], + "claims": [ + "Shadows max_tokens with clamped budget for correct length evaluation, and maps unfittable prompt errors to a 400 response." + ] + }, + { + "file": "crates/aprender-serve/src/infer/run_report.rs", + "line": 39, + "lanes_agreeing": [ + 1 + ], + "claims": [ + "Implements budget-aware finish reason determination correctly handling clamped runs and intrinsic stop token consumption." + ] + } + ], + "uncovered": [], + "coverage_source": "lanes", + "partial": false, + "partial_reasons": [ + "lane models: only 2 distinct model ids across 3 lanes (gemini-3.1-pro-high, gemini-3.1-pro-low) — lanes sharing an id are resamples, not independent reviewers (PMAT-125)" + ], + "fallback": { + "same_family_width": 1, + "chain": [ + { + "model": "gemini-3.1-pro-high", + "family": "gemini", + "disposition": "configured" + }, + { + "model": "gemini-3.1-pro-low", + "family": "gemini", + "disposition": "configured" + }, + { + "model": "gpt-oss-120b-medium", + "family": "openai", + "disposition": "fallback" + }, + { + "model": "qwen3.5", + "family": "qwen", + "disposition": "not-run", + "why": "no quorum.local_lane in the config — the aprender lane has no model to load" + }, + { + "model": "claude-opus-4-6-thinking", + "family": "claude", + "disposition": "width", + "why": "same family as the author: at most 1 lane, recorded role width, counted toward no floor (R-15a)" + }, + { + "model": "claude-sonnet-4-6", + "family": "claude", + "disposition": "width", + "why": "same family as the author: at most 1 lane, recorded role width, counted toward no floor (R-15a)" + } + ], + "precheck": [ + { + "family": "gemini", + "model": "gemini-3.1-pro-high", + "probe": 1, + "outcome": "live" + } + ], + "prah": { + "source": "install-receipt", + "path": "/home/noah/.claude/skills/paiml-implement/bin/prah" + } + }, + "auto_merge": { + "checked": false, + "was_armed": false, + "disarmed": false, + "note": "no --pr given: nothing to disarm" + }, + "lint": { + "ok": true, + "output": "receipt complete: kind=artifact lanes=3 author=claude-opus-5/claude" + } +} diff --git a/docs/audits/quorum-PMAT-3726.json b/docs/audits/quorum-PMAT-3726.json new file mode 100644 index 0000000000..a2ef9ab3bc --- /dev/null +++ b/docs/audits/quorum-PMAT-3726.json @@ -0,0 +1,186 @@ +{ + "ticket": "PMAT-3726", + "base": "262dfb2f6", + "base_resolved": "262dfb2f6", + "base_note": "no origin/262dfb2f6 exists; judged against the local ref", + "head": "585c9a67bd2e3fd36e73aa4548ed633c9a8c820c", + "diff_sha256": "51861c65831e3a0012715fcb847c3b881d6da36f345fe7919f3750d868b4061a", + "width": 3, + "executor": "agy", + "prompt_mode": "file", + "prompt_bytes": 144046, + "author": { + "model": "claude-opus-5", + "family": "claude", + "source": "flag" + }, + "agreed": true, + "lanes": [ + { + "lane": 1, + "status": "SUCCESS", + "verdict": "PASS", + "summary": "The diff implements the canonical byte-level BPE tokenizer (pre-tokenizer + ranked merges) for models with a 'gpt2' tokenizer model, replacing the buggy greedy longest-match tokenizer that corrupted copy tasks (#3693). It correctly implements the qwen2/qwen35 pre-tokenizers, the GPT-2 byte-to-unicode map, and properly extracts special tokens using token_types. A thorough tokenizer parity gate is added and configured to run against llama-tokenize to ensure exact parity with llama.cpp, which effectively satisfies all acceptance criteria in PMAT-3726. I found no rule violations or refuting evidence; the receipts back the claim.", + "findings": [], + "raw_bytes": 2629, + "err_bytes": 0, + "envelope_status": "SUCCESS", + "verdict_source": "structured_output", + "grounding_check": "parity", + "model": "gemini-3.1-pro-high", + "model_measured": "gemini-3.1-pro-high", + "model_source": "measured", + "family": "gemini", + "role": "independent", + "fallback": { + "judged_by": "gemini-3.1-pro-high", + "exhausted": false, + "attempts": [ + { + "model": "gemini-3.1-pro-high", + "family": "gemini", + "outcome": "answered" + } + ], + "skipped": [] + } + }, + { + "lane": 2, + "status": "SUCCESS", + "verdict": "PASS", + "summary": "The diff accurately implements the canonical byte-level BPE tokenizer ported from llama.cpp. It successfully includes the new `ByteLevelBpe` implementation handling regex splits according to `tokenizer.ggml.pre` and correctly performs byte-to-unicode mappings. It also adds a tokenizer parity testing framework (`scripts/tokenizer_parity.sh` and `check_tokenizer_parity.sh`) which performs identical roundtrip encoding verifications, exactly as demanded by the ticket. All receipts correspond with the logic in the tests, and changes to coverage numbers correctly mirror the increase in tracked files. No rules or assertions were weakened.", + "findings": [], + "raw_bytes": 2657, + "err_bytes": 0, + "envelope_status": "SUCCESS", + "verdict_source": "structured_output", + "grounding_check": "parity", + "model": "gemini-3.1-pro-low", + "model_measured": "gemini-3.1-pro-low", + "model_source": "measured", + "family": "gemini", + "role": "independent", + "fallback": { + "judged_by": "gemini-3.1-pro-low", + "exhausted": false, + "attempts": [ + { + "model": "gemini-3.1-pro-low", + "family": "gemini", + "outcome": "answered" + } + ], + "skipped": [] + } + }, + { + "lane": 3, + "status": "SUCCESS", + "verdict": "PASS", + "summary": "All review criteria pass; no refutations found in the diff.", + "findings": [], + "raw_bytes": 1374, + "err_bytes": 0, + "envelope_status": "SUCCESS", + "verdict_source": "structured_output", + "grounding_check": "parity", + "model": "gpt-oss-120b-medium", + "model_measured": "gpt-oss-120b-medium", + "model_source": "measured", + "family": "openai", + "role": "independent", + "fallback": { + "judged_by": "gpt-oss-120b-medium", + "exhausted": false, + "attempts": [ + { + "model": "gpt-oss-120b-medium", + "family": "openai", + "outcome": "answered" + } + ], + "skipped": [] + } + } + ], + "dissent": [], + "dedup": [], + "uncovered": [], + "coverage_source": "lanes", + "partial": false, + "partial_reasons": [], + "fallback": { + "same_family_width": 1, + "chain": [ + { + "model": "gemini-3.1-pro-high", + "family": "gemini", + "disposition": "configured" + }, + { + "model": "gemini-3.1-pro-low", + "family": "gemini", + "disposition": "configured" + }, + { + "model": "gpt-oss-120b-medium", + "family": "openai", + "disposition": "configured" + }, + { + "model": "qwen3.5", + "family": "qwen", + "disposition": "not-run", + "why": "no quorum.local_lane in the config — the aprender lane has no model to load" + }, + { + "model": "claude-opus-4-6-thinking", + "family": "claude", + "disposition": "width", + "why": "same family as the author: at most 1 lane, recorded role width, counted toward no floor (R-15a)" + }, + { + "model": "claude-sonnet-4-6", + "family": "claude", + "disposition": "width", + "why": "same family as the author: at most 1 lane, recorded role width, counted toward no floor (R-15a)" + } + ], + "precheck": [ + { + "family": "gemini", + "model": "gemini-3.1-pro-high", + "probe": 1, + "outcome": "live" + }, + { + "family": "openai", + "model": "gpt-oss-120b-medium", + "probe": 1, + "outcome": "capacity", + "reason": "UNAVAILABLE (code 503): No capacity available for model gpt-oss-120b-medium on the server" + }, + { + "family": "openai", + "model": "gpt-oss-120b-medium", + "probe": 2, + "outcome": "live" + } + ], + "prah": { + "source": "install-receipt", + "path": "/home/noah/.claude/skills/paiml-implement/bin/prah" + } + }, + "auto_merge": { + "checked": false, + "was_armed": false, + "disarmed": false, + "note": "no --pr given: nothing to disarm" + }, + "lint": { + "ok": true, + "output": "receipt complete: kind=artifact lanes=3 author=claude-opus-5/claude" + } +} diff --git a/docs/audits/quorum-PMAT-3733.json b/docs/audits/quorum-PMAT-3733.json new file mode 100644 index 0000000000..57c43acfd9 --- /dev/null +++ b/docs/audits/quorum-PMAT-3733.json @@ -0,0 +1,178 @@ +{ + "ticket": "PMAT-3733", + "base": "52f43da71", + "base_resolved": "52f43da71", + "base_note": "no origin/52f43da71 exists; judged against the local ref", + "head": "7dad33b28bd31f07b0efbea6ef65c1eb872b44aa", + "diff_sha256": "8ab6f85d14ef1d6954f4ec6fd5cb506c4f06a3191c2587c401fc7437ce006438", + "width": 3, + "executor": "agy", + "prompt_mode": "inline", + "prompt_bytes": 31205, + "author": { + "model": "claude-opus-5", + "family": "claude", + "source": "flag" + }, + "agreed": true, + "lanes": [ + { + "lane": 1, + "status": "SUCCESS", + "verdict": "PASS", + "summary": "The diff comprehensively and accurately implements PMAT-3733. By parsing all metadata keys and placing non-allowlisted ones into `display_only_metadata`, the diff successfully ensures `apr inspect --json` yields every KV from the GGUF header without perturbing `metadata` mappings that downstream config accessors expect. The related refactoring for CLI testing (`rosetta_json`) and the essential array-allocation hardening correctly resolve the ticket's architectural implications. All test expectations mentioned in the ticket notes are met in `api_tests_all_keys_3733.rs` and `inspect_tests.rs`.", + "findings": [], + "raw_bytes": 2471, + "err_bytes": 0, + "envelope_status": "SUCCESS", + "verdict_source": "structured_output", + "grounding_check": "parity", + "model": "gemini-3.1-pro-high", + "model_measured": "gemini-3.1-pro-high", + "model_source": "measured", + "family": "gemini", + "role": "independent", + "fallback": { + "judged_by": "gemini-3.1-pro-high", + "exhausted": false, + "attempts": [ + { + "model": "gemini-3.1-pro-high", + "family": "gemini", + "outcome": "answered" + } + ], + "skipped": [] + } + }, + { + "lane": 2, + "status": "SUCCESS", + "verdict": "PASS", + "summary": "The diff successfully implements PMAT-3733. It ensures that every GGUF metadata key is parsed and retained without an arch-prefixed allowlist. Keys outside the configuration allowlist are safely directed to `display_only_metadata` so they appear in `apr inspect --json` without unexpectedly widening the model configuration path. The RosettaStone inspection path is correctly traced and tested, and array parsing is hardened against out-of-bounds/out-of-memory errors on hostile inputs. Tests verifying the KV counts and the unknown architecture prefix (`zzz_never_heard_of`) are present and complete.", + "findings": [], + "raw_bytes": 2465, + "err_bytes": 0, + "envelope_status": "SUCCESS", + "verdict_source": "structured_output", + "grounding_check": "parity", + "model": "gemini-3.1-pro-low", + "model_measured": "gemini-3.1-pro-low", + "model_source": "measured", + "family": "gemini", + "role": "independent", + "fallback": { + "judged_by": "gemini-3.1-pro-low", + "exhausted": false, + "attempts": [ + { + "model": "gemini-3.1-pro-low", + "family": "gemini", + "outcome": "answered" + } + ], + "skipped": [] + } + }, + { + "lane": 3, + "status": "SUCCESS", + "verdict": "PASS", + "summary": "The diff fully and accurately implements PMAT-3733. By parsing all GGUF metadata keys and storing non-allowlisted keys in `display_only_metadata`, all header keys (including `qwen35.*`, `quantize.*`, and arbitrary architecture keys such as `zzz_never_heard_of.*`) reach `apr inspect --json` via RosettaStone inspection without unexpectedly exposing them to configuration accessors (e.g. `context_length()`). Array parsing in `read_metadata_array` was properly hardened against hostile counts to prevent OOM or panics. All tests in `aprender-core` (645 passed) and `apr-cli` (140 passed) were measured and confirmed passing as claimed.", + "findings": [], + "raw_bytes": 5187, + "err_bytes": 0, + "envelope_status": "SUCCESS", + "verdict_source": "structured_output", + "grounding_check": "parity", + "model": "gemini-3.8-flash-high", + "model_measured": "gemini-3.8-flash-high", + "model_source": "measured", + "family": "gemini", + "role": "independent", + "fallback": { + "judged_by": "gemini-3.8-flash-high", + "exhausted": false, + "attempts": [ + { + "model": "gemini-3.8-flash-high", + "family": "gemini", + "outcome": "answered" + } + ], + "skipped": [] + } + } + ], + "dissent": [], + "dedup": [], + "uncovered": [], + "coverage_source": "lanes", + "partial": false, + "partial_reasons": [], + "fallback": { + "same_family_width": 1, + "chain": [ + { + "model": "gemini-3.1-pro-high", + "family": "gemini", + "disposition": "configured" + }, + { + "model": "gemini-3.1-pro-low", + "family": "gemini", + "disposition": "configured" + }, + { + "model": "gemini-3.8-flash-high", + "family": "gemini", + "disposition": "configured" + }, + { + "model": "gpt-oss-120b-medium", + "family": "openai", + "disposition": "fallback" + }, + { + "model": "qwen3.5", + "family": "qwen", + "disposition": "not-run", + "why": "no quorum.local_lane in the config — the aprender lane has no model to load" + }, + { + "model": "claude-opus-4-6-thinking", + "family": "claude", + "disposition": "width", + "why": "same family as the author: at most 1 lane, recorded role width, counted toward no floor (R-15a)" + }, + { + "model": "claude-sonnet-4-6", + "family": "claude", + "disposition": "width", + "why": "same family as the author: at most 1 lane, recorded role width, counted toward no floor (R-15a)" + } + ], + "precheck": [ + { + "family": "gemini", + "model": "gemini-3.1-pro-high", + "probe": 1, + "outcome": "live" + } + ], + "prah": { + "source": "install-receipt", + "path": "/home/noah/.claude/skills/paiml-implement/bin/prah" + } + }, + "auto_merge": { + "checked": false, + "was_armed": false, + "disarmed": false, + "note": "no --pr given: nothing to disarm" + }, + "lint": { + "ok": true, + "output": "receipt complete: kind=artifact lanes=3 author=claude-opus-5/claude" + } +} diff --git a/docs/audits/quorum-PMAT-3747.json b/docs/audits/quorum-PMAT-3747.json new file mode 100644 index 0000000000..aa4c9ad875 --- /dev/null +++ b/docs/audits/quorum-PMAT-3747.json @@ -0,0 +1,430 @@ +{ + "ticket": "PMAT-3747", + "base": "52f43da71", + "base_resolved": "52f43da71", + "base_note": "no origin/52f43da71 exists; judged against the local ref", + "head": "c5047bd942f5b7f76946fec7aabf9c2f7c70e528", + "diff_sha256": "a8382b71c09386d50431ddcc97fccbddab4ebc27aa8408688745186ec1e499b7", + "width": 3, + "executor": "agy", + "prompt_mode": "inline", + "prompt_bytes": 84589, + "author": { + "model": "claude-opus-5", + "family": "claude", + "source": "flag" + }, + "agreed": true, + "lanes": [ + { + "lane": 1, + "status": "SUCCESS", + "verdict": "PASS", + "summary": "The diff faithfully implements all the requirements of PMAT-3747 without violating the negative constraints. The `TokenConstraint` trait and the `llguidance`-powered engine are introduced correctly and hidden behind the `structured-output` feature. The `load_schema` function appropriately handles inline JSON schemas or file paths via `@path` prefix. The `GGUFModel::constraint_vocab` securely maps `decode()`'s token logic while ensuring control tokens never manifest in constrained text. The test suite thoroughly verifies the constraint engine behavior with RAH's quorum-lane and rmedia's schemas as per the ticket, and the mask overhead evaluation is securely included as an example. All criteria are fully met.", + "findings": [ + { + "claim": "`constrain::TokenConstraint` {mask, accept, is_complete} exists", + "file": "crates/aprender-serve/src/constrain/mod.rs", + "grounding": "cited", + "line": 35 + }, + { + "claim": "`ConstraintError` names every refusal (SchemaInvalid, SchemaUnsupported, DeadEnd, Rejected, Vocab, NotCompiled), each rendered with its name first", + "file": "crates/aprender-serve/src/constrain/mod.rs", + "grounding": "cited", + "line": 51 + }, + { + "claim": "`load_schema` takes the schema inline OR as `@path` and refuses non-JSON, non-schema JSON and unreadable paths as SchemaInvalid, before any token", + "file": "crates/aprender-serve/src/constrain/mod.rs", + "grounding": "cited", + "line": 114 + }, + { + "claim": "llguidance 1.8.0 sits behind the trait, as an optional dependency behind a feature that apr-cli's `inference` enables", + "file": "crates/aprender-serve/Cargo.toml", + "grounding": "cited", + "line": 113 + }, + { + "claim": "`GGUFModel::constraint_vocab()` builds each token's bytes from the SAME per-token decoding `decode()` uses; control/unused tokens are special and can never be constrained text", + "file": "crates/aprender-serve/src/gguf/token.rs", + "grounding": "cited", + "line": 307 + }, + { + "claim": "case table: RAH's quorum-lane schema and rmedia's three schemas (#3716) compile exactly, accept their real documents, refuse violations at the byte, and allow nothing after one complete value; an approximation is refused; without the feature a constraint is refused (NotCompiled), never ignored", + "file": "crates/aprender-serve/src/constrain/tests.rs", + "grounding": "cited", + "line": 312 + }, + { + "claim": "a measured per-token mask overhead on the Qwen3.5-9B vocabulary is provided as an example", + "file": "crates/aprender-serve/examples/constrain_mask_overhead.rs", + "grounding": "cited", + "line": 33 + }, + { + "claim": "No generation loop is wired (apr run's CPU paths, etc. are NOT in this PR)", + "file": "crates/aprender-serve/src/constrain/mod.rs", + "grounding": "asserted", + "line": 1 + } + ], + "raw_bytes": 6536, + "err_bytes": 0, + "envelope_status": "SUCCESS", + "verdict_source": "structured_output", + "grounding_check": "parity", + "model": "gemini-3.1-pro-high", + "model_measured": "gemini-3.1-pro-high", + "model_source": "measured", + "family": "gemini", + "role": "independent", + "fallback": { + "judged_by": "gemini-3.1-pro-high", + "exhausted": false, + "attempts": [ + { + "model": "gemini-3.1-pro-high", + "family": "gemini", + "outcome": "answered" + } + ], + "skipped": [] + } + }, + { + "lane": 2, + "status": "SUCCESS", + "verdict": "PASS", + "summary": "Reviewed diff (52f43da71...HEAD) for ticket PMAT-3747 (#3568 PR 1/4). Verified that all acceptance criteria are fully met, no gates are weakened, no tests assert opposite behavior, no receipt claims are unbacked, and no generation loop is wired. Empirically verified with cargo test (24/24 passed with feature, 12/12 without feature), cargo check -p apr-cli --features inference (passed), cargo fmt (clean), cargo clippy (-D warnings clean), and cargo deny check (advisories ok, bans ok, licenses ok, sources ok). Verdict: PASS.", + "findings": [ + { + "claim": "`TokenConstraint` trait defines `mask`, `accept`, and `is_complete`", + "file": "crates/aprender-serve/src/constrain/mod.rs", + "grounding": "cited", + "line": 35 + }, + { + "claim": "`ConstraintError` defines all 6 refusal variants (SchemaInvalid, SchemaUnsupported, DeadEnd, Rejected, Vocab, NotCompiled) with Display prefix formatting matching ticket spec", + "file": "crates/aprender-serve/src/constrain/mod.rs", + "grounding": "cited", + "line": 50 + }, + { + "claim": "`load_schema` parses inline JSON schema or `@path` schema file, refusing unreadable paths, non-JSON text, and non-schema JSON as `SchemaInvalid`", + "file": "crates/aprender-serve/src/constrain/mod.rs", + "grounding": "cited", + "line": 112 + }, + { + "claim": "`llguidance 1.8.0` is an optional dependency under `structured-output` feature, and `cargo deny check` (licenses, advisories, bans, sources) passes cleanly", + "file": "crates/aprender-serve/Cargo.toml", + "grounding": "measured", + "line": 111 + }, + { + "claim": "`apr-cli`'s `inference` feature enables `realizar/structured-output`", + "file": "crates/apr-cli/Cargo.toml", + "grounding": "cited", + "line": 88 + }, + { + "claim": "`GGUFModel::constraint_vocab()` extracts raw token bytes using `push_token_bytes` (sharing logic with `decode()`) and marks control/unused token types as special", + "file": "crates/aprender-serve/src/gguf/token.rs", + "grounding": "cited", + "line": 316 + }, + { + "claim": "Case table tests pass with 24/24 tests green under `--features structured-output` and 12/12 tests green without the feature", + "file": "crates/aprender-serve/src/constrain/tests.rs", + "grounding": "measured", + "line": 304 + }, + { + "claim": "Per-token constraint overhead example `constrain_mask_overhead.rs` is implemented and builds cleanly under `--features structured-output`", + "file": "crates/aprender-serve/examples/constrain_mask_overhead.rs", + "grounding": "cited", + "line": 1 + } + ], + "raw_bytes": 8919, + "err_bytes": 59, + "envelope_status": "SUCCESS", + "verdict_source": "structured_output", + "model": "gemini-3.6-flash-high", + "model_measured": "gemini-3.6-flash-high", + "model_source": "measured", + "family": "gemini", + "role": "independent", + "fallback": { + "judged_by": "gemini-3.6-flash-high", + "exhausted": false, + "attempts": [ + { + "model": "gemini-3.6-flash-high", + "family": "gemini", + "outcome": "answered" + } + ], + "skipped": [] + } + }, + { + "lane": 3, + "status": "SUCCESS", + "verdict": "PASS", + "summary": "The diff accurately implements all requirements specified in PMAT-3747. \n- The `TokenConstraint` trait and `ConstraintError` are correctly implemented, with Display logic matching the specific format in the ticket notes.\n- `load_schema` successfully supports both inline schemas and `@path` resolution, explicitly catching invalid/non-JSON paths.\n- The `llguidance` dependency is properly placed behind the `structured-output` feature and enabled via the `inference` feature in `apr-cli` Cargo.toml.\n- `GGUFModel::constraint_vocab()` correctly reuses `push_token_bytes` and marks control tokens as special so they can't be constrained text.\n- The comprehensive test suite in `tests.rs` fully captures the validation for RAH's quorum-lane schema and rmedia schemas, verifying proper rejection of mutations and out-of-spec JSON configurations.\n- The `constrain_mask_overhead.rs` example fulfills the overhead measurement requirement.\n\nNo contradictions or forbidden features were introduced.", + "findings": [], + "raw_bytes": 3298, + "err_bytes": 0, + "envelope_status": "SUCCESS", + "verdict_source": "structured_output", + "grounding_check": "parity", + "model": "gemini-3.1-pro-low", + "model_measured": "gemini-3.1-pro-low", + "model_source": "measured", + "family": "gemini", + "role": "independent", + "fallback": { + "judged_by": "gemini-3.1-pro-low", + "exhausted": false, + "attempts": [ + { + "model": "gemini-3.1-pro-low", + "family": "gemini", + "outcome": "answered" + } + ], + "skipped": [] + } + } + ], + "dissent": [], + "dedup": [ + { + "file": "crates/apr-cli/Cargo.toml", + "line": 88, + "lanes_agreeing": [ + 2 + ], + "claims": [ + "`apr-cli`'s `inference` feature enables `realizar/structured-output`" + ] + }, + { + "file": "crates/aprender-serve/Cargo.toml", + "line": 111, + "lanes_agreeing": [ + 2 + ], + "claims": [ + "`llguidance 1.8.0` is an optional dependency under `structured-output` feature, and `cargo deny check` (licenses, advisories, bans, sources) passes cleanly" + ] + }, + { + "file": "crates/aprender-serve/Cargo.toml", + "line": 113, + "lanes_agreeing": [ + 1 + ], + "claims": [ + "llguidance 1.8.0 sits behind the trait, as an optional dependency behind a feature that apr-cli's `inference` enables" + ] + }, + { + "file": "crates/aprender-serve/examples/constrain_mask_overhead.rs", + "line": 1, + "lanes_agreeing": [ + 2 + ], + "claims": [ + "Per-token constraint overhead example `constrain_mask_overhead.rs` is implemented and builds cleanly under `--features structured-output`" + ] + }, + { + "file": "crates/aprender-serve/examples/constrain_mask_overhead.rs", + "line": 33, + "lanes_agreeing": [ + 1 + ], + "claims": [ + "a measured per-token mask overhead on the Qwen3.5-9B vocabulary is provided as an example" + ] + }, + { + "file": "crates/aprender-serve/src/constrain/mod.rs", + "line": 1, + "lanes_agreeing": [ + 1 + ], + "claims": [ + "No generation loop is wired (apr run's CPU paths, etc. are NOT in this PR)" + ] + }, + { + "file": "crates/aprender-serve/src/constrain/mod.rs", + "line": 35, + "lanes_agreeing": [ + 1, + 2 + ], + "claims": [ + "`TokenConstraint` trait defines `mask`, `accept`, and `is_complete`", + "`constrain::TokenConstraint` {mask, accept, is_complete} exists" + ] + }, + { + "file": "crates/aprender-serve/src/constrain/mod.rs", + "line": 50, + "lanes_agreeing": [ + 2 + ], + "claims": [ + "`ConstraintError` defines all 6 refusal variants (SchemaInvalid, SchemaUnsupported, DeadEnd, Rejected, Vocab, NotCompiled) with Display prefix formatting matching ticket spec" + ] + }, + { + "file": "crates/aprender-serve/src/constrain/mod.rs", + "line": 51, + "lanes_agreeing": [ + 1 + ], + "claims": [ + "`ConstraintError` names every refusal (SchemaInvalid, SchemaUnsupported, DeadEnd, Rejected, Vocab, NotCompiled), each rendered with its name first" + ] + }, + { + "file": "crates/aprender-serve/src/constrain/mod.rs", + "line": 112, + "lanes_agreeing": [ + 2 + ], + "claims": [ + "`load_schema` parses inline JSON schema or `@path` schema file, refusing unreadable paths, non-JSON text, and non-schema JSON as `SchemaInvalid`" + ] + }, + { + "file": "crates/aprender-serve/src/constrain/mod.rs", + "line": 114, + "lanes_agreeing": [ + 1 + ], + "claims": [ + "`load_schema` takes the schema inline OR as `@path` and refuses non-JSON, non-schema JSON and unreadable paths as SchemaInvalid, before any token" + ] + }, + { + "file": "crates/aprender-serve/src/constrain/tests.rs", + "line": 304, + "lanes_agreeing": [ + 2 + ], + "claims": [ + "Case table tests pass with 24/24 tests green under `--features structured-output` and 12/12 tests green without the feature" + ] + }, + { + "file": "crates/aprender-serve/src/constrain/tests.rs", + "line": 312, + "lanes_agreeing": [ + 1 + ], + "claims": [ + "case table: RAH's quorum-lane schema and rmedia's three schemas (#3716) compile exactly, accept their real documents, refuse violations at the byte, and allow nothing after one complete value; an approximation is refused; without the feature a constraint is refused (NotCompiled), never ignored" + ] + }, + { + "file": "crates/aprender-serve/src/gguf/token.rs", + "line": 307, + "lanes_agreeing": [ + 1 + ], + "claims": [ + "`GGUFModel::constraint_vocab()` builds each token's bytes from the SAME per-token decoding `decode()` uses; control/unused tokens are special and can never be constrained text" + ] + }, + { + "file": "crates/aprender-serve/src/gguf/token.rs", + "line": 316, + "lanes_agreeing": [ + 2 + ], + "claims": [ + "`GGUFModel::constraint_vocab()` extracts raw token bytes using `push_token_bytes` (sharing logic with `decode()`) and marks control/unused token types as special" + ] + } + ], + "uncovered": [], + "coverage_source": "lanes", + "partial": false, + "partial_reasons": [ + "lane 2: grounding NOT COMPARED — structured_output holds a verdict but no single verdict object could be read out of .response (8919 raw bytes), so agy's labels stand unchecked for this lane (PMAT-082)" + ], + "fallback": { + "same_family_width": 1, + "chain": [ + { + "model": "gemini-3.1-pro-high", + "family": "gemini", + "disposition": "configured" + }, + { + "model": "gemini-3.6-flash-high", + "family": "gemini", + "disposition": "configured" + }, + { + "model": "gemini-3.1-pro-low", + "family": "gemini", + "disposition": "configured" + }, + { + "model": "gpt-oss-120b-medium", + "family": "openai", + "disposition": "fallback" + }, + { + "model": "qwen3.5", + "family": "qwen", + "disposition": "not-run", + "why": "no quorum.local_lane in the config — the aprender lane has no model to load" + }, + { + "model": "claude-opus-4-6-thinking", + "family": "claude", + "disposition": "width", + "why": "same family as the author: at most 1 lane, recorded role width, counted toward no floor (R-15a)" + }, + { + "model": "claude-sonnet-4-6", + "family": "claude", + "disposition": "width", + "why": "same family as the author: at most 1 lane, recorded role width, counted toward no floor (R-15a)" + } + ], + "precheck": [ + { + "family": "gemini", + "model": "gemini-3.1-pro-high", + "probe": 1, + "outcome": "live" + } + ], + "prah": { + "source": "install-receipt", + "path": "/home/noah/.claude/skills/paiml-implement/bin/prah" + } + }, + "auto_merge": { + "checked": false, + "was_armed": false, + "disarmed": false, + "note": "no --pr given: nothing to disarm" + }, + "lint": { + "ok": true, + "output": "receipt complete: kind=artifact lanes=3 author=claude-opus-5/claude" + } +} diff --git a/docs/audits/quorum-PMAT-3756.json b/docs/audits/quorum-PMAT-3756.json new file mode 100644 index 0000000000..598b6999ae --- /dev/null +++ b/docs/audits/quorum-PMAT-3756.json @@ -0,0 +1,225 @@ +{ + "ticket": "PMAT-3756", + "base": "8fe36c080", + "base_resolved": "8fe36c080", + "base_note": "no origin/8fe36c080 exists; judged against the local ref", + "head": "d0c506a053c7c5ea7a37c67272fe9489eb8f95d3", + "diff_sha256": "7a7ed5496c2fc3d119e874efacf99e0c49c249e9cce4c89a337657f4cacc366e", + "width": 3, + "executor": "agy", + "prompt_mode": "inline", + "prompt_bytes": 11235, + "author": { + "model": "claude-opus-5", + "family": "claude", + "source": "flag" + }, + "agreed": true, + "lanes": [ + { + "lane": 1, + "status": "SUCCESS", + "verdict": "PASS", + "summary": "The diff correctly fulfills the first clause of PMAT-3756. It fixes the `llama_bin.sh` sed compatibility issue on macOS (mini) by passing the commands as separate `-e` flags, which resolves the 'undefined label' BSD sed error. It also includes the required resolution evidence for both 'intel' and 'mini' hosts in the `evidence/parity` directory. The roadmap correctly specifies that this PR only completes the first clause and refs (rather than closes) the ticket. The changes strictly follow the ticket requirements and forbid nothing.", + "findings": [ + { + "claim": "The sed command was correctly updated to use separate -e arguments, fixing the 'undefined label' error on BSD sed (mini).", + "file": "scripts/llama_bin.sh", + "grounding": "cited" + }, + { + "claim": "The evidence file for intel host is properly committed and shows the pin resolving correctly.", + "file": "evidence/parity/pin-bump-d1d3c3396/RESOLVE-intel.txt", + "grounding": "cited" + }, + { + "claim": "The evidence file for mini host is properly committed and shows the pin resolving correctly.", + "file": "evidence/parity/pin-bump-d1d3c3396/RESOLVE-mini.txt", + "grounding": "cited" + } + ], + "raw_bytes": 3589, + "err_bytes": 100, + "envelope_status": "SUCCESS", + "verdict_source": "structured_output", + "grounding_check": "parity", + "model": "gemini-3.1-pro-high", + "model_measured": "gemini-3.1-pro-high", + "model_source": "measured", + "family": "gemini", + "role": "independent", + "fallback": { + "judged_by": "gemini-3.1-pro-high", + "exhausted": false, + "attempts": [ + { + "model": "gemini-3.1-pro-high", + "family": "gemini", + "outcome": "answered" + } + ], + "skipped": [] + } + }, + { + "lane": 2, + "status": "SUCCESS", + "verdict": "PASS", + "summary": "The diff for PMAT-3756 satisfies the ticket requirements and refutation criteria:\n1. It delivers the first clause of PMAT-3756's acceptance criteria: `llama_bin.sh` resolves the pin on intel and mini with committed evidence in `evidence/parity/pin-bump-d1d3c3396/RESOLVE-{intel,mini}.txt` matching the format of lambda/gx10.\n2. It correctly scopes the delivery as 'Refs #3756' rather than 'Closes #3756' because the second clause (host_receipt.sh parity block) depends on #3731; the roadmap entry records status as `in_progress` (50% progress).\n3. The portable sed fix in `scripts/llama_bin.sh` (using separate `-e` options for `t` and `s/.../p`) resolves BSD sed's branch label parsing issue on macOS (`undefined label`) while remaining completely compatible with GNU sed.\n4. Verified via `scripts/check_llama_pin.sh`: all 43 tests pass and both mutants remain killed.\n5. Verified via `make roadmap-aggregate-check`: roadmap.yaml matches aggregate of 107 fragments and is idempotent.\n6. Shellcheck on `scripts/llama_bin.sh` reports 0 errors. No gates were weakened, no receipt claims were unsupported, and no out-of-scope changes were introduced.", + "findings": [], + "raw_bytes": 3589, + "err_bytes": 0, + "envelope_status": "SUCCESS", + "verdict_source": "structured_output", + "grounding_check": "parity", + "model": "gemini-3.8-flash-high", + "model_measured": "gemini-3.8-flash-high", + "model_source": "measured", + "family": "gemini", + "role": "independent", + "fallback": { + "judged_by": "gemini-3.8-flash-high", + "exhausted": false, + "attempts": [ + { + "model": "gemini-3.8-flash-high", + "family": "gemini", + "outcome": "answered" + } + ], + "skipped": [] + } + }, + { + "lane": 3, + "status": "SUCCESS", + "verdict": "PASS", + "summary": "PASS. The diff (8fe36c080...HEAD) precisely implements the first clause of PMAT-3756 (Refs #3756):\n1. Fixes the non-portable BSD sed `t` label interpretation in `scripts/llama_bin.sh` (line 415) by splitting `-e` arguments (`-e 's/.*commit ([0-9a-f]{7,40}).*/\\1/p' -e 't' -e 's/.*\\(([0-9a-f]{7,40})\\).*/\\1/p'`), restoring resolution on Darwin / macOS (mini) while remaining 100% compatible with GNU sed (intel / linux).\n2. Commits resolution proof files `evidence/parity/pin-bump-d1d3c3396/RESOLVE-intel.txt` and `evidence/parity/pin-bump-d1d3c3396/RESOLVE-mini.txt` with rc=0.\n3. Adds roadmap fragment `docs/roadmaps/entries/PMAT-3756.yaml` and aggregates into `docs/roadmaps/roadmap.yaml`.\n4. Verified `bash scripts/check_llama_pin.sh` passes (43 ok rows, both mutants killed, rc 0).\nNo gates were weakened, no receipt claims are unbacked, and the diff contains no out-of-scope changes.", + "findings": [], + "raw_bytes": 3091, + "err_bytes": 0, + "envelope_status": "SUCCESS", + "verdict_source": "structured_output", + "grounding_check": "parity", + "model": "gemini-3.7-flash-high", + "model_measured": "gemini-3.7-flash-high", + "model_source": "measured", + "family": "gemini", + "role": "independent", + "fallback": { + "judged_by": "gemini-3.7-flash-high", + "exhausted": false, + "attempts": [ + { + "model": "gemini-3.7-flash-high", + "family": "gemini", + "outcome": "answered" + } + ], + "skipped": [] + } + } + ], + "dissent": [], + "dedup": [ + { + "file": "evidence/parity/pin-bump-d1d3c3396/RESOLVE-intel.txt", + "line": null, + "lanes_agreeing": [ + 1 + ], + "claims": [ + "The evidence file for intel host is properly committed and shows the pin resolving correctly." + ] + }, + { + "file": "evidence/parity/pin-bump-d1d3c3396/RESOLVE-mini.txt", + "line": null, + "lanes_agreeing": [ + 1 + ], + "claims": [ + "The evidence file for mini host is properly committed and shows the pin resolving correctly." + ] + }, + { + "file": "scripts/llama_bin.sh", + "line": null, + "lanes_agreeing": [ + 1 + ], + "claims": [ + "The sed command was correctly updated to use separate -e arguments, fixing the 'undefined label' error on BSD sed (mini)." + ] + } + ], + "uncovered": [], + "coverage_source": "lanes", + "partial": false, + "partial_reasons": [], + "fallback": { + "same_family_width": 1, + "chain": [ + { + "model": "gemini-3.1-pro-high", + "family": "gemini", + "disposition": "configured" + }, + { + "model": "gemini-3.8-flash-high", + "family": "gemini", + "disposition": "configured" + }, + { + "model": "gemini-3.7-flash-high", + "family": "gemini", + "disposition": "configured" + }, + { + "model": "gpt-oss-120b-medium", + "family": "openai", + "disposition": "fallback" + }, + { + "model": "qwen3.5", + "family": "qwen", + "disposition": "not-run", + "why": "no quorum.local_lane in the config — the aprender lane has no model to load" + }, + { + "model": "claude-opus-4-6-thinking", + "family": "claude", + "disposition": "width", + "why": "same family as the author: at most 1 lane, recorded role width, counted toward no floor (R-15a)" + }, + { + "model": "claude-sonnet-4-6", + "family": "claude", + "disposition": "width", + "why": "same family as the author: at most 1 lane, recorded role width, counted toward no floor (R-15a)" + } + ], + "precheck": [ + { + "family": "gemini", + "model": "gemini-3.1-pro-high", + "probe": 1, + "outcome": "live" + } + ], + "prah": { + "source": "install-receipt", + "path": "/home/noah/.claude/skills/paiml-implement/bin/prah" + } + }, + "auto_merge": { + "checked": false, + "was_armed": false, + "disarmed": false, + "note": "no --pr given: nothing to disarm" + }, + "lint": { + "ok": true, + "output": "receipt complete: kind=artifact lanes=3 author=claude-opus-5/claude" + } +} diff --git a/docs/audits/quorum-PMAT-3792.json b/docs/audits/quorum-PMAT-3792.json new file mode 100644 index 0000000000..8d7cf8c560 --- /dev/null +++ b/docs/audits/quorum-PMAT-3792.json @@ -0,0 +1,193 @@ +{ + "ticket": "PMAT-3792", + "base": "ded8a932a", + "base_resolved": "ded8a932a", + "base_note": "no origin/ded8a932a exists; judged against the local ref", + "head": "f523c8889789399a33c6f1d06c67363246f1e0b2", + "diff_sha256": "cfbc45719554983e0046af8c06cec04b60ca8f44c02a8e3aa467e6b1eddf9a3f", + "width": 3, + "executor": "agy", + "prompt_mode": "inline", + "prompt_bytes": 19278, + "author": { + "model": "claude-opus-5", + "family": "claude", + "source": "flag" + }, + "agreed": false, + "lanes": [ + { + "lane": 1, + "status": "SUCCESS", + "verdict": "PASS", + "summary": "The PR is a pure refactor that extracts parts of `run_shapes_gate_with` into helper functions (`prepare`, `order_by_family`, `needs_receipts`, `verdict_of`, `by_shape`, and `by_entity_type`). I have verified that the logic of the helper functions maps exactly to the original logic byte-for-byte in behavior. Variable bindings, error early-returns, and execution order are preserved. The failure in `check_complexity_ratchet.sh` was accurately addressed by the author as belonging to an unrelated function (`generate_intent`). Therefore, the implementation exactly fulfills PMAT-3792 without introducing any unrequested feature changes or behavior regressions.", + "findings": [], + "raw_bytes": 2681, + "err_bytes": 100, + "envelope_status": "SUCCESS", + "verdict_source": "structured_output", + "grounding_check": "parity", + "model": "gemini-3.1-pro-high", + "model_measured": "gemini-3.1-pro-high", + "model_source": "measured", + "family": "gemini", + "role": "independent", + "fallback": { + "judged_by": "gemini-3.1-pro-high", + "exhausted": false, + "attempts": [ + { + "model": "gemini-3.1-pro-high", + "family": "gemini", + "outcome": "answered" + } + ], + "skipped": [] + } + }, + { + "lane": 2, + "status": "SUCCESS", + "verdict": "PASS", + "summary": "The diff successfully performs a pure refactor of `run_shapes_gate_with` to reduce complexity (extracting out `prepare`, `order_by_family`, `needs_receipts`, `verdict_of`, `by_shape`, and `by_entity_type`), with absolutely zero changes to the underlying logic or behavior. The extracted functions execute the exact same steps in the exact same order. The receipt file is properly added and the ticket requirements are fully met.", + "findings": [], + "raw_bytes": 2104, + "err_bytes": 0, + "envelope_status": "SUCCESS", + "verdict_source": "structured_output", + "grounding_check": "parity", + "model": "gemini-3.1-pro-low", + "model_measured": "gemini-3.1-pro-low", + "model_source": "measured", + "family": "gemini", + "role": "independent", + "fallback": { + "judged_by": "gemini-3.1-pro-low", + "exhausted": false, + "attempts": [ + { + "model": "gemini-3.1-pro-low", + "family": "gemini", + "outcome": "answered" + } + ], + "skipped": [] + } + }, + { + "lane": 3, + "status": "NO-VERDICT", + "verdict": "NO-VERDICT", + "summary": "", + "findings": [], + "raw_bytes": 1699, + "err_bytes": 140, + "envelope_status": "ERROR", + "transport_status": "ERROR", + "transport_error": "Individual quota reached. Please upgrade your subscription to increase your limits. Resets in 30h27m26s.", + "salvageable": "/home/noah/.gemini/antigravity-cli/brain/9c134b73-a54f-43c3-b5f4-d1208362785f (1476159 bytes)", + "carried_verdict": "", + "blind_reason": "", + "agy_exit": "", + "model": "gemini-3.8-flash-high", + "model_measured": "gemini-3.8-flash-high", + "model_source": "measured", + "family": "gemini", + "role": "independent", + "fallback": { + "judged_by": null, + "exhausted": false, + "attempts": [ + { + "model": "gemini-3.8-flash-high", + "family": "gemini", + "outcome": "other", + "reason": "Individual quota reached. Please upgrade your subscription to increase your limits. Resets in 30h27m26s." + } + ], + "skipped": [] + } + } + ], + "dissent": [ + { + "lane": 3, + "verdict": "NO-VERDICT", + "summary": "", + "findings": [] + } + ], + "dedup": [], + "uncovered": [], + "coverage_source": "lanes", + "partial": true, + "partial_reasons": [ + "lane 3: non-empty .err (140 bytes, 1 line(s) beyond agy-lane's workspace narration)", + "lane 3: lane envelope status=ERROR transport_error=Individual quota reached. Please upgrade your subscription to increase your limits. Resets in 30h27m26s. salvageable=/home/noah/.gemini/antigravity-cli/brain/9c134b73-a54f-43c3-b5f4-d1208362785f (1476159 bytes) [UNREVIEWED CLAIM MATERIAL — never a PASS]" + ], + "fallback": { + "same_family_width": 1, + "chain": [ + { + "model": "gemini-3.1-pro-high", + "family": "gemini", + "disposition": "configured" + }, + { + "model": "gemini-3.1-pro-low", + "family": "gemini", + "disposition": "configured" + }, + { + "model": "gemini-3.8-flash-high", + "family": "gemini", + "disposition": "configured" + }, + { + "model": "gpt-oss-120b-medium", + "family": "openai", + "disposition": "fallback" + }, + { + "model": "qwen3.5", + "family": "qwen", + "disposition": "not-run", + "why": "no quorum.local_lane in the config — the aprender lane has no model to load" + }, + { + "model": "claude-opus-4-6-thinking", + "family": "claude", + "disposition": "width", + "why": "same family as the author: at most 1 lane, recorded role width, counted toward no floor (R-15a)" + }, + { + "model": "claude-sonnet-4-6", + "family": "claude", + "disposition": "width", + "why": "same family as the author: at most 1 lane, recorded role width, counted toward no floor (R-15a)" + } + ], + "precheck": [ + { + "family": "gemini", + "model": "gemini-3.1-pro-high", + "probe": 1, + "outcome": "live" + } + ], + "prah": { + "source": "install-receipt", + "path": "/home/noah/.claude/skills/paiml-implement/bin/prah" + } + }, + "auto_merge": { + "checked": false, + "was_armed": false, + "disarmed": false, + "note": "no --pr given: nothing to disarm" + }, + "lint": { + "ok": true, + "output": "receipt complete: kind=artifact lanes=3 author=claude-opus-5/claude" + } +} diff --git a/docs/audits/surface_audit.csv b/docs/audits/surface_audit.csv index f87c0b7e58..eb554f829c 100644 --- a/docs/audits/surface_audit.csv +++ b/docs/audits/surface_audit.csv @@ -116,7 +116,7 @@ apr,apr cgp profile wgpu,6,x86_64-linux,NONE,no,9,profiling,crates/aprender-cgp/ apr,apr cgp roofline,6,x86_64-linux,NONE,no,7,qa-cgp,crates/aprender-cgp/src/cli.rs:64,low apr,apr cgp trace,6,x86_64-linux,NONE,no,7,qa-cgp,crates/aprender-cgp/src/cli.rs:99,low apr,apr cgp tui,6,x86_64-linux,NONE,no,7,qa-cgp,crates/aprender-cgp/src/cli.rs:115,low -apr,apr chat,6,x86_64-linux,llama.cpp,yes,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:40 (ACCELERATOR_GUARD crates/apr-cli/src/dispatch_analysis.rs:1699),high +apr,apr chat,6,x86_64-linux,llama.cpp,yes,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:40 (ACCELERATOR_GUARD crates/apr-cli/src/dispatch_analysis.rs:1710),high apr,apr chat --backend cpu,6,x86_64-linux,ollama,no,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:85 (BACKEND_VALUES crates/apr-cli/src/commands_enum.rs:16),high apr,apr chat --backend cuda,6,x86_64-linux;nvidia-cuda,ollama,no,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:85 (BACKEND_VALUES crates/apr-cli/src/commands_enum.rs:16),medium apr,apr chat --backend wgpu,6,UNKNOWN,ollama,no,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:85 (BACKEND_VALUES crates/apr-cli/src/commands_enum.rs:16),high @@ -183,16 +183,16 @@ apr,apr embed-viz-lint,1,x86_64-linux,NONE,yes,1,apr-lint-diag,crates/apr-cli/sr apr,apr embeddings-lint,1,x86_64-linux,NONE,yes,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:1172,high apr,apr encrypt,4,x86_64-linux,NONE,yes,10,apr-core-commands,crates/apr-cli/src/tool_commands.rs:121,high apr,apr eval,6,x86_64-linux,llama.cpp,yes,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:121,high -apr,apr eval --task classify,6,x86_64-linux,scikit-learn,no,8,rag-eval,crates/apr-cli/src/dispatch_analysis.rs:1394 (flag crates/apr-cli/src/extended_commands.rs:139),high -apr,apr eval --task code,6,x86_64-linux,NONE,no,8,rag-eval,crates/apr-cli/src/dispatch_analysis.rs:1402 (flag crates/apr-cli/src/extended_commands.rs:139),high -apr,apr eval --task compare,6,x86_64-linux,NONE,no,8,rag-eval,crates/apr-cli/src/dispatch_analysis.rs:1426 (flag crates/apr-cli/src/extended_commands.rs:139),high -apr,apr eval --task contamination,6,x86_64-linux,huggingface-transformers,no,8,rag-eval,crates/apr-cli/src/dispatch_analysis.rs:1423 (flag crates/apr-cli/src/extended_commands.rs:139),high -apr,apr eval --task correlation,6,x86_64-linux,NONE,no,8,rag-eval,crates/apr-cli/src/dispatch_analysis.rs:1428 (flag crates/apr-cli/src/extended_commands.rs:139),high -apr,apr eval --task human,6,x86_64-linux,NONE,no,8,rag-eval,crates/apr-cli/src/dispatch_analysis.rs:1429 (flag crates/apr-cli/src/extended_commands.rs:139),high -apr,apr eval --task humaneval,6,x86_64-linux,NONE,no,8,rag-eval,crates/apr-cli/src/dispatch_analysis.rs:1405 (flag crates/apr-cli/src/extended_commands.rs:139),high -apr,apr eval --task mbpp,6,x86_64-linux,NONE,no,8,rag-eval,crates/apr-cli/src/dispatch_analysis.rs:1414 (flag crates/apr-cli/src/extended_commands.rs:139),high -apr,apr eval --task plan,6,x86_64-linux,NONE,no,8,rag-eval,crates/apr-cli/src/dispatch_analysis.rs:1430 (flag crates/apr-cli/src/extended_commands.rs:139),high -apr,apr eval --task verify,6,x86_64-linux,NONE,no,8,rag-eval,crates/apr-cli/src/dispatch_analysis.rs:1427 (flag crates/apr-cli/src/extended_commands.rs:139),high +apr,apr eval --task classify,6,x86_64-linux,scikit-learn,no,8,rag-eval,crates/apr-cli/src/dispatch_analysis.rs:1405 (flag crates/apr-cli/src/extended_commands.rs:139),high +apr,apr eval --task code,6,x86_64-linux,NONE,no,8,rag-eval,crates/apr-cli/src/dispatch_analysis.rs:1413 (flag crates/apr-cli/src/extended_commands.rs:139),high +apr,apr eval --task compare,6,x86_64-linux,NONE,no,8,rag-eval,crates/apr-cli/src/dispatch_analysis.rs:1437 (flag crates/apr-cli/src/extended_commands.rs:139),high +apr,apr eval --task contamination,6,x86_64-linux,huggingface-transformers,no,8,rag-eval,crates/apr-cli/src/dispatch_analysis.rs:1434 (flag crates/apr-cli/src/extended_commands.rs:139),high +apr,apr eval --task correlation,6,x86_64-linux,NONE,no,8,rag-eval,crates/apr-cli/src/dispatch_analysis.rs:1439 (flag crates/apr-cli/src/extended_commands.rs:139),high +apr,apr eval --task human,6,x86_64-linux,NONE,no,8,rag-eval,crates/apr-cli/src/dispatch_analysis.rs:1440 (flag crates/apr-cli/src/extended_commands.rs:139),high +apr,apr eval --task humaneval,6,x86_64-linux,NONE,no,8,rag-eval,crates/apr-cli/src/dispatch_analysis.rs:1416 (flag crates/apr-cli/src/extended_commands.rs:139),high +apr,apr eval --task mbpp,6,x86_64-linux,NONE,no,8,rag-eval,crates/apr-cli/src/dispatch_analysis.rs:1425 (flag crates/apr-cli/src/extended_commands.rs:139),high +apr,apr eval --task plan,6,x86_64-linux,NONE,no,8,rag-eval,crates/apr-cli/src/dispatch_analysis.rs:1441 (flag crates/apr-cli/src/extended_commands.rs:139),high +apr,apr eval --task verify,6,x86_64-linux,NONE,no,8,rag-eval,crates/apr-cli/src/dispatch_analysis.rs:1438 (flag crates/apr-cli/src/extended_commands.rs:139),high apr,apr experiment view,6,x86_64-linux,pytorch,no,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:1567,low apr,apr explain,6,x86_64-linux,safetensors,yes,10,apr-core-commands,crates/apr-cli/src/commands_enum.rs:447,high apr,apr explain-token-lint,1,x86_64-linux,NONE,yes,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:1065,low @@ -238,9 +238,9 @@ apr,apr oom-lint,1,x86_64-linux,NONE,yes,1,apr-lint-diag,crates/apr-cli/src/exte apr,apr oracle,9,x86_64-linux,NONE,yes,10,apr-core-commands,crates/apr-cli/src/tool_commands.rs:84,medium apr,apr otlp-lint,1,x86_64-linux,NONE,yes,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:1097,high apr,apr parity,9,x86_64-linux,NONE,yes,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:317,medium -apr,apr parity --per-op,9,"x86_64-linux,aarch64-linux",NONE,no,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:330 (dispatch crates/apr-cli/src/dispatch_analysis.rs:1505),high -apr,apr parity --out,7,"x86_64-linux,aarch64-linux",NONE,no,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:333 (dispatch crates/apr-cli/src/dispatch_analysis.rs:1506),high -apr,apr parity --threshold,7,"x86_64-linux,aarch64-linux",NONE,no,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:336 (dispatch crates/apr-cli/src/dispatch_analysis.rs:1506),high +apr,apr parity --per-op,9,"x86_64-linux,aarch64-linux",NONE,no,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:330 (dispatch crates/apr-cli/src/dispatch_analysis.rs:1516),high +apr,apr parity --out,7,"x86_64-linux,aarch64-linux",NONE,no,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:333 (dispatch crates/apr-cli/src/dispatch_analysis.rs:1517),high +apr,apr parity --threshold,7,"x86_64-linux,aarch64-linux",NONE,no,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:336 (dispatch crates/apr-cli/src/dispatch_analysis.rs:1517),high apr,apr pipeline apply,6,x86_64-linux,huggingface-transformers,no,10,apr-core-commands,crates/apr-cli/src/pipeline_commands.rs:28,low apr,apr pipeline plan,6,x86_64-linux,huggingface-transformers,no,10,apr-core-commands,crates/apr-cli/src/pipeline_commands.rs:9,low apr,apr pipeline status,6,x86_64-linux,huggingface-transformers,no,10,apr-core-commands,crates/apr-cli/src/pipeline_commands.rs:51,low @@ -368,10 +368,11 @@ apr,apr test llm bench,6,x86_64-linux,vllm,no,1,apr-lint-diag,crates/apr-cli/src apr,apr test llm bench --band,6,x86_64-linux,vllm,no,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:1785 (protocol crates/apr-cli/src/commands/test_llm_band.rs:348),medium apr,apr test tensor,4,x86_64-linux,NONE,no,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:1622,high apr,apr tokenize apply,6,x86_64-linux,huggingface-transformers,no,10,apr-core-commands,crates/apr-cli/src/tokenize_commands.rs:38,low -apr,apr tokenize encode-corpus,6,x86_64-linux,huggingface-transformers,no,10,apr-core-commands,crates/apr-cli/src/tokenize_commands.rs:131,low +apr,apr tokenize encode,6,x86_64-linux,llama.cpp,no,10,apr-core-commands,crates/apr-cli/src/tokenize_commands.rs:125,high +apr,apr tokenize encode-corpus,6,x86_64-linux,huggingface-transformers,no,10,apr-core-commands,crates/apr-cli/src/tokenize_commands.rs:149,low apr,apr tokenize import-hf,6,x86_64-linux,huggingface-transformers,no,10,apr-core-commands,crates/apr-cli/src/tokenize_commands.rs:104,low apr,apr tokenize plan,6,x86_64-linux,huggingface-transformers,no,10,apr-core-commands,crates/apr-cli/src/tokenize_commands.rs:14,low -apr,apr tokenize repair-manifest,6,x86_64-linux,huggingface-transformers,no,10,apr-core-commands,crates/apr-cli/src/tokenize_commands.rs:221,low +apr,apr tokenize repair-manifest,6,x86_64-linux,huggingface-transformers,no,10,apr-core-commands,crates/apr-cli/src/tokenize_commands.rs:239,low apr,apr tokenize train,6,x86_64-linux,huggingface-transformers,no,10,apr-core-commands,crates/apr-cli/src/tokenize_commands.rs:63,low apr,apr tool-use-lint,1,x86_64-linux,NONE,yes,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:1124,high apr,apr trace,6,x86_64-linux,safetensors,yes,10,apr-core-commands,crates/apr-cli/src/commands_enum.rs:369,high diff --git a/docs/roadmaps/entries/PMAT-3543.yaml b/docs/roadmaps/entries/PMAT-3543.yaml new file mode 100644 index 0000000000..4503f3afd1 --- /dev/null +++ b/docs/roadmaps/entries/PMAT-3543.yaml @@ -0,0 +1,20 @@ +- id: PMAT-3543 + github_issue: 3543 + item_type: task + title: 'The post-publish dogfood CANNOT pass: version-unpublished is not phase-aware and the banner still says pre-release' + status: in_progress + priority: high + assigned_to: aprender-f0 + created: 2026-09-20T08:17:54Z + updated: 2026-09-21T16:50:00Z + spec: null + acceptance_criteria: + - 'Issue #3543 defect 1 (the issue has no done_when list; its defects and its falsifier are the acceptance), verbatim: "1. **`version-unpublished` is not phase-aware.** It is a pre-publish gate ("this version must be absent from the index"). In the post-publish phase its precondition is inverted by construction: a successful publish makes it RED forever. `--phase` is parsed and validated (dogfood.sh:42-53) and 13 sites read `DOGFOOD_PHASE`, but this row is not one of them. Post-publish it should assert the opposite — the version IS on the index, resolvable, and `cargo install` of it works — which is the receipt the train actually wants."' + - 'defect 2, verbatim: "2. **The banner lies about which phase ran** (dogfood.sh:269 prints `dogfood pre-release:` unconditionally). Anyone reading the log believes they are looking at the pre-release gate set."' + - 'the issue''s falsifier, verbatim: "**Falsifier for the fix:** run `--phase post-publish` at a version that IS published and require GO; run it at a version that is NOT published and require RED on the new post-publish assertion. Both directions, or the row is decoration again."' + phases: [] + subtasks: [] + estimated_effort: null + labels: + - kind:code + notes: 'Release-tooling lane (cop, aprender-04), on the 0.69.1 release path. Its own branch off main 225b2a9ab: dogfood.sh is in none of the #3699/#3708/#3717 stack. SCOPE: scripts/dogfood.sh, the new scripts/check_dogfood_version_row.sh, and this fragment. SUPERSEDES open PR #3202 (PMAT-1098-dogfood-post-publish-phase, 2026-09-13), which the cop will close with these reasons. It does NOT meet this issue: (a) it patches only the dry-run branch, while main has since moved pre-publish onto a direct index.crates.io lookup (dogfood.sh:387-441 at 225b2a9ab); (b) it makes post-publish PASS unconditionally, so the falsifier''s second direction (a version NOT published must be RED) would read "not yet published" and PASS; (c) it leaves the banner. HOW IT IS MET. DEFECT 1: the sparse-index lookup is ONE function, index_version_state CRATE VERSION, which prints "present", "absent", "crate-absent" or "unknown ". Its body is the pre-publish code moved verbatim: the lowercased shard path, HTTP status read apart from the body, and a body that is not the index being UNKNOWN, never absent. version_row PHASE CRATE VERSION STATE decides the row. pre-publish gives version-unpublished, PASS only when absent or crate-absent (unchanged). post-publish gives a NEW row, version-published, PASS only when present AND installable (next), FAIL when absent ("… is NOT in the crates.io index — this phase runs after the cascade, so the version was not published"), crate-absent, or unknown. mark_version_row PHASE marks whichever row version_row decides. POST-PUBLISH ALSO INSTALLS (round 2): defect 1''s own text says post-publish must assert "the version IS on the index, resolvable, and `cargo install` of it works". Round 1''s lanes 1 and 2 FAILed an index-only row on that clause, and they were right. When the index carries the version, published_install_check runs `cargo install --version = --locked --root --target-dir ` with the registry token unset, exactly as a user installs. Every binary that lands must report the version in `--version`. A failed install (which is also where "resolvable" is decided: a version cargo cannot resolve, e.g. a yanked one, fails it), a binary reporting another version, or no binary at all turns version-published FAIL: "… is in the crates.io index but is NOT installable: …". No install is attempted for a version the index does not carry. The pre-publish branch calls it as before. The post-publish phase still runs the publish dry-run, because row 10 (publish-dry-run) reads its exit code and is what discharges the pre-publish DEFER. Its version row is then mark_version_row post-publish, and the pre-publish row is NOT emitted in that phase. The full phase is unchanged: the dry-run path, already exists -> FAIL. DEFECT 2: `phase_banner "$DOGFOOD_PHASE" "$CRATE" "$VERSION"` prints "══ dogfood : v ══". The only other mention of "dogfood pre-release" in scripts/ is a comment in dogfood-use.sh, which nothing parses. FALSIFIER, as a wired table: scripts/check_dogfood_version_row.sh, a scripts/check_*.sh with no bare cargo token, so guard_tree''s --no-cargo set runs it. It runs the REAL "2 + 10" section of dogfood.sh, extracted from its header to the next section header, so the call sites run as well as the helpers. It stubs `mark` (records rows), the publish dry-run, and curl (serving a fixture sparse index: present / absent / 404 / an HTML 200 / down). ROWS: pre-present FAIL; pre-absent PASS; pre-404 PASS; pre-down FAIL UNKNOWN; post-present version-published PASS (in the index, installed with `--version =9.9.9 --locked`, the binary reports 9.9.9) with NO version-unpublished row; post-install-fails FAIL; post-install-wrongver FAIL; post-install-nobin FAIL; post-absent FAIL with no install attempted; post-404 FAIL; post-html FAIL UNKNOWN; post-dry-run (the dry-run still runs, DRC 0); full-unchanged (already exists -> version-unpublished FAIL, no index call); banner ("══ dogfood post-publish: aprender v9.9.9 ══", printed from DOGFOOD_PHASE); index-path (crate "Aprender" is looked up at index.crates.io/ap/re/aprender). That is both directions in both phases, as the falsifier asks. MUTANTS, one per row, each killed: pre-present-pass, pre-absent-fail, pre-404-fail, down-is-absent, post-phase-blind (post-publish emitting the pre-publish row), post-absent-pass (#3202''s shape), post-404-pass, html-is-absent, no-post-call (the `mark_version_row post-publish` call deleted), post-skips-dry-run, full-uses-index, banner-fixed ("pre-release"), index-case (no lowercasing -> "looked up: curl https://index.crates.io/Ap/re/Aprender"); install-ignored, version-unchecked, nobin-ok, install-on-absent, unlocked (the install without --locked). The table''s first full run caught a fixture bug of mine: CRATE was exported before the per-row env, so index-path never saw the uppercase name and passed for the wrong reason until its mutant survived. MEASURED at this commit: `bash scripts/check_dogfood_version_row.sh` rc 0, "PASS 33 row(s)" (15 rows, 18 mutants); bashrs on dogfood.sh and the table, 0 errors (an `eval` in the banner row was replaced by sourcing the extracted line, after check_bashrs_gate''s SEC001); check_bashrs_gate, check_guards_are_wired and check_no_pipe_into_grep_q all rc 0. NOT RUN: a real `dogfood.sh --phase post-publish` against crates.io, which would include a real `cargo install aprender --version = --locked`, a full release build. The next train''s post-publish step is that measurement, hence Refs #3543.' diff --git a/docs/roadmaps/entries/PMAT-3563.yaml b/docs/roadmaps/entries/PMAT-3563.yaml new file mode 100644 index 0000000000..370dc34f70 --- /dev/null +++ b/docs/roadmaps/entries/PMAT-3563.yaml @@ -0,0 +1,19 @@ +- id: PMAT-3563 + github_issue: 3563 + item_type: task + title: 'Two llama.cpp comparator pins are live in-tree at once — 216 refs to d1d3c3396, 124 to 39173bcac; some committed artifacts measure against a superseded oracle' + status: in_progress + priority: critical + assigned_to: aprender-f0 + created: 2026-09-20T08:51:31Z + updated: 2026-09-21T17:00:00Z + spec: null + acceptance_criteria: + - 'Issue #3563 done_when, as the cop added it (comment 2026-09-21T16:35:16Z), verbatim: "done_when (added): exactly one pin in-tree; `llama_bin.sh` resolves and PROVES it on lambda and gx10 (a version+sha line) or fails naming why; every comparator consumer reads it"' + - 'THE SPLIT, recorded on the issue (comment 2026-09-21T16:49:53Z, "Split (cop ruling, aprender-04, 2026-09-21, order A → C → B). This issue now delivers (A) only"): this PR delivers clause 2 only, verbatim: "- **(A), this issue:** "`llama_bin.sh` resolves and PROVES it on lambda and gx10 (a version+sha line) or fails naming why". With nothing named, the resolver takes the pinned build at its CANONICAL location, `${LLAMA_PIN_SRC_ROOT:-$HOME/src}/llama.cpp-`. That path is derived from the pin, not searched for and not a host literal, and the build found there is verified by the same oracle-version, commit and cmake-cache checks as a named one. When it is absent the resolver FAILs naming the path and the declared build line. d1d3c3396 is built at that path on lambda (llama-bench and llama-tokenize added to the existing build) and on gx10 (a fresh build with `build_flags_gx10`)."; clause 3 is #3740 and clause 1 is #3741' + phases: [] + subtasks: [] + estimated_effort: null + labels: + - kind:code + notes: 'P0 for the 0.69.1 path (cop, aprender-04): four 0.69.1 rows need ONE resolvable pinned comparator (#3726 tokenizer parity, #3725, #3596, #3693). SPLIT (cop ruling, recorded on the issue, comment 5764188590): this issue delivers (A) only, "llama_bin.sh resolves and PROVES it on lambda and gx10 (a version+sha line) or fails naming why". (C) "every comparator consumer reads it" is #3740; (B) "exactly one pin in-tree" (classify the 142 refs to 39173bcac and the 3 to 60b06ab9a, plus the citation guard) is #3741. Its own branch off main 52f43da71 (the merge-base; used as the quorum base). SCOPE: scripts/llama_bin.sh, scripts/check_llama_pin.sh, two evidence files under evidence/parity/pin-bump-d1d3c3396/, and this fragment. MEASURED BEFORE THE CHANGE: the tree has ONE pin, d1d3c3396 (scripts/llama_pin.toml build_commit). gx10''''s "39173bcac" was only its stale ~/src/aprender checkout (de0e3e182, 2026-08-28). On lambda, `bash scripts/llama_bin.sh` printed "FAIL … (no_binary_named) … reported: ": the resolver never searches (NEVER PATH), $LLAMA_BENCH_PATH was unset, and the pinned build sat unnamed at ~/src/llama.cpp-d1d3c3396, with llama-cli and llama-server but NO llama-bench (the resolver''''s candidate) and no llama-tokenize. ~/src/llama.cpp there is 60b06ab9a, whose llama-cli dies "undefined symbol: llama_memory_breakdown_print". gx10 had NO d1d3c3396 build (~/src/llama.cpp-pin is the old 39173bcac). HOW (A) IS MET. (1) THE CANONICAL LOCATION: with nothing named, llama_bin.sh takes ${LLAMA_PIN_SRC_ROOT:-$HOME/src}/llama.cpp-/build/bin/llama-bench. That path is DERIVED FROM THE PIN, not a PATH search and not a host literal, and the build found there passes the same oracle-version, commit and cmake-cache checks as a named one. A named $LLAMA_BENCH_PATH always wins. When it is absent, the executable mode FAILs naming the canonical path and the pinned commit to build ("fails naming why"). (2) THE ABBREVIATION LENGTH, found by running the resolver on gx10: gx10''''s correct build prints "commit d1d3c339" (8 hex digits; git abbreviates to what is unique in THAT clone), lambda''''s "commit d1d3c3396". The old test (the 9-digit pin as a substring of the version line) refused gx10''''s correct build as wrong_build. The commit token is now read out of either --version shape ("commit " or "()") and matched when one abbreviates the other at git''''s minimum of 7 hex digits (llama_commit_match). Like the build number, which llama_pin.toml already says is not an identity, the length is not. (3) THE BUILDS: on lambda, `cmake --build build --target llama-bench llama-tokenize` in ~/src/llama.cpp-d1d3c3396 (incremental, BUILD_EXIT=0; cache GGML_CUDA=ON with no CMAKE_CUDA_ARCHITECTURES, matching build_flags_lambda). On gx10, a fresh clone of ~/src/llama.cpp detached at d1d3c3396aa13a5f239109a822666c4870490ad5 into ~/src/llama.cpp-d1d3c3396, `cmake -B build -DGGML_CUDA=ON -DCMAKE_CUDA_ARCHITECTURES=121` (build_flags_gx10 verbatim), `cmake --build build -j 16`: BUILD_EXIT=0, 285M, llama-bench/cli/server/tokenize/completion, cache GGML_CUDA=ON CMAKE_CUDA_ARCHITECTURES=121. The disk was checked first (174G free, 81%). No GPU lock was needed for a build. PROOF, committed as evidence/parity/pin-bump-d1d3c3396/RESOLVE-lambda.txt and RESOLVE-gx10.txt: `env -u LLAMA_BENCH_PATH bash scripts/llama_bin.sh` rc 0 on BOTH hosts. lambda: "ok llama.cpp pinned: …/llama.cpp-d1d3c3396/build/bin/llama-bench / build: version: 0.4.1-dev (build 10987, commit d1d3c3396) / cmake: GGML_CUDA=ON CMAKE_CUDA_ARCHITECTURES=unset / pin expires 2026-12-01". gx10: "… build: version: 0.4.1-dev (build 2423, commit d1d3c339) / cmake: GGML_CUDA=ON CMAKE_CUDA_ARCHITECTURES=121 …" (run from a scratch copy of this branch''''s llama_bin.sh + llama_pin.toml, since gx10''''s own checkout is stale). CASE TABLE, scripts/check_llama_pin.sh (ci-wired as before). Every row now runs against a FIXTURE canonical root, never the operator''''s $HOME. New rows: nothing named with the canonical build present -> rc 0; the canonical build at the WRONG commit -> wrong_build; the canonical build''''s cmake disagrees -> cmake_mismatch; a NAMED path wins over the canonical build -> wrong_build; UNPINNED never falls back to a canonical build -> unpinned; an oracle abbreviating SHORTER (7) and LONGER (10) than an 8-digit pin -> rc 0; 6 hex digits (below git''''s minimum) -> wrong_build; a different commit of the same length -> wrong_build. MUTANTS, each killed: canonical-lookup (the path pointed nowhere, as before) -> "expected rc=0, got rc=1 (no_binary_named)"; substring-match (the pre-#3563 test) -> "oracle abbreviates SHORTER than the pin expected rc=0, got rc=1 (wrong_build)". MEASURED at this commit: `bash scripts/check_llama_pin.sh` rc 0. #3726''''s owner (aprender-c3) was told to use the pinned llama-tokenize at the canonical path on both hosts, not ~/src/llama.cpp.' diff --git a/docs/roadmaps/entries/PMAT-3571.yaml b/docs/roadmaps/entries/PMAT-3571.yaml index 0eb4de2c64..fcae6e73d1 100644 --- a/docs/roadmaps/entries/PMAT-3571.yaml +++ b/docs/roadmaps/entries/PMAT-3571.yaml @@ -4,7 +4,7 @@ title: 'apr serve routes the Qwen3.5 hybrid to Qwen35Model instead of the dense path' status: in_progress priority: medium - assigned_to: null + assigned_to: aprender-c7 created: 2026-09-21T00:00:00Z updated: 2026-09-21T00:00:00Z spec: null @@ -14,4 +14,6 @@ estimated_effort: null labels: - kind:code - notes: 'ACCEPTANCE (hand-entered from gh#3571 done_when; see paiml-mcp-agent-toolkit#1414). Layer 1 of a surface no test had ever traversed: apr serve could not load the flagship architecture at all, and each assumption was visible only once the one above it cleared. Done when: handler_gpu_completion.rs branches on is_qwen35 and calls Qwen35Model::create_base_model rather than falling through to the dense OwnedQuantizedModel path, which is the wrong architecture for a Gated DeltaNet hybrid. Scope is deliberately ONE LAYER: land what works rather than hold the whole stack behind the last unknown. STOP: do not claim the serve path is end-to-end green on Qwen3.5 from this row alone — the layers above it are separate rows.' + notes: 'ACCEPTANCE (cop rulings, moved from acceptance_criteria at the 0.69.1 fold: check_roadmap_diff_additive forbids editing an existing entry''s acceptance_criteria, and notes is a lifecycle field): Cop ruling (aprender-04, 2026-09-21, after an external user hit #3571 on main), unit (1), verbatim: "FIRST, the smallest safe unit: no serve request handler may serve a model whose transformer stack resolved to 0 layers, on any arch and both routes. Refuse at load, naming the arch and #3571, with a case row plus a mutant. Stop at its receipt; I fold it with priority." | Cop condition (a), verbatim: "Siblings. List every other `apr serve` load entry point (APR and SafeTensors starts, and any CLI-side serve path in aprender-serve `cli`). In the body, say for each one whether it can reach a 0-layer model: guarded, or unreachable and why. A fix to one copy makes the other copies look reviewed." | Cop condition (b), verbatim: "(1) must not block (2). Qwen3.5''s generic stack is legitimately 0 layers, so when (2) lands, qwen35 must route to Qwen35Session BEFORE the check, or the check must test the stack the chosen route actually executes. Add a row to (2) proving qwen35 serve passes the refusal. (1) alone refuses qwen35 on main, which is correct until (2)." || ACCEPTANCE (hand-entered from gh#3571 done_when; see paiml-mcp-agent-toolkit#1414). Layer 1 of a surface no test had ever traversed: apr serve could not load the flagship architecture at all, and each assumption was visible only once the one above it cleared. Done when: handler_gpu_completion.rs branches on is_qwen35 and calls Qwen35Model::create_base_model rather than falling through to the dense OwnedQuantizedModel path, which is the wrong architecture for a Gated DeltaNet hybrid. Scope is deliberately ONE LAYER: land what works rather than hold the whole stack behind the last unknown. STOP: do not claim the serve path is end-to-end green on Qwen3.5 from this row alone — the layers above it are separate rows. + + UNIT (1), the zero-layer refusal (aprender-c7). Measured on #3571 (issuecomment-5763718338) at release/0.70-batch-1 262dfb2f6, which has the fix of #3678: the qwen35 base reached the dense servers with 0 layers — CPU route HTTP 200, content "", 1024 tokens of "\n" in 6.0 s; GPU route HTTP 500. On main it stops earlier at the refusal, so this unit is what keeps main honest when batch-1 lands. THE CHECK: `zero_layer_refusal(arch, layers)` (pure; ModelLoadFailed, exit 6, naming the arch, the 0 and #3571; the apr run/apr chat hint only for qwen35) inside `build_serve_model`, the one GGUF loader of apr serve. EVERY apr serve LOAD ENTRY POINT (condition a) — found by sweeping crates/apr-cli/src/commands/serve for from_mapped / from_apr / from_apr_file / SafetensorsToAprConverter / spawn_apr_q4k_inference_thread; ALL GUARDED: [1] GGUF --gpu, [2] GGUF --gpu --batch, [3] GGUF CPU — start_gguf_server takes its model from build_serve_model BEFORE the route is chosen. [4] GGUF --backend wgpu (try_start_wgpu_backend) and [5] the CPU fallback start_gguf_server_cuda rebuilds when CUDA init fails — both built their own with from_mapped; both now load through build_serve_model (one GGUF loader, one check). [6] APR CPU transformer (AprTransformer::from_apr_file), [7] SafeTensors single-file and [8] sharded (SafetensorsToAprConverter::convert / convert_sharded): the helper on t.config.num_layers where the transformer is in hand. [9] APR GPU (start_apr_server_gpu), [10] APR CPU quantized (try_apr_quantized_cpu), [11] SafeTensors GPU (start_safetensors_server_gpu), [12] SafeTensors CPU quantized (start_safetensors_server_cpu_quantized): the helper on the OwnedQuantizedModel before its Model ready line. [13] APR Q4K pool GPU (start_apr_q4k_server_gpu) parses its layer count on the scheduler thread after the upload begins, so the count the file DECLARES is read first (AprV2Model::load is mmap-based) and 0 is refused before the spawn; a file declaring none is refused by name inside the pool path (parse_apr_q4k_config: APR metadata missing num_layers). [14] aprender-serve cli (serve_model / prepare_serve_state): UNREACHABLE from apr — `git grep realizar::cli:: -- crates/apr-cli/src` has no hit — and it builds GGUF models with from_mapped, which refuses the qwen35 hybrid by name. REPO-LAYOUT FACTS a reviewer needs (round 1 lanes 1-3 misread them as compile errors; each checked at the head): handlers.rs:1642 is `include!("handler_gpu_completion.rs")`, handlers.rs:1643 `include!("server.rs")`, and handler_apr_cpu_completion.rs:869 `include!("handlers_include_01.rs")` — so build_serve_model and zero_layer_refusal, defined in handler_gpu_completion.rs, ARE items of the `handlers` module, callable unqualified from handlers.rs and handlers_include_01.rs, and reachable from the sibling module as `super::handlers::zero_layer_refusal` (pub(super)). Proof it compiles: `cargo test -p apr-cli --lib --features cuda zero_layer_refusal` builds apr-cli and passes both tests. CONDITION (b): on main this refuses qwen35 serve outright (correct until unit 2). Unit 2 (branch PMAT-3571-serve-qwen35-session) returns to the resident Qwen35Session BEFORE build_serve_model is called and adds the row proving qwen35 serve passes. TESTS: zero_layers_is_refused_by_name_and_any_layer_is_admitted (the case table: qwen35/0 and llama/0 refused, 1/28/24 admitted); serve_refuses_the_qwen35_base_at_load (real Qwen3.5-0.8B through build_serve_model; SKIPs where the file is absent — lambda/gx10 carry it, CI does not). MUTANT: deleting the check in build_serve_model turns serve_refuses_the_qwen35_base_at_load RED ("a 0-layer stack reached the serve routes"), measured.' diff --git a/docs/roadmaps/entries/PMAT-3676.yaml b/docs/roadmaps/entries/PMAT-3676.yaml new file mode 100644 index 0000000000..46e139e00f --- /dev/null +++ b/docs/roadmaps/entries/PMAT-3676.yaml @@ -0,0 +1,17 @@ +- id: PMAT-3676 + github_issue: 3676 + item_type: task + title: 'CI diet: aprender itself burns 99 intel runner-hours/day — coverage on tag/nightly only, path-filter book jobs, perf/model suites nightly, mutants only on .rs diffs' + status: planned + priority: critical + assigned_to: null + created: 2026-09-21T09:50:38Z + updated: 2026-09-21T09:50:38Z + spec: null + acceptance_criteria: [] + phases: [] + subtasks: [] + estimated_effort: null + labels: + - kind:code + notes: 'ACCEPTANCE (quoted from issue #3676 "Proposal" + "done_when"; operator 08:38Z: "YES, coverage on tags release only"; Noah approved all five workflow items directly to aprender-b3): 1. Coverage (ci / coverage) on the release tag and the nightly only, never on pull_request or merge_group. COV_FLOOR enforcement moves to the tag gate (T-2) and the nightly, where a regression still blocks the release. 2. Book Chapter Examples and mdBook: path-filtered. They run when book/** or the crates a chapter imports change, and nightly always. 3. The perf gate case table and the model-tests falsification suites move to nightly. The PR path keeps a smoke row. 4. mutants runs only when the diff has .rs. 5. Every moved job still runs on the tag. check_workflow_path_filters.sh and the gate aggregator are updated so a skipped job is a pass only when the skip reason is the documented filter. done_when: intel runner-minutes per 24 h drop by >= 40% at equal PR throughput (measured after merge); the tag gate still runs coverage with COV_FLOOR, and a planted coverage drop fails the tag (control); a book-only PR runs the book jobs, a crate-only PR not imported by a chapter does not (both rows); gate accepts skipped only for jobs whose filter says skip, any other skipped state fails (mutation). COP ADDENDUM (aprender-04, relaying rmedia-88): before removing coverage from ci.yml, enumerate every consumer of the coverage number and prove each still has a producer, with a guard asserting the chain. (A), THE ci / gate FIX (cop ruling 2026-09-21 ~15:00Z; batch-1 #3707 left this row out): the first spelling, `skip_coverage: true` on sovereign-ci@70e51ec, made `ci / gate` RED on every PR: that reusable''s gate counts a skipped coverage job as a mandatory failure ("coverage did not succeed (result: skipped)"). This PR''s own CI was red on it (run 35585906113, cancelled unread) and so was batch-1. The fix: pin sovereign-ci to paiml/.github f713290c86fcd70d6f26a0faab14e67e6713586f (#74 `coverage_on` + #77), pass `coverage_on: tag`, add `push: tags: [v*]` to ci.yml (the reusable''s second required edit), and remove coverage-nightly''s v* tag trigger so each tag is measured once, by ci.yml. The same pin also brings #73''s roadmap-fragment-parity, which would have refused every aprender PR (aprender still commits roadmap.yaml). #77 keys it on the BASE tree: NOT-ACTIVE while roadmap.yaml is tracked, and it arms by itself when #3709 migrates. scripts/check_coverage_has_producers.sh now asserts R3 (the nightly runs on schedule, NOT on v* tags) and R5 (ci.yml has coverage_on: tag, no live skip_coverage: true, and on: push tags [v*]), with 16/16 self-test rows including "skip_coverage: true instead of coverage_on -> RED". On f7b25a6c6 the guard reports R3 and R5 RED; after the change it PASSES. Also in this change (cop, found by aprender-f0): ci.yml runs `check_pr_closes_issue.sh --self-test` beside `--body`, because guard_tree skips it as wired-with-args and its case table ran in no job, and the ci.yml comment that claimed otherwise is corrected. PROOF REQUIRED before re-folding: this PR''s own CI green, with the gate logging `coverage: NOT MEASURED` and `roadmap-fragment-parity: NOT-ACTIVE`.' diff --git a/docs/roadmaps/entries/PMAT-3691.yaml b/docs/roadmaps/entries/PMAT-3691.yaml new file mode 100644 index 0000000000..87d28b0123 --- /dev/null +++ b/docs/roadmaps/entries/PMAT-3691.yaml @@ -0,0 +1,21 @@ +- id: PMAT-3691 + github_issue: 3691 + item_type: task + title: 'apr hex / apr trace still say "Invalid APR format" for GGUF and SafeTensors failures — 6 more InvalidFormat sites after #3661' + status: in_progress + priority: medium + assigned_to: aprender-37 + created: 2026-09-21T14:25:00Z + updated: 2026-09-21T14:25:00Z + spec: null + acceptance_criteria: + - 'Issue #3691 done_when 1, verbatim: "Each of the six sites uses `CliError::invalid_model_file(path, context, &e)` (added by #3661), or a variant naming the format it already knows. Exit code stays 4."' + - 'done_when 2, verbatim: "A row per command (`hex`, `trace`, and the SafeTensors header reader) on a truncated GGUF / malformed SafeTensors asserts that the message names the format and does not say "Invalid APR"."' + - 'done_when 3, verbatim: "The remaining `CliError::InvalidFormat(` sites (130 at `d0771c5c5`, including these six) are classified: APR-only input (keep) versus possibly-non-APR (move). The classification goes in the PR body."' + - 'The six sites, verbatim from the issue (line numbers at `d0771c5c5`): hex.rs:323 and hex.rs:350 `InvalidFormat(format!("Failed to parse GGUF: {e}"))`; safe_tensors_header.rs:180 `InvalidFormat(format!("Invalid SafeTensors header UTF-8: {e}"))`; safe_tensors_header.rs:182 `InvalidFormat(format!("Invalid SafeTensors JSON: {e}"))`; trace_likely_has_repeated.rs:267 `InvalidFormat(format!("Failed to parse GGUF: {e}"))`; trace_likely_has_repeated.rs:332 `InvalidFormat(format!("Failed to inspect SafeTensors: {e}"))`.' + phases: [] + subtasks: [] + estimated_effort: null + labels: + - kind:code + notes: 'Dispatched by the cop, aprender-04, to aprender-37 (2026-09-21 ~14:25Z). BASE: `0517e75a7` (PMAT-3661), not origin/main. #3691 reuses `CliError::invalid_model_file`, which #3661 added, and #3661 is not on main yet: it is folded into the unpushed `release/0.70-batch-1`. This branch is one commit on top of it, so the batch folds only this delta. HOW EACH done_when IS MET. (1) The six sites use `CliError::invalid_model_file(path, context, &e)`: hex.rs `parse_gguf` and `get_gguf_tensor_f32` (context "Failed to parse GGUF"), and trace_likely_has_repeated.rs `trace_gguf` ("Failed to parse GGUF") and `trace_safetensors` ("Failed to inspect SafeTensors"). Each has a path and an `AprenderError`, so the helper names the format from the file''s magic bytes and strips the core FormatError prefix. The two SafeTensors header-reader sites (UTF-8, JSON) have neither a path nor an `AprenderError`: `parse_safetensors_header(bytes)` gets the whole file, which `hex::run` already read, and a `Utf8Error` / `serde_json::Error`. They use a new sibling constructor, `CliError::invalid_model_bytes(bytes, message)`. It follows the same rule, reading the format from the bytes'' magic and never assuming it, through a new `hex::detected_format_of(bytes)`; `detected_model_format(path)` now calls that function after reading its 9 bytes. That is the done_when''s "variant naming the format it already knows". Every moved site keeps exit code 4: `InvalidModelFile` is on the same arm as `InvalidFormat`, and each row asserts `exit_code_value() == 4`. SCOPE, with one rule and its exclusions named. Besides the six, four more sites move: the rest of "the SafeTensors header reader", the unit done_when 2 names. Those are `parse_safetensors_header`''s size check ("SafeTensors file too small") and length check ("SafeTensors header length exceeds file size"), plus the "SafeTensors header is not a JSON object" check in each of its two callers (`run_safetensors`, `slice_safetensors`). The issue''s grep matched `format!(` calls only, so it missed these four string literals. Leaving them would keep printing "Invalid APR format: SafeTensors …" from the same reader. That makes 10 sites in 4 files. NOT moved, deliberately: hex.rs:353, `get_gguf_tensor_f32`''s tensor read. It also fires on `apr hex --slice … --tensor NAME` with a NAME that is not in the file (core says "Tensor ''NAME'' not found in GGUF"), so moving it would call a healthy file "Invalid GGUF file". It needs a split, not a move, and is classified below. The other 116 sites are classified, not changed. `CliError::InvalidFormat` itself is unchanged. TESTS (done_when 2). New `commands/model_file_error_tests_3691.rs`, declared as a `#[cfg(test)]` child of `hex` so it can call the private reader. It has one row per moved site, plus a second-caller row and an APR control. Each row renders `error: {e}` exactly as main.rs prints it and checks four things, the same as #3661''s table: the expected prefix; no "Invalid APR" unless the file is APR; no core "Invalid model format" anywhere; and exit 4. Rows that a file can reach call the real `hex::run` / `trace::run`: a truncated GGUF (hex, `hex --slice`, trace); SafeTensors with an overrunning header length, a non-UTF-8 header, and a cut-off header `{"a":` (hex, `hex --slice`, trace); and #3661''s 4 KiB noise named `.safetensors` (trace, by extension). Three sites cannot be reached through `hex::run`, because its `detect_format` picks SafeTensors only for 9+ bytes with `{` at byte 8: the 4-byte size check, and "not a JSON object" in both callers (a header starting with `{` is an object or not JSON at all). Their rows call the private function directly with the bytes that would reach them: 4 bytes, and a `[1]` header. The expected messages are: GGUF rows "Invalid GGUF file: Failed to parse GGUF: Unexpected EOF reading u64 at offset 24"; hex SafeTensors rows "Invalid SafeTensors file: "; the 4-byte and array rows "Invalid model file: …", because their magic matches no format; trace on the cut-off header "Invalid SafeTensors file: Failed to inspect SafeTensors: …"; trace on noise "Invalid model file: Failed to inspect SafeTensors: …"; and the APR control (a truncated .apr through `run_apr`, unchanged) "Invalid APR format: Failed to read APR: …". The fixtures are #3661''s, made `pub(super)` in model_file_error_tests_3661.rs instead of copied; that file''s own test is unchanged and still passes. CLASSIFICATION (done_when 3). It is in the commit message and goes in the PR body. There are 126 sites at `0517e75a7`: 125 `CliError::InvalidFormat(` lines plus one function-pointer use (dispatch_run.rs:242 `.map_err(CliError::InvalidFormat)`, which the `(` grep misses). The issue''s 130 was `git grep ''InvalidFormat(''` at `d0771c5c5`: 128 sites, the variant declaration, and its exit-code arm. #3661 moved 3. Classes, decided by the input that reaches each site: MOVED here, 10; MOVE, model file (may be GGUF, SafeTensors, or unrecognised, so "Invalid APR format" can be false), 33; MOVE, not a model file (CLI arguments, JSON/CSV/corpus inputs, HF responses, serialising our own reports, IO), 58; KEEP, APR-only input, 15; TEST, asserting on the variant, 10. A script checked that every one of the 126 appears in exactly one class. STOP LIST: no change to `InvalidFormat`''s Display or to any of the 116 unmoved sites; no new error variant; no edit outside apr-cli except this fragment and the regenerated roadmap.yaml; no push and no arm, because the cop folds this into the batch. MEASURED at this commit: `cargo test -p apr-cli --lib`: 7295 passed, 0 failed, 12 ignored (#3661''s 7294 plus this table). `cargo test -p apr-cli --lib model_file_error_tests`: 2 passed, this table and #3661''s, whose fixtures it now shares. `cargo clippy -p apr-cli --lib -- -D warnings`: rc 0. `--tests` fails only on 2 findings in the untouched pre-existing `commands/nf4_classifier.rs`, the same state #3661 recorded; none are in a touched file. `cargo fmt --all -- --check`: rc 0. Rustfmt does not reach safe_tensors_header.rs through its `include!`, so its changed lines were formatted by hand, and a whole-file rustfmt was not applied because it rewrites untouched lines. `scripts/check_complexity_ratchet.sh`: "a877fa056 vs … none new, none grown". `bash scripts/guard_tree.sh --no-cargo`: 76 checks, 0 failed. `check_roadmap_fragment_required.sh`, `roadmap_fragments.py aggregate --check`, `check_roadmap_sorted.sh` and `check_roadmap_ids_unique.sh`: all PASS. MUTANTS, run one at a time against this table, each RED: S01–S10, each moved site reverted alone to its pre-fix `CliError::InvalidFormat(…)`, fails exactly its own row. For example, S01 renders "error: Invalid APR format: Failed to parse GGUF: Invalid model format: Unexpected EOF reading u64 at offset 24", the issue''s string. With all 10 reverted, the 12 non-control rows fail (29 checks) and the APR control still passes. H1, `invalid_model_bytes` hardcoded to "SafeTensors" (the assumed format), fails the 3 "Invalid model file" rows (4 bytes, array header ×2). H2, `detected_format_of` always returning "model", fails the 8 rows that expect GGUF or SafeTensors. Sources were restored and checked by md5 before the full runs. NOT DONE: a run of a freshly built `apr` binary. The host was at load 55–64 on 48 cores, and a release build of the facade would have added to it. The table renders the same `error: {e}` line main.rs prints and asserts the exit-code mapping directly.' diff --git a/docs/roadmaps/entries/PMAT-3693.yaml b/docs/roadmaps/entries/PMAT-3693.yaml new file mode 100644 index 0000000000..a384ecaa3c --- /dev/null +++ b/docs/roadmaps/entries/PMAT-3693.yaml @@ -0,0 +1,21 @@ +- id: PMAT-3693 + github_issue: 3693 + item_type: task + title: 'Qwen3.5-9B Q4_K_M on CUDA at ~12k-token context corrupts identifiers it quotes from the prompt and runs at 1.0 tok/s: discriminate model weakness from long-context CUDA divergence' + status: in_progress + priority: critical + assigned_to: aprender-c3 + created: 2026-09-21T15:55:00Z + updated: 2026-09-21T15:55:00Z + spec: null + acceptance_criteria: + - 'Issue #3693 done_when 1, verbatim: "**discriminate first, then fix.** The same prompt, same seed, temperature 0: CPU (`--no-gpu`) vs CUDA, token by token. Identical corruption means model/quant weakness (document it, then compare a Q8_0/F16 9B); divergence means a CUDA path defect."' + - 'done_when 2, verbatim: "`apr parity` (or `--per-op`) at ≥ 8k positions, CPU vs CUDA, with the first position where cosine drops below the C14 floor named. C14''s corpus gains a long-context row, so the gate covers the context length the product is used at."' + - 'done_when 3, verbatim: "decode rate at 12k context measured separately from prefill, with `--json` timing fields. 1.0 tok/s end-to-end has to be attributed (prefill vs decode, and KV-cache growth)."' + - 'done_when 4, verbatim: "if CUDA diverges: the fix plus a falsifier that goes RED on the 12k-position corpus"' + phases: [] + subtasks: [] + estimated_effort: null + labels: + - kind:code + notes: 'Routed by the cop (aprender-04) at 2026-09-21 ~15:45Z onto the 0.69.1 path (#3710 bar, context row: "each of the four at the largest prompt the three consumers actually send, derived from their measured traffic, never picked here", with RAH''s review lane measured at ~44 KB, about 12k tokens). The input is paiml-implement-c9''s prompt at /mnt/nvme-raid0/agent-wt/gtmp-318/apr-lane/lane.json.prompt.txt on lambda (44667 bytes, sha256 ed8b3007…). Model /home/noah/models/Qwen3.5-9B-Q4_K_M.gguf, sha256 03b74727…, identical on lambda and gx10. Binary: the 0.69.0 release build, `apr 0.69.0 (225b2a9ab)`, at /mnt/nvme-raid0/targets/ladder-069/release/apr on each host. Measurements are recorded on the issue as they land. DISCRIMINATION RESULT (aprender-c3, 2026-09-21): not a CUDA defect and not model weakness. (a) The gdn_decode_attention multi-pass path (>4096 positions) is correct: a new device test at the default cap, seq_len up to 12,288, passes against the CPU port. (b) RoPE is ruled out: rope.dimension_sections [11,11,10,0] make IMROPE equal NeoX for text. (c) On arbiter''s r891 prompt, apr CPU and apr CUDA BOTH lose to the pinned llama.cpp at copy positions from about 250 tokens (pos 252: llama.cpp 17114 `maps` at logprob -0.001, apr 2117 `map`). (d) The cause both share is the tokenizer: apr''s GGUF encoder is greedy longest-match, so the prompt reaches the model as ids it never saw. That is #3726, which carries the fix. This row closes as #3726 if the re-measure after the fix shows no corruption; any remainder stays here. The 1.0 tok/s is serial prefill (qwen35 CUDA has no batched prefill), owned by #3596.' diff --git a/docs/roadmaps/entries/PMAT-3694.yaml b/docs/roadmaps/entries/PMAT-3694.yaml new file mode 100644 index 0000000000..56ebd9be6c --- /dev/null +++ b/docs/roadmaps/entries/PMAT-3694.yaml @@ -0,0 +1,21 @@ +- id: PMAT-3694 + github_issue: 3694 + item_type: task + title: 'parity_receipt_denominator.sh needs a python-free classify() — then drop its UNMEASURED path' + status: in_progress + priority: high + assigned_to: aprender-0e + created: 2026-09-21T14:52:00Z + updated: 2026-09-21T14:52:00Z + spec: null + acceptance_criteria: + - 'Issue #3694 done_when 1, verbatim: "`parity_receipt_denominator.sh` classifies record / legacy / other with no python3 on PATH, by a real parse, and its self-test runs every row on a python-free runner."' + - 'done_when 2, verbatim: "The exit-3 UNMEASURED path and `ont4c3_parity_receipts`''s acceptance of exit 3 are removed, so a runner without the tool can no longer pass the test unmeasured."' + - 'Issue body, verbatim: "The predicate stays an independent implementation, deliberately not the Rust extractor''s, and it must still PARSE the JSON rather than substring-match it (a substring test standing in for a parse is refused). One option is a tiny parse in a tool the runners do carry."' + - 'Cop dispatch (aprender-04 to aprender-0e, 2026-09-21), verbatim: "done_when: the real corpus still gives `PASS 7 receipt(s)`; a stub PATH with NO python3 gives the same PASS; the git-refusal row (#3682) is unchanged; a mutant goes RED. No jq either unless you prove the runner has it (clean-rooms have none, and yoga''s is jq 1.6)."' + phases: [] + subtasks: [] + estimated_effort: null + labels: + - kind:code + notes: 'Follow-up to #3695 (the stop-gap that made a python-free runner UNMEASURED exit 3) under the cop ruling on #3689. SCOPE: scripts/parity_receipt_denominator.sh, crates/aprender-contracts-cli/tests/ont4c3_parity_receipts.rs, and this fragment. HOW IT IS MET. classify() is POSIX awk, run under LC_ALL=C: a strict RFC 8259 parser, iterative (an explicit container stack, so no recursion limit), that validates UTF-8 inside strings (no overlongs, no surrogates, nothing past U+10FFFF) and keeps only the ROOT object''s keys, each value''s kind and the decoded text of a string value. The rule is unchanged: record iff the root is an object whose schema is the string apr-parity-receipt/v2; legacy iff not, and metrics is an array or parity is present; anything else, including any syntax error, is other. A later duplicate key wins, as in json.load and serde_json. A raw NUL byte is decided in bash (other), because busybox awk holds C strings and would truncate the document. It is still an independent implementation: no code shared with extract:parity-receipt. The one deliberate difference from the python it replaces: NaN / Infinity are not JSON (RFC 8259), and serde_json refuses them too, so they are other. REMOVED: PY_BIN / PARITY_PYTHON, the command -v check and its UNMEASURED return 3, the --self-test UNMEASURED early exit, and the Rust test''s exit-3 acceptance. There is no exit 3 any more. A missing awk, or one that answers nothing, is ENV exit 2, RED (seam PARITY_AWK). SELF-TEST, 9 rows (was 7): the 5 original rows unchanged, including the #3682 git-refusal row; NEW "a PATH with no python3 (only git awk sort grep head tr wc) measures the same PASS", which also lists the script''s dependencies; NEW "a missing classifier is ENV exit 2 naming it, never a count of 0 or UNMEASURED", replacing the #3695 no-interpreter row; "a classifier that answers nothing is ENV exit 2" (#3695, kept); NEW a 35-row classification table (escaped slash, a \u-escaped key, quotes and brackets inside strings, schema nested or only in prose, a root array, a truncated document, trailing content, a trailing comma, single quotes, NaN, a leading zero, a raw tab, a bad escape, latin-1, a UTF-8 surrogate, a NUL, a BOM, duplicate keys in both orders, and legacy metrics[] / parity / an escaped metrics key). MEASURED at this commit. `bash scripts/parity_receipt_denominator.sh` rc 0 "PASS 7 receipt(s) under evidence/parity/**, and evidence/parity/EXPECTED_RECEIPTS says 7." --self-test rc 0, 9 ok, under gawk 5.1, mawk 1.3.4-20200120 and busybox awk (PARITY_AWK). THE YOGA SHAPE, a PATH of every /usr/bin and /bin executable except python* (2832 tools, command -v python3 finds nothing): the real corpus gives the same rc 0 PASS 7, --print gives 7, and --self-test gives 9 ok. THE RUNNER IMAGE: sovereign-ci is FROM rust:1.95-slim-bookworm (infra machines/intel/sovereign-ci/Dockerfile). In debian:bookworm-slim plus git (mawk 1.3.4.20200120-3.1, python3 NONE, jq NONE) the self-test gives 9 ok and the real corpus gives PASS 7 rc 0; in rust:1.95-slim (trixie, mawk 1.3.4.20250131-1, python3 NONE, jq NONE) the same. awk is Priority: required in Debian, and 52 of 131 scripts/check_*.sh already run it on every runner. DIFFERENTIAL against the old python classify(), over all 118 tracked files under evidence/parity/** plus 55 adversarial documents including a 3000-deep nesting: identical under gawk and mawk except NaN and -Infinity (the deliberate difference above); 7 records in both. The 35 table rows agree with the python rule on 34; the 35th is the NaN row. MUTANTS, each run against --self-test, all RED: a substring classifier (grep for the schema) gives rc 1 with 23 FAIL rows; the python classify() restored gives rc 1 ("a python-free PATH gave exit 2 … python3: command not found"); the first duplicate key wins gives rc 1 on both duplicate rows; UTF-8 validation removed gives rc 1 (latin-1, surrogate); trailing content accepted gives rc 1; the schema-kind check removed gives rc 1 ({"schema":"","schema":2} -> record); the silent-classifier arm restored to fall through as other (#3695) gives rc 1, "the tree holds 0"; the NUL pre-check removed gives rc 1 under busybox awk and rc 0 under gawk, so that guard is load-bearing only for C-string awks. THE RUST TEST: `cargo test -p aprender-contracts-cli --test ont4c3_parity_receipts` 9 passed with python3 on PATH, and 9 passed on the yoga-shape PATH plus ~/.cargo/bin (no python3). PARITY_AWK=/bin/true fails it, "exit Some(2)" plus the ENV line. MUTANT: the pre-fix script (225b2a9ab) on the python-free PATH fails the new test with "exit Some(3)" and the UNMEASURED line, so the exit-3 acceptance is gone. bashrs lint 0 errors; check_shell_lint_ratchet.sh PASS (ratcheted); cargo fmt --all -- --check rc 0. Not in scope: #3697 (the other guards that need python3), which is the next row in this lane.' diff --git a/docs/roadmaps/entries/PMAT-3697.yaml b/docs/roadmaps/entries/PMAT-3697.yaml new file mode 100644 index 0000000000..f0bc5428b2 --- /dev/null +++ b/docs/roadmaps/entries/PMAT-3697.yaml @@ -0,0 +1,21 @@ +- id: PMAT-3697 + github_issue: 3697 + item_type: task + title: 'guard-tree python dependence, batch 1 of #3697: 4 guards python-free or UNMEASURED (t2_preflight_verdict, no_hosted_runners, nextest_ci_profile_no_fail_fast, roadmap_ids_unique) + scripts/lib/python_fleet_state.sh' + status: in_progress + priority: high + assigned_to: aprender-0e + created: 2026-09-21T15:20:00Z + updated: 2026-09-21T16:40:00Z + spec: null + acceptance_criteria: + - 'Issue #3697, verbatim: "**What lands (0.70):** per guard, one of: - (a) remove the python dependency (the root fix, the direction of #3694: no python in automation). For YAML reads, prefer `pv`/bashrs/Rust readers the repo already builds. - (b) where a python reader stays, classify its absence as fleet state: `UNMEASURED … reason=`, exit 0, surfaced by guard_tree (#3651), never red. That is the ruling applied to fleet-pv (#3669''s andon) and to pathonly (#3692)."' + - 'Issue #3697, verbatim: "My own `check_nextest_ci_profile_no_fail_fast.sh` (#3626) is on the no-python3 list. It reads TOML through a purpose-built fallback on 3.10, but with no interpreter at all it reports ENV rc 2, which guard_tree counts as FAIL: same class, (b) applies."' + - 'Cop dispatch (aprender-04 to aprender-0e, 2026-09-21), verbatim: "Each guard becomes python-free, or UNMEASURED per the #3692 ruling. One PR per 3–4 guards so the reviews stay small." and "same proof shape (no-python3 PATH + debian:bookworm-slim)".' + - 'NOT in this batch (it lands with the last batch, when it can be green): the issue''s "one guard-tree row: `guard_tree.sh --no-cargo` under a `python3 -S` stub and a no-python3 stub. It must end `0 failed`". Today 18 other guards still fail it.' + phases: [] + subtasks: [] + estimated_effort: null + labels: + - kind:code + notes: 'SCOPE: scripts/lib/python_fleet_state.sh (new, sourced), scripts/check_t2_preflight_verdict.sh, scripts/check_no_hosted_runners.sh, scripts/check_nextest_ci_profile_no_fail_fast.sh, scripts/check_roadmap_ids_unique.sh, and this fragment. The rest of #3697 (18 guards, and the stub-shaped guard-tree row) is later batches. (a) PYTHON-FREE. t2_preflight_verdict: the self-test''s mutant splice (replace decide() with the pre-#3561 absence-as-consent body) is awk over a quoted-heredoc mutant, cut from the `decide() {` line through the first line that is exactly `}`; a subject with no such close is ENV. The spliced file is BYTE-IDENTICAL to the python splice''s output under gawk 5.1, mawk 1.3.4 and busybox awk. no_hosted_runners: the scan is awk; POSIX ERE has no \b, so the token''s boundaries are spelled `(^|[^A-Za-z0-9_])…([^A-Za-z0-9_]|$)`. DIFFERENTIAL vs the python scan (cwd-relative paths, same baseline semantics, sorted stale list): identical stdout+rc on the real 21 workflows (with and without a baseline), on a 611-violation adversarial corpus (every token shape x 11 prefixes x 12 suffixes), and on two baseline corpora, under all three awks, EXCEPT a token touching a non-ASCII letter (e.g. ubuntu-latesté): python (Unicode \w) passed it, awk (ASCII word chars) flags it. 0 violations lost, 111 added, all with é. Rows 13-15 pin both boundaries and the ASCII choice (rows 13/14 agree with python; row 15 is the deliberate difference). (b) UNMEASURED. scripts/lib/python_fleet_state.sh: py_fleet_state [module...] probes the interpreter (PY_FLEET_PYTHON, default python3) by importing each module; only ModuleNotFoundError is `not installed`. rc 0 ok; rc 3 fleet state with ONE stderr line `UNMEASURED runner= reason=no-interpreter interpreter= guard=` (absent, or exec 126/127) or `reason=no-module module=`; rc 2 ENV for an interpreter that exists and dies or answers nothing. Reason names: the tree already says `no-interpreter` (#3692 pathonly, #3695 denominator), so that is kept over the issue''s example `no-python`, and `no-module module=yaml` names the module rather than coining one reason per module. Sourced, option-neutral (check_sourced_libs_option_neutral.sh OK: 9 libraries). Its case table py_fleet_state_self_test (5 stub rows + 2 real-interpreter rows that print UNMEASURED where there is none) runs inside each consuming guard''s --self-test, because a scripts/tests file would be dark without a workflow edit. nextest_ci_profile_no_fail_fast: judge() probes first; no interpreter -> rc 3 -> the run exits 0 with the UNMEASURED line and no PASS line; /bin/false still ENV rc 2; the old row `no interpreter at all -> ENV rc=2` is now `-> UNMEASURED rc=3` plus a run-level row; the reader table (both readers) is UNMEASURED where python is absent. roadmap_ids_unique: scan() needs python3 + PyYAML; either missing -> UNMEASURED exit 0; a dying python3 -> ENV; the 5 YAML rows are UNMEASURED where there is no reader; the missing-file row and two new fleet-state rows run everywhere. MEASURED, guard_tree.sh --no-cargo on 225b2a9ab + this batch, each shape a python3 stub first on PATH: normal python3 76 checks, 0 failed (unchanged); python3.10 -S 14 -> 12 FAIL rows (roadmap_ids_unique s+r gone); no python3 (stub exits 127) 29 -> 22 FAIL rows (all 7 rows of these 4 guards gone), and under PASS guard_tree surfaces the UNMEASURED lines (#3651). debian:bookworm-slim (sovereign-ci''s FROM base) + git, python3 NONE, jq NONE, mawk 1.3.4.20200120-3.1: t2_preflight_verdict self-test rc 0 (2 ok) run rc 0; no_hosted_runners self-test rc 0 (15 ok) run rc 0 `OK: 21 workflow files`; nextest self-test rc 0 (9 ok, 3 UNMEASURED) run rc 0; roadmap_ids_unique self-test rc 0 (8 ok, 3 UNMEASURED) run rc 0 with the UNMEASURED line. MUTANTS, each against a scratch copy, all RED: py_fleet_state mapping every failure to UNMEASURED -> roadmap_ids_unique self-test rc 1 (`a python3 that dies -> ENV exit 2, never UNMEASURED`, plus the helper''s crash rows); no_hosted_runners leading boundary dropped -> rc 1 (row 13 xubuntu-latest); t2 splice made a no-op -> rc 1 (`the pre-#3561 mutant PASSED the case table`). bashrs: check_shell_lint_ratchet.sh PASS (errors only may not rise).' diff --git a/docs/roadmaps/entries/PMAT-3699.yaml b/docs/roadmaps/entries/PMAT-3699.yaml new file mode 100644 index 0000000000..7aefb0a0dd --- /dev/null +++ b/docs/roadmaps/entries/PMAT-3699.yaml @@ -0,0 +1,20 @@ +- id: PMAT-3699 + github_issue: 3699 + item_type: task + title: 'prepare_bump.sh --ship opens a bump PR whose body fails §6 R-2 (check_pr_closes_issue): the CHANGELOG''s context refs (#EPIC, ''refs #N'') have no keep-open line — hit on 0.68.2 (#3498) and 0.69.0 (#3698)' + status: in_progress + priority: high + assigned_to: aprender-f0 + created: 2026-09-21T13:03:24Z + updated: 2026-09-21T15:20:00Z + spec: null + acceptance_criteria: + - 'Issue #3699 done_when 1 AS AMENDED on the issue by its author (comment 2026-09-21T15:17:48Z, "done_when 1 amended (issue author, cop, 2026-09-21), replacing the original wording"), verbatim: "`prepare_bump.sh --ship` derives every `#N` the §6 R-2 guard owes a reason for (`check_pr_closes_issue.sh --list-owed`: not under a closing keyword, not proven a PR or a closed issue; an unresolvable ref counts) and writes ONE `keep-open:` line naming them (\"cited by the CHANGELOG for context; each closes via its own PR; the release EPIC closes at T-4\")"' + - 'done_when 2, verbatim: "before `gh pr create` it runs `scripts/check_pr_closes_issue.sh --body ` and refuses to open the PR on rc != 0"' + - 'done_when 3, verbatim: "a case-table row: a CHANGELOG with an EPIC ref and a `refs #N` produces a body the guard PASSes; a mutant that drops the keep-open line goes RED"' + phases: [] + subtasks: [] + estimated_effort: null + labels: + - kind:code + notes: 'Assigned by the cop (aprender-04) as the first item of the release-tooling lane, to land before the 0.70 bump. Fixtures and stubs only: nothing here ran against the live 0.69 train, its state dir, or a publish token. SCOPE: scripts/release/prepare_bump.sh, scripts/check_pr_closes_issue.sh, the new scripts/check_release_bump_pr_body.sh, and this fragment. HOW IT IS MET. (1) --ship builds the body BEFORE the cargo checks, the commit and the push (it used to be built after the push). It asks the guard for the refs that owe a reason through a new `check_pr_closes_issue.sh --list-owed --body FILE`, which prints the output of the SAME classify_refs() the judge uses (refactored out of check_body_text, behaviour unchanged), and appends ONE line `keep-open: #A #B -- cited by the CHANGELOG for context; each closes via its own PR; the release EPIC closes at T-4`, whose reason is done_when 1''s parenthetical, verbatim. The epic-and-refs row pins the whole line. When nothing is owed there is no line. WHICH REFS (done_when 1 as amended; the ORIGINAL read "derives every `#N` in the generated body that isn''t under a closing keyword", and quorum round 2''s lane 1 FAILed on that literal reading, which is why the issue author amended it): the guard''s own set, so references PROVEN to be pull requests or closed issues are left out, exactly as the judge leaves them out, and an unresolvable ref is kept (fails closed). Measured on the real 0.69.0 body: 61 distinct #N, of which the guard owes 5 (#3080 #3545 #3571 #3602 #3658), which is the same 5 the CI step flagged and aprender-62 named by hand. Naming all 61 would list merged PRs in a keep-open line. THE REASON TEXT is the ticket''s own. Quorum round 1 FAILed an earlier wording that described the close MECHANISM instead (lane 1, gemini-3.1-pro-high: the reason must be done_when 1''s text), and the ticket''s text wins. Its last clause is a statement about WHEN, and it holds today: the epic is closed at T-4, by hand (0.68.2''s #3477, closer=null, 2026-09-20T08:23Z; the cop is closing 0.69.0''s #3080 right after the crates.io publish). #3708, same owner and queued before the 0.70 bump that first emits this line, makes the close step do it mechanically: autopilot.sh close (lines 301-307) comments on the epic, then requires 0 open milestone items, and closes nothing. The reason carries no `#`, so the line cannot add a closing reference (a line that closed the epic on the bump''s merge would be the #3400 landmine). (2) `bash check_pr_closes_issue.sh --body pr_body.md || die "the bump PR body fails §6 R-2; nothing committed, pushed or opened"` runs before the cargo checks, the commit, the push and `gh pr create`. (3) scripts/check_release_bump_pr_body.sh is a scripts/check_*.sh, so guard_tree.sh runs it bare in guard-tree with no workflow edit (the check_t2_preflight_verdict.sh pattern). It runs the REAL prepare_bump.sh --ship end to end in a throwaway fixture: a local bare origin, a bump clone on release-9.9.9 with the CHANGELOG section uncommitted, a T-2 GO receipt, stubs for gh (records calls, captures --body-file), cargo and the arm helper, and ref kinds from the guard''s PR_CLOSES_REF_KIND_CMD seam. ROWS: guard-case-table (runs check_pr_closes_issue.sh --self-test, see DARK TABLE below); epic-and-refs (EPIC #9002 + `refs #9005` + PR #9001 + `closes #9006` -> PR opened, body PASSes the guard, ONE keep-open line naming exactly #9002 #9005, not the PR and not the closed-by-keyword ref); no-keep-open (that body minus its keep-open line -> guard rc 1); adds-no-close (the keep-open line alone -> --list-owed lists every ref it names); prs-only (no line, still PASS); landmine (`no-close: #9005` -> non-zero, no `gh pr create`, no branch on origin). COMMITTED MUTANTS, run on every bare run: drop-keep-open (the printf line removed) is killed by epic-and-refs, "exited 1: STOP the bump PR body fails §6 R-2"; drop-refusal is killed by landmine, "exited 0 on a landmine body". A missing subject or mutation anchor is ENV rc 2, never a pass. The guard table gains list-owed (#9001 pr, #9002 open, #9003 unresolvable, #9004 closed, #9005 under Closes -> exactly "#9002 #9003") and list-owed-empty-body (rc 2). MEASURED at this commit: `bash scripts/check_release_bump_pr_body.sh` rc 0, "PASS 8 row(s)"; `bash scripts/check_pr_closes_issue.sh` rc 0, "self-test OK: 21 case(s)". The first run of the new table caught a real bug in this change: the keep-open line read "#9005-- cited", because $(...) strips the trailing newline. LIVE REPLAY: the real CHANGELOG [0.69.0] section through the real prepare_bump.sh --ship, with gh `api` passed to the real gh and only `pr create` stubbed. rc 0, 122 api calls (two passes over 61 refs), 1 pr create. Keep-open line "keep-open: #3080 #3545 #3571 #3602 #3658 -- cited by the CHANGELOG for context; each closes via its own PR; the release EPIC closes at T-4". The unmodified guard with LIVE resolution on the opened body: "PASS: non-closing ref(s) #3080 #3545 #3571 #3602 #3658 allowed by a keep-open reason." Zero closing references in the body. AD HOC MUTANTS, not committed, each RED: a reason ending "the release EPIC closes #9002" turns 2 rows RED (epic-and-refs, adds-no-close: "names '#9002 #9005 #9002 ' but only '#9005 ' are still owed"); naming every #N instead of --list-owed turns 3 rows RED (epic-and-refs, adds-no-close, prs-only); moving the refusal after the push turns landmine RED, "the branch was pushed before the refusal"; a --list-owed that prints every ref turns 3 rows of the WIRED table RED (guard-case-table, epic-and-refs naming #9006, prs-only). DARK TABLE, pre-existing and found here: `bash scripts/guard_tree.sh --no-cargo` prints `skipped: scripts/check_pr_closes_issue.sh -- wired-with-args in ci.yml`, and ci.yml line 1398 runs that guard only with --body. So its case table ran in NO CI job, even though the ci.yml comment at lines 1368-1373 still says guard_tree runs it bare. Its new list-owed rows would have been dark too. The guard-case-table row gives that table a caller inside guard-tree with no workflow edit. The stale ci.yml comment is left for a workflow-authorized change. GUARD_TREE: `bash scripts/guard_tree.sh --no-cargo` at fc5fdee22 (7 rows then): "77 checks, 0 failed", with "PASS scripts/check_release_bump_pr_body.sh [run]". OUT OF SCOPE, observed: a CHANGELOG bullet whose PR title carries a closing keyword ("fix: closes #N") would still close #N when the bump merges. The guard PASSes that, correctly by its own rule, and nothing here changes it.' diff --git a/docs/roadmaps/entries/PMAT-3703.yaml b/docs/roadmaps/entries/PMAT-3703.yaml new file mode 100644 index 0000000000..ed5da31591 --- /dev/null +++ b/docs/roadmaps/entries/PMAT-3703.yaml @@ -0,0 +1,20 @@ +- id: PMAT-3703 + github_issue: 3703 + item_type: task + title: 'aprender-verify-ml test_pipeline_generate_small asserts wall-clock (generation_time_ms > 0) — failed v0.69.0''s clean-room B2 on a fast gx10 runner' + status: in_progress + priority: critical + assigned_to: aprender-fd + created: 2026-09-21T14:20:00Z + updated: 2026-09-21T15:22:00Z + spec: null + acceptance_criteria: + - 'Issue #3703 done_when 1, verbatim: "the assertion tests what the code promises (the stat is populated and monotone), never elapsed wall time. E.g. drop the timing clause, or record µs and assert `generation_time_us` is set, not `> 0`"' + - 'done_when 2, verbatim: "a sweep of every test in the crates clean-room B2 runs (`--lib` of the published crates) for `as_millis() > 0` / `elapsed ... > ` / `Instant`-based assertions, with each hit fixed or listed with its reason. `scripts/check_no_timing_in_required.sh` extended to see this form (it passed while this sat in a required gate)"' + - 'done_when 3, verbatim: "mutation proof: a test build with an instantaneous generator must still PASS"' + phases: [] + subtasks: [] + estimated_effort: null + labels: + - kind:code + notes: 'Handed to aprender-62 by the cop after infra-c8 was redirected by the operator (#3703 was unstarted). #3702, filed a minute earlier for the same defect, was closed as a duplicate. THE FAILURE: v0.69.0 clean-room (paiml/infra run 35608872676, gx10-pool1, "Run clean-room gates") GATE B2: aprender-verify-ml data::pipeline::tests::test_pipeline_generate_small, "assertion failed: stats.generation_time_ms > 0 || stats.total_generated < 10". `generation_time_ms` is `elapsed.as_millis() as u64` and the test asks for count(10), so a host that generates 10 programs in under 1 ms reads 0. It did not reproduce on lambda''s debug build (0/40 runs failed there), which is why PR CI and the merge group were green. DONE_WHEN 1: the timing clause is replaced by what generate() promises, `assert_eq!(stats.total_generated, programs.len())`, which holds by construction (`total_generated: all_programs.len()`). No clock is asserted. DONE_WHEN 3, measured: with the generator made instantaneous (`generation_time_ms` forced to 0), the fixed test PASSES ("1 passed"), and the OLD clause under the same mutant FAILS with exactly the clean-room message, "assertion failed: stats.generation_time_ms > 0 || stats.total_generated < 10". So #3703 reproduces deterministically on any host. DONE_WHEN 2, the sweep. Scope: every `crates/*/src/**.rs` (the `--lib` surface). The #3703 shape is an INTEGER duration (`.as_millis()`, `.as_micros()`, `.as_secs()`, or a name ending _ms/_us/_secs) asserted `> 0`, `>= 1` or `!= 0`, which truncates to 0 on a fast host. A float `> 0.0` on `as_secs_f64()` does not truncate. 104 hits. FIXED, 3, each a measured duration with nothing guaranteeing it is positive: pipeline.rs:438 (above), aprender-orchestrate bug_hunter/tests_patterns.rs `hunt_result.duration_ms > 0` (dropped; result.is_ok() and the parsed claims already prove the hunt ran) and aprender-train finetune/instruct_trainer.rs `result.total_time_ms > 0` (dropped; the epoch count is asserted). LISTED WITH ITS REASON, 101, in the new scripts/wallclock_assert_baseline.txt, each line read and classified: 76 declared render budgets (`budget()` returns BrickBudget::uniform(16) and similar; constants), 9 configured values (timeouts, windows, intervals, chunk sizes, options), 7 epoch timestamps (`timestamp_ms` from SystemTime), 1 cost-model estimate, 4 fixture inputs (health_tests Duration::from_millis(100), process_tracer from_micros(100), stats u64::MAX/2, format_chat latency_us=5000), and 4 real measurements taken AFTER a sleep of at least the truncation unit (cache/tests.rs sleeps 2 ms; falsification/types.rs sleeps 1 ms; brick_tracer sleeps 10 us and 100 us). THE GUARD: check_no_timing_in_required.sh PART 4 scans with that pattern (comment lines excluded by the same matcher) and ships a 15-row case table (7 must-match, including the exact pipeline line; 8 must-not-match, including `> 0.0`, `as_secs_f64() > 0.0`, `>= 10`, a comment, and a name that only contains `_ms`). A pattern that disagrees with its table is refused before any scan result is trusted. THE LEDGER IS KEYED ON THE ASSERTION TEXT, not on `path:line`: each line is ``, so the 101 hits are 32 recorded texts. A coordinate key was the first version (handed over at 0221c7c1b) and was measured to be a trap. With the ledger at a comparand (BASELINE_RATCHET_BASE_REF=HEAD), one unrelated comment line inserted at the top of widgets/border.rs made PART 4 report border.rs:755 NEW and :754 stale. Re-pointing the entry, which is the remedy it printed, was then REFUSED by the set-aperture ratchet ("the owning guard is unchanged in this diff"), so any unrelated edit above any of the 101 lines in ~50 files would have turned guard-tree red with no in-branch fix. The ratchet is now `keyed` (classified in check_baseline_ratchets.sh): no text may appear and no count may rise. PART 4 fails on NEW (a text the tree holds more often than recorded, so a COPY of a recorded line fails too), on STALE (a recorded count above the tree''s: lower it in the commit that fixed the line, which the ratchet allows), on a malformed ledger line, and on a scan of 0 files. GUARD MUTANTS, measured on this tree with BASELINE_RATCHET_BASE_REF=HEAD: M0 unmutated rc 0; M1 the #3703 line restored rc 1 (NEW, pipeline.rs:442); M2 the trap above, an unrelated insertion over a recorded line, rc 0; M3 a copy of `assert!(budget.total_ms > 0);` added in a new place rc 1 (count rises); M4 every data line removed from the ledger with the hits still in the tree rc 1; M5 a recorded count raised rc 1 (stale, and the ratchet reports RAISED); M6 a new key appended rc 1 (NEW KEY); M7 a malformed ledger line rc 1; M8 the `_ms` alternative dropped from the pattern rc 1 (4 case-table rows flip, scan not trusted); M9 the scan made to answer 0 files rc 1; M10 M4 under `NR == FNR` instead of `FILENAME == ARGV[1]` rc 0. M10 is the proof that the awk file test is load-bearing: with an emptied ledger `NR == FNR` reads every hit as a recorded entry and passes them all, so the guard uses FILENAME. ONE MORE HIT, OUTSIDE THE INTEGER SHAPE AND FIXED: aprender-train monitor/inference/tests/inference_monitor_tests.rs test_monotonic_ns_non_zero asserted `monotonic_ns() > 0` where monotonic_ns() is `START.get_or_init(Instant::now).elapsed().as_nanos()`. When that test is the first caller in the process, the call initializes the epoch and reads the gap between two back-to-back clock reads, which is 0 on a coarse timer. Its sleep ran BEFORE that first call, so it guarded nothing. The test now initializes, sleeps 1 us, and asserts the second read is greater than the first; the sleep, not host speed, carries the assertion. bashrs on the guard: 0 errors (the shell lint ratchet counts errors). The added warnings are the parser reading awk array syntax, `$''\t''` and `[![:space:]]` as shell. SCOPE AMENDMENT, the issue author (cop) on #3703, 2026-09-21T14:49:28Z, verbatim: "done_when 2''s sweep is satisfied by *listing* each remaining hit with its reason. The slow-host `elapsed < N` upper-bound class is the named remainder, tracked in **#3705**, and #3703''s ledger may list those hits with reason `#3705`. The PR for #3703 may therefore say `Closes #3703` + `Refs #3705`." THE LISTING: docs/audits/pmat-3703-timing-sweep.md, a snapshot at 8c08dd8dd with the commands that regenerate it (they print 58 + 45 = 103 lines). 58 upper bounds carry reason `#3705`. The 45 other lower bounds on a duration each carry a reason: 10 are sleep-guarded, 8 are tautologies, 8 are not clock readings, 17 are nanosecond or float readings over real work (0 only inside one clock tick; the closest is trueno-ublk adaptive_batch.rs:494, which times two add_items calls), 1 is a hidden upper bound routed to #3705 (tuner_tests_callbacks.rs:191, `remaining >= 999 s`), and 1 is `#[ignore]`d. It is listed rather than put in the PART 4 ledger because the ledger''s data lines are ENFORCED by text and count, and a coordinate list there would read as enforced while drifting. OUT OF THIS ROW, and stated rather than stretched: the SLOW-host twin, an upper bound on elapsed time (`elapsed < N`), with 59 candidate lines in the same sweep, is filed as #3705 (0.70). done_when 2''s "`Instant`-based assertions" reads that class in; this row covers the fast-host truncation class that failed the release, and a reviewer should judge #3705 as the named remainder.' diff --git a/docs/roadmaps/entries/PMAT-3708.yaml b/docs/roadmaps/entries/PMAT-3708.yaml new file mode 100644 index 0000000000..e2dcd0fb39 --- /dev/null +++ b/docs/roadmaps/entries/PMAT-3708.yaml @@ -0,0 +1,22 @@ +- id: PMAT-3708 + github_issue: 3708 + item_type: task + title: 'autopilot''s INHERITED dogfood GO writes no receipt that check_publish_preflight.sh R5 accepts — v0.69.0 stopped at the publish preflight 43 min after tagging' + status: in_progress + priority: critical + assigned_to: aprender-f0 + created: 2026-09-21T14:42:13Z + updated: 2026-09-21T15:45:00Z + spec: null + acceptance_criteria: + - 'Issue #3708 done_when 1, verbatim: "ONE rule, enforced at both ends. EITHER autopilot never inherits and runs the real dogfood at T-1 in `$AP/wt` (pre-tag, where a NO-GO costs no public tag), OR R5 accepts the inherited receipt under exactly the conditions autopilot checked (parent GO receipt present, bump diff = version surface only, parent is the release commit''s first parent), and both read the same file. Pick one; don''t keep both."' + - 'done_when 2 AS AMENDED on the issue by its author (comment 2026-09-21T15:37:51Z, "done_when 2 amended (issue author, cop, 2026-09-21), replacing the original, which assumed the inherit option. The option picked is: NEVER inherit."), verbatim: "a case table, wired as a scripts/check_*.sh so guard-tree runs it (the check_t2_preflight_verdict.sh pattern), that drives the real scripts/release/autopilot.sh dogfood step with stubs: a parent GO receipt plus a version-only bump diff still runs dogfood.sh (never inherits); a receipt R5 refuses (NO-GO, other commit, other version, absent) STOPs at T-1 before any tag; a GO receipt passes R5 at T-1, and `check_publish_preflight.sh` R5 at T-4 reads the same receipt file with the same function. Plus the close step: the epic as the milestone''s only open item is closed, then the milestone; any other open item refuses both. A mutant for each."' + - 'done_when 3, verbatim (measured at the NEXT train; this PR says Refs #3708, not Closes): "measured on the 0.70 train: no STOP between TAGGED and CASCADE for this reason"' + - 'The issue body''s close-step finding, verbatim: "Also found the same day (aprender-f0): the `close` step requires the milestone to have 0 open items, but nothing closes the release EPIC, so `close` dies on its own epic. 0.68.2''s was closed by hand. Same fix owner."' + - 'Obligation ADDED on the issue by its author (comment 2026-09-21T15:24:47Z, "Obligation added (issue author, cop, 2026-09-21)"), verbatim: "`prepare_bump.sh --ship` refuses to open the bump PR unless per-host model-ladder receipts for the NEW version are committed on the bump branch: `evidence/dogfood/models//{lambda,gx10}.json`, written by `scripts/model_ladder.sh` with an apr built from the bump tree. 0.68.2 did this by hand (#3498); 0.69.0 missed it, and that made `check_model_ladder` unmeasurable until after the tag. The script may REQUIRE the receipts without producing them; gx10''s is measured over the operator-authorized lambda→gx10 SSH. Case row: missing gx10.json refuses, naming the file; mutant."' + phases: [] + subtasks: [] + estimated_effort: null + labels: + - kind:code + notes: 'P0 on the 0.69.1 critical path (cop, aprender-04). Fixtures and stubs only: nothing ran against the live 0.69 train, its state dir, GitHub writes or a publish token. STACKED on PMAT-3699 (branch PMAT-3699-bump-pr-body-keep-open, 0cb1e4bd6), because both change scripts/release/prepare_bump.sh. The judged diff is base 0cb1e4bd6 to this head. SCOPE: scripts/release/autopilot.sh, scripts/check_publish_preflight.sh, scripts/release/prepare_bump.sh, scripts/check_release_bump_pr_body.sh (extended), the new scripts/check_release_autopilot_dogfood_close.sh, and this fragment. PICK (done_when 1 says pick one): autopilot NEVER inherits. The parent''s T-2 GO is at the OLD version, so rows keyed on the version (check_model_ladder) were never measured at the release version. And the T-4 recovery ran the same ~35 min dogfood anyway, after the tag was public. The inherit branch is DELETED, not flagged off, so there is no second rule to keep in sync. HOW EACH IS MET. (1) The dogfood step runs `scripts/dogfood.sh --phase pre-publish` in $AP/wt (the release worktree), then `check_publish_preflight.sh --receipt-only`. That is a new mode running R2 plus R5 only (R3 cannot hold before the tag). R5 is now ONE function, rule_r5(), which the full gate calls at T-4 and --receipt-only calls at T-1, both on the newest $AP/wt/.dogfood/receipt-*.json: the same file and the same function. A refusal is `STOP T-1 R5 refused …`, before the tag step. (2) The case table is scripts/check_release_autopilot_dogfood_close.sh. It is a scripts/check_*.sh because guard_tree.sh''s universe is the flat `git ls-files ''scripts/check_*.sh''`, so a table living only in scripts/release/ would run nowhere (the check_t2_preflight_verdict.sh pattern). It drives the REAL autopilot.sh through its own from/to steps (`dogfood dogfood`, `close close`). The fixture: a local bare origin whose main holds a parent (9.9.8) and a bump touching only Cargo.toml and CHANGELOG.md (9.9.9), a T-2 GO receipt for the parent (exactly the shape the old code inherited on), a real copy of check_publish_preflight.sh, and stubs for gh, cargo metadata and dogfood.sh. ROWS: never-inherits (dogfood.sh RUNS, no dogfood-inherited.receipt, "DOGFOOD GO at (R5 holds at T-1)"); t4-agrees-go (after tagging, the FULL gate prints "ok R5 dogfood receipt receipt-….json: GO for at 9.9.9" for the receipt T-1 accepted); nogo-stops; t1-r5-version (a receipt for 9.9.8 on a 9.9.9 cut STOPs at T-1, and the T-4 gate''s R5 refuses the same receipt, so the two ends agree on refusal too); t1-r5-commit (a receipt naming the parent STOPs at T-1); t1-r5-absent (the dogfood exits 0 and writes no receipt: STOPs at T-1); close-epic-last; close-other-open. MUTANTS, one per claim ("A mutant for each"), run on every bare run, each killed: inherit (the pre-#3708 step verbatim) -> "dogfood.sh never ran: the parent GO was inherited"; drop-nogo-stop -> "it stopped, but not on the NO-GO" (T-1 R5 caught it instead: defence in depth, but the NO-GO row still names the line); drop-t1-r5, run against each of t1-r5-version/-commit/-absent -> "autopilot passed T-1 with a receipt R5 refuses (version|commit|absent)"; t1-reads-another-file (--receipt-only pointed at a receipt dir the T-4 gate does not read) -> never-inherits "STOP T-1 R5 refused", so "the same file" is load-bearing; t1-skips-rule-r5 (--receipt-only no longer calls rule_r5) -> t1-r5-version passes a 9.9.8 receipt, so "the same function" is load-bearing; drop-epic-close -> "STOP milestone 9.9.9 still has 1 open item(s) after the epic"; drop-others-check -> "the epic was closed while #9010 is open". check_publish_preflight.sh --selftest (run by ci.yml:1978) gains 6 --receipt-only rows: an untagged tree with a GO receipt passes (the full gate refuses that tree on R3); NO-GO, stale commit, other version, absent, and a disallowed deferral each refuse with "FAIL R5". (3) is measured at the next train; it cannot be met by this PR, hence Refs. CLOSE STEP: it lists the milestone''s open items EXCEPT the epic. Any remaining one is a STOP naming them, closing neither. Otherwise it closes the epic if it is OPEN, so a re-run after a failure is idempotent, then requires open_issues == 0 and PATCHes the milestone closed. OBLIGATION: `prepare_bump.sh --ship` runs `bash scripts/check_model_ladder.sh --version "$V"` in the bump tree. That is the same judge the dogfood''s check_model_ladder row runs. Its FAIL lines go to stderr, and it refuses before the cargo checks, the commit, the push and `gh pr create`. It also refuses receipts that are present but gitignored (`git ls-files --others --ignored --exclude-standard -- evidence/dogfood/models/$V`), which `git add -A` would silently leave out. check_release_bump_pr_body.sh gives every fixture green receipts, a real copy of check_model_ladder.sh and a MINIMAL fixture ladder (the two required hosts, lambda and gx10, and one rung), so a rung added to the real ladder cannot break this table. New rows: ladder-missing-gx10 (refuses, naming "evidence/dogfood/models/9.9.9/gx10.json"; no push, no PR) and ladder-ignored. New mutants: drop-ladder (judge replaced by `true`) -> "exited 0 (ladder=no-gx10)"; drop-ignored -> "exited 0 (ladder=ignored)". MEASURED at this commit: `bash scripts/check_release_autopilot_dogfood_close.sh` rc 0, "PASS 17 row(s)" (8 behaviours, 9 mutants); `bash scripts/check_release_bump_pr_body.sh` rc 0, "PASS 12 row(s)"; `bash scripts/check_publish_preflight.sh --selftest` rc 0, "25/25 rows"; `bash scripts/check_model_ladder.sh --version 0.68.2` rc 0 on the real ladder and receipts (the judge the gate calls is live); bashrs lint on the 5 changed scripts: 0 errors each; check_release_scripts_derive_identity.sh PASS. GUARD_TREE (`bash scripts/guard_tree.sh --no-cargo`) at the first cut of this commit: "77 checks, 2 failed". check_guards_are_wired: the new table was UNWIRED, because a comment said "cargo metadata" and guard_tree''s textual `(^|[^a-z_-])cargo ` test moved it out of the --no-cargo set; the table never runs cargo (it stubs the metadata call), so the comment was reworded. check_no_pipe_into_grep_q: two `printf | grep -q` sites in the new table (76 vs ceiling 74), now here-strings. Both guards pass after the fix (grep-q "measured 74", guards_are_wired "PASS (ratcheted)"), and `guard_tree.sh --list --no-cargo` lists the table. Full re-run at 3a0513d35 (11-row table): "78 checks, 0 failed", with "PASS scripts/check_release_autopilot_dogfood_close.sh [run]". After the table grew to 17 rows: check_no_pipe_into_grep_q, check_guards_are_wired, and check_bashrs_gate ("PASS 363 file(s) linted … 0 SEC/DET/IDEM errors") re-run, all rc 0. PyYAML (needed by check_model_ladder.sh) is present in guard-tree: check_roadmap_ids_unique.sh and check_roadmap_completion_is_cited.sh already hard-fail without it, and they are green there.' diff --git a/docs/roadmaps/entries/PMAT-3711.yaml b/docs/roadmaps/entries/PMAT-3711.yaml new file mode 100644 index 0000000000..a822336f25 --- /dev/null +++ b/docs/roadmaps/entries/PMAT-3711.yaml @@ -0,0 +1,21 @@ +- id: PMAT-3711 + github_issue: 3711 + item_type: task + title: 'apr qa golden_output: a GPU golden-generation ERROR is reported as a SKIP (note_gpu_golden_skip) — ''GPU broken'' passes the gate; the Q4_K CUDA rule can''t be proven while it does' + status: in_progress + priority: critical + assigned_to: aprender-0e + created: 2026-09-21T15:55:00Z + updated: 2026-09-21T15:55:00Z + spec: null + acceptance_criteria: + - 'Issue #3711 done_when 1, verbatim: "a GPU golden generation error on a host that HAS a CUDA device and a `--features cuda` build is a **FAIL** naming the error, never a skip. SKIP stays only for \"no GPU on this host / no cuda feature\", and it names which."' + - 'done_when 2, verbatim: "case rows: GPU error gives FAIL; no-device gives SKIP-with-reason; GPU wrong answer gives FAIL; GPU right answer gives PASS. Plus a mutant (route the error back to skip and it goes RED)"' + - 'done_when 3, verbatim: "the Q4_K sweep and `scripts/model_ladder.sh` read the new verdict. Re-measure lambda + gx10 with the 0.69.1 binary, and no model is GREEN whose GPU leg errored"' + - 'Cop ruling (aprender-04 to aprender-0e, 2026-09-21), verbatim: "FOLD the sibling in: `run_golden_output_gate_runtime` saying \"GPU hybrid forward\" on a host with no device is the same gate labelling a run by intent. The pass message must name the backend that actually ran (a measured signal, e.g. the device the dispatch reported), plus a case row for cuda-feature-no-device. Same PR, same receipt."' + phases: [] + subtasks: [] + estimated_effort: null + labels: + - kind:code + notes: 'SCOPE: crates/apr-cli/src/commands/output_verification.rs, golden_output.rs, qa_gguf.rs (one line: golden_gate_for passes cpu_only), and this fragment. THE DEFECT IS WORSE THAN THE TITLE. On main, a CUDA init or GPU generation error in validate_gpu_golden_output went to note_gpu_golden_skip (a verbose-only println) and returned Ok(None); the case then judged only the CPU answer and the gate said PASSED, "3 golden test cases passed", not SKIP. Sibling (cop ruling, folded in): run_golden_output_gate_runtime labelled every cuda build "GPU hybrid forward, #3090" and ignored InferenceResult::used_gpu, and the qwen35 GPU dispatch falls back to the CPU on ANY GPU failure (forward_qwen35::run_qwen35_generate_dispatch), so a broken GPU passed on the CPU''s answer with a GPU label. HOW IT IS MET. GpuGoldenLeg {Passed, WrongAnswer, Errored, NotRun(why)}: judge() maps an Err (CUDA init on device 0, or GPU generation) to Errored, and failure() makes Errored a gate FAIL naming the error ("... a broken GPU, not a skip (#3711): "); WrongAnswer is the existing "GPU output failed (CPU passed)" FAIL. gpu_golden_not_run(cuda_feature, device, gguf) is the ONLY source of a skip and names which: "this build has no cuda feature" / "no CUDA device on this host" / "the dense GPU leg judges GGUF only"; the hybrid fail-safe arm is NotRun naming the runtime rung. The pass message names the leg: "(GPU leg judged on CUDA device 0)" or "(GPU leg SKIPPED: )". Runtime rung: golden_output_runtime returns (text, used_gpu); runtime_golden_backend(used_gpu, gpu_not_run) names the backend the dispatch REPORTED, and fails when the GPU should have served (cuda build + device + an architecture the GPU runs, i.e. not cpu_only) and the dispatch reported CPU. cuda_device_present() is one probe for both rungs. CASE ROWS (crates/apr-cli golden_output_tests, cargo test -p apr-cli --lib golden_output_tests: 14 passed, 10 new): GPU generation error -> FAIL naming CUDA_ERROR_ILLEGAL_ADDRESS; CUDA init error -> FAIL naming CUDA_ERROR_OUT_OF_MEMORY; GPU wrong answer -> FAIL; GPU right answer (plain and with a think block) -> PASS, judged; no device -> SKIP naming it, not a failure; each skip names which and a cuda host+GGUF starts the leg; runtime: cuda build + no device -> labelled CPU, never GPU; runtime GPU fallback on a cuda host -> FAIL; runtime GPU served -> labelled GPU from the dispatch; architecture the GPU declines -> CPU pass saying why. MUTANTS, each RED: the error routed back to skip (Err -> NotRun) fails 2 rows (generation error, init error); Errored mapped to no failure fails the same 2; the runtime fallback labelled by intent ("GPU hybrid forward, #3090") fails runtime_gpu_fallback_on_a_cuda_host_fails_the_gate; a no-device run labelled GPU fails 2 runtime rows. LIVE, RTX 4090 sm_89, apr 0.69.0 built from this branch with --features cuda, apr qa --json --offline with the perf/parity gates skipped (the ladder''s flags): qwen2.5-1.5b-instruct-q4_k_m rc 0 "3 golden test cases passed (GPU leg judged on CUDA device 0)"; the same with CUDA_VISIBLE_DEVICES= rc 0 "... (GPU leg SKIPPED: no CUDA device on this host)" [main said "3 golden test cases passed"]; Qwen3.5-0.8B-Q4_K_M rc 0 "... through the runtime entry point (served by GPU: the dispatch reported used_gpu=true)"; the same with no device rc 0 "served by CPU: the dispatch reported used_gpu=false, and the GPU was not expected (no CUDA device on this host)" [main, same run: "(GPU hybrid forward, #3090)" -- the label-by-intent, measured]; and ONE RUN under GPU memory contention from another session: Qwen3.5-0.8B rc 5 FAIL "the GPU should have served this model ... fell back ... A broken GPU, not a pass (#3711)" with stderr "GPU (CUDA) qwen35 path rejected, falling back to CPU: ... CUDA_ERROR_OUT_OF_MEMORY" -- exactly the run main would have passed as GPU. FOUND, NOT CAUSED HERE: qwen2.5-coder-0.5b-instruct-q4_k_m on the RTX 4090 fails the golden gate on BOTH main and this branch, identically: "GPU output failed (CPU passed): golden_output_gpu: gibberish (fragment \"```\\n\" repeats 3+ times)". Reported to the cop for the 0.69.1 Q4_K CUDA rule. DONE_WHEN 3: scripts/model_ladder.sh (qa_row gate()) and scripts/check_model_ladder.sh both require golden_output passed and not skipped, so an Errored GPU leg (passed:false) turns the rung RED with no ladder change; the red-golden case fixture already pins that path. The re-measure of lambda + gx10 with the 0.69.1 binary is the train''s T-2 step and is NOT done here -- this PR is Refs #3711, not Closes. clippy -p apr-cli --lib -D warnings (default features) rc 0; --features cuda: apr-cli has no clippy findings (the -D warnings run stops in aprender-compute on a pre-existing unused GemmOp import, same on main). cargo fmt --all -- --check rc 0.' diff --git a/docs/roadmaps/entries/PMAT-3712.yaml b/docs/roadmaps/entries/PMAT-3712.yaml new file mode 100644 index 0000000000..d69415e8b9 --- /dev/null +++ b/docs/roadmaps/entries/PMAT-3712.yaml @@ -0,0 +1,17 @@ +- id: PMAT-3712 + github_issue: 3712 + item_type: task + title: Model gate universe = each host's measured Q4_K inventory; the gate refuses an optional Q4_K rung (the contract's last one flips in the 0.69.1 fold, per the done_when 1 amendment); skip/fallback/rc!=0 RED; cells = verb x thinking x context rung + status: in_progress + priority: critical + assigned_to: null + created: '2026-09-21T00:00:00Z' + updated: '2026-09-21T00:00:00Z' + spec: null + acceptance_criteria: [] + phases: [] + subtasks: [] + estimated_effort: null + labels: + - kind:code + notes: 'ISSUE AMENDMENT (#3712 comment by the issue author, quoted verbatim): "done_when 1 amended (issue author, cop aprender-04, 2026-09-21 16:56Z): the guard that refuses `required: false` on a Q4_K rung (case row + mutant) is met by #3721. The contract''s own `qwen3-8b-q4km` rung flips to `required: true` in the 0.69.1 fold, together with aprender-fd''s qwen3-8b fix and a committed green lambda receipt. Flipping it earlier would turn `pv lint contracts/` red on every PR, because the armed ladder-green shape reads the committed 0.68.2 lambda receipt, in which qwen3-8b is red. This changes only the order of the work, not the bar: 0.69.1 does not tag until qwen3-8b is `required: true` and green on both lambda and gx10." ISSUE AMENDMENT 2 (#3712 comment 5764495191 by the issue author, quoted verbatim; lanes judge the cells scope against it): "done_when 5 added (issue author, cop aprender-04, 2026-09-21 17:12Z). Operator, verbatim, 2026-09-21: "lets ensure this release has MORE than enough tokens for our needs (overdo it), and our verbs are dogfooded, i.e chat with and without thinking, ditto run, ditto code, etc," - [ ] The cells matrix. A cell is (Q4_K model, required host, verb ∈ {run, chat, serve, code}, thinking mode derived from the model''s chat template, context rung). Rungs come from `evidence/release/context-rungs.json` up to the model''s declared `context_length`, which is read from the GGUF. The long rungs are owed by qwen35 (always the full set through the declared length) plus one representative per other arch, named in the ladder contract and echoed in the receipt (issue author''s ruling). - [ ] What each cell must do. Where the cell fits the host''s total GPU memory by the declared arithmetic, it must pass on CUDA. Where it doesn''t fit, it must be an honest pre-load refusal naming the arithmetic, never a CPU fallback (operator "a", #3710). - [ ] Coverage. Every rung must pass on at least one required host. The operator''s sentence is the requirement. The cell definition above is the issue author''s reading of it, so a lane judging scope judges against this comment. This is not unrequested scope." ACCEPTANCE (hand-entered from gh#3712; this row = done_when 1 + 2, the GATE side; cop assignment 2026-09-21: worker B). Issue done_when 1, verbatim: "Universe = measured inventory, not a list. The release gate derives the model set from what''s ON each required host (every `*Q4_K*` GGUF/APR under the declared models dir on lambda AND gx10), unioned with the ladder. A model present on a host but missing from the gate''s run is a FAIL naming it. The ladder has no `required: false` for any Q4_K rung; a guard refuses the key on a Q4_K rung (case row + mutant)." Issue done_when 2, verbatim: "Every (model, host) cell is GREEN on CUDA: capability_match passed (not skipped), golden_output passed (not skipped; #3711), `apr run --gpu` rc 0 with no fallback line. One red cell = NO-GO. There is no known, optional or pre-existing exemption, and no threshold like >= N%." WHAT THIS ROW DOES: (a) contracts/model-capability-ladder-v1.yaml gains `ladder.inventory` {dirs, patterns (case-insensitive *q4_k*/*q4k* .gguf/.apr), backends [cuda]}; (b) scripts/model_ladder.sh measures ladder UNION the host''s inventory (every match, depth 1), judges each inventory model on cuda with the SAME measure() as a rung, and writes receipt schema apr-model-ladder-receipt/v2 carrying `inventory` [{file, sha256, bytes}], `file` per row, and `apr_sha` (full 40-hex, for #3715); (c) scripts/check_model_ladder.sh FAILs: `required` not true on a Q4_K rung; a Q4_K rung that does not claim cuda; a ladder with no inventory; a receipt that is not v2 or has no inventory; an EMPTY inventory; an inventory file with no present row (MISSING from the run, named); an inventory-only model not green on cuda (skip, fallback, rc != 0 are RED); (d) case table grows 18 -> 26 (every case red for exactly its own reason, one FAIL line each except the two-host cases), plus 3 self-mutants each killed by its case (q4k-required-false, q4k-without-cuda, inventory-missing); --case with no such case is now RED, and the root is derived from the script path, not git rev-parse (#3581). WHAT IT DOES NOT DO (cop ruling 2026-09-21): qwen3-8b-q4km stays `required: false` in the contract, and the gate refuses that key at T-2, so the release cannot ship on it. Flipping it here would turn pv lint red on EVERY PR: the armed ladder-green shape reads the committed 0.68.2 lambda receipt (golden_output: Empty output), measured rc 0 -> 1 with only that flip. The flip folds into the 0.69.1 batch with the qwen3-8b fix and a committed green lambda receipt. So done_when 1''s `no required: false` clause is NOT met by this row: Refs, not Closes. Done_when 2''s cells turn green only through the model fixes under EPIC #3710; done_when 3 and 4 are aprender-f0''s (#3708). cells[] per verb x context (the widened bar, #3715 proposal) ships with the cop''s widened-bar delta, not here. WHERE THE GATE RUNS (so its rc=1 on the real contract is the release NO-GO, not a PR red): check_model_ladder.sh is declared ONLY in Cargo.toml [package.metadata.dogfood] gates (line 612, the T-2 pre-publish dogfood) and is listed in scripts/unwired_guards_baseline.txt (line 12): no PR workflow runs it, and guard_tree skips it (unwired-baseline). pv lint is a different tool that never runs this script; it reads the contract and PASSES with qwen3-8b at required: false. Measured on this head: guard_tree --no-cargo 0 failed, pv lint contracts/ PASS, cargo test -p aprender-contracts --lib 0 failed. THE GPU LOCK (cop ruling 2026-09-21, after a gx10 global OOM at 15:56:08Z killed CI containers): every apr call in model_ladder.sh goes through apr_locked = flock -E 75 -w ${MODEL_LADDER_LOCK_WAIT:-1800} /tmp/apr-gpu.lock choom -n 1000 --; a lock still held after the wait is an ENV decline (exit 2) naming the holder pid from /proc/locks, never a hang and never a model verdict. T-1 wraps the script in choom only (a second flock outside would deadlock). check_model_ladder.sh audits it statically in the REAL run (a raw "$APR" call is RED) and behaviourally in --self-test (fake apr via --lock-probe: lock held + oom 1000; a held lock declines naming the pid), with four producer mutants each killed: raw-apr-call, no-lock, no-choom, unbounded. FALSIFY-MCL-015. ROW B1 (the judge side of cells; stacked on row A; operator bar raise 2026-09-21: "lets ensure this release has MORE than enough tokens for our needs (overdo it), and our verbs are dogfooded, i.e chat with and without thinking, ditto run, ditto code, etc,"). A cell is (model, host, verb, thinking, context rung). contracts/model-capability-ladder-v1.yaml gains `ladder.cells` {verbs [run, chat, serve, code], long_rungs_for {families [qwen35], representatives: one file per other arch}}. The RUNGS are deliberately NOT in the contract: they live in evidence/release/context-rungs.json (apr-release-context-rungs/v1, landing in #3715), one declaration shared with pv; consumer-max = max over consumers[].max_prompt_tokens. The judge (scripts/lib/model_ladder_cells.py, called from check_model_ladder.sh) derives the owed set from the MEASURED inventory: short rungs for every model up to its context_length, long rungs for qwen35 and each arch''s representative; thinking modes from the template evidence (enable_thinking -> both; always-open -> on; else off, #3723), never the producer''s claim; fit := weights + kv_bytes_per_token * tokens + workspace <= gpu_mem_total_bytes (operator ''a'', #3710; an unmeasured term keeps the cell owed in full). A fitting cell must pass (cuda, no fallback, rc 0, prompt_tokens >= rung, answer_chars > 0, think_closed when on); a non-fitting cell must be `refused` with required_bytes > the host TOTAL (a co-tenant refusal is RED); every owed cell must pass on >= 1 required host; every arch in any inventory has a representative some required host holds. A missing context-rungs file, a consumer without a number, a missing context_length (#3733) and a receipt with no cells are each a named FAIL, never an empty owed set. Anti-shrink floors vs origin/main: the cells block (block, verbs, long-rung families and representatives) AND the rung file evidence/release/context-rungs.json (its rung ids and consumers; git show origin/main, absent there = bootstrap), because the rungs live in that file, not in the cells block. A `pass` on a cell the declared arithmetic says cannot fit is RED (MCL-INV-008: not fits => refused): the arithmetic decides what is owed, so a pass beyond it proves the arithmetic wrong (#3596). The module itself keeps only REQUIRED hosts'' receipts. Case table 26 -> 47; 10 module mutants each killed by its case. The PRODUCER side (row B2: measuring the cells) is not in this row: it needs #3723 (thinking toggle), #3733 (inspect keys) and #3596 (capacity::plan, the pre-load refusal). Until it lands, the real T-2 run is RED on ''receipt carries no cells'', which is the operator''s bar, not a PR red: check_model_ladder.sh runs only in the release dogfood (Cargo.toml [package.metadata.dogfood]) and is listed in scripts/unwired_guards_baseline.txt. MEASURED (answers a round-3 lane claim that qwen35-family GGUFs report general.architecture = qwen2): on lambda, Qwen3.5-0.8B-Q4_K_M.gguf and Qwen3.5-4B-Q4_K_M.gguf report general.architecture = qwen35 (apr inspect --json, and a direct GGUF v3 header read whose keys are qwen35.block_count, qwen35.context_length 262144, qwen35.full_attention_interval 4, ...), so `families: [qwen35]` matches them by arch. The fixture''s q.gguf is arch qwen35 and owes 16 cells in green-cells, every long rung included; a mutant deleting the families check (family-long) is killed by red-cells-missing-cell. 11 cells mutants.' diff --git a/docs/roadmaps/entries/PMAT-3715.yaml b/docs/roadmaps/entries/PMAT-3715.yaml new file mode 100644 index 0000000000..2ddf533ddf --- /dev/null +++ b/docs/roadmaps/entries/PMAT-3715.yaml @@ -0,0 +1,22 @@ +- id: PMAT-3715 + github_issue: 3715 + item_type: task + title: 'pv SHACL release-readiness shape: the tag is REFUSED unless every (Q4_K model in host inventory × {lambda,gx10} × {run,chat,serve,code} × consumer-context) cell has a Pass receipt — a missing cell is a violation, never a skip' + status: in_progress + priority: critical + assigned_to: aprender-97 + created: 2026-09-21T15:55:00Z + updated: 2026-09-21T15:55:00Z + spec: null + acceptance_criteria: + - 'Issue #3715 done_when 1, verbatim: "**the evidence graph**: `pv extract` ingests the release receipts (per-host model inventory, ladder and matrix receipts from #3712/#3708, dogfood receipt) as nodes: `:Model` (sha256, quant, arch), `:Host`, `:Verb` (run/chat/serve/code), `:ContextRung` (token counts DERIVED from the consumer-traffic records on #3710, never invented), `:Receipt` (verdict, backend, fallback, apr version plus sha)"' + - 'done_when 2, verbatim: "**the shape** `release-readiness-v1`: focus = every `:Model` in the inventory of every required `:Host`. For each (verb, context rung) exactly one `:Receipt` with `verdict=Pass`, `backend=cuda`, `fallback=false`, `apr_sha = release commit`. A missing, skipped, stale-sha or fallback receipt is a violation naming the cell."' + - 'done_when 3, verbatim: "**armed with a red-turning falsifier**: fixtures where each single cell is missing, skipped, falls back, or is stale, and each one turns the gate RED (the ONT-001 rule: a shape that has never gone red isn''t armed)"' + - 'done_when 4, verbatim: "**wired where the decision is made**: autopilot''s T-1 step and `check_publish_preflight.sh` both run `pv lint --gate shapes --shape release-readiness-v1` on the release commit''s evidence, and a violation STOPs before the tag and refuses at T-4"' + - 'done_when 5, verbatim: "**proof**: on the v0.69.0 evidence the shape reports the actual leaks (qwen3-8b lambda, qwen2.5-coder lambda, qwen3moe both hosts, chat/serve/code cells absent). On 0.69.1 it passes with N/N cells named."' + phases: [] + subtasks: [] + estimated_effort: null + labels: + - kind:code + notes: 'KNOWN REMAINDER (cop ruling, verbatim): release_evidence.rs `VERBS` is a hand-coded list; #3745 S2 replaces it with the derived cell set via `Inputs.cells`. 0.69.1 does not tag while it exists. QUORUM: round 3 AGREED at 84289cb7ba38adf96c0a1c5f836eb84ccb8d0646 (gemini-3.1-pro-high, gemini-3.1-pro-low, gpt-oss-120b-medium all PASS; docs/audits/quorum-PMAT-3715.json). Assigned by the cop (aprender-04) to aprender-97 on the 0.69.1 all-hands (EPIC #3710). The receipt FORMAT is shared and was agreed on #3712 before building: with aprender-62 (measuring side: issuecomment-5763332078, -5763540875, -5763596544, memory fields -5763795158) and aprender-f0 (T-1/T-4 wiring, issuecomment-5763344947); kernel-diff with aprender-37/aprender-eb (-5763412455, -5763633743, -5763653200); tokenizer parity on #3726 (-5763745060). SCOPE ADDED ON THE ISSUE by its author (cop), each built: kernel differential per GPU arch (#3715 issuecomment-5763396082); thinking {on,off} x context rungs {4k,8k,20k,60k,consumer-max,declared} (-5763490841, #3710 -5763489958); tokenizer parity per model (#3726 done_when 3); the memory-fit rule, operator decision "a" (#3710 -5763753907); three-state thinking_modes (#3723). WAIVERS: none, cop ruling (c) - a comment-based waiver is forgeable by every agent session (all post as noahgift); the operator-only surface is #3722. DONE HERE: done_when 1 (pv extract --release-* --out ingests inventory, cells, kernel/tokenizer/dogfood receipts as Release/Host/Model/Verb/ContextRung/Cell/CellReceipt/... nodes); done_when 2 (the nine-shape release-readiness-v1 family, armed by --shape); done_when 3 (34 red-turning CLI cases in crates/aprender-contracts-cli/tests/ont_release_readiness.rs, wired as ci/explicit-test-commands.d/450; two mutants caught: skip-as-pass turns 1 red, always-fresh turns 3 red); the pv side of done_when 4 (the invocation and exit contract f0 calls: 0 Pass, 1 Fail naming cells, 2 decline, 3 caller error); the first half of done_when 5 (evidence/release/proof-0.69.0: RED, 795 findings naming qwen3-8b lambda verdict=fail, qwen2.5-coder lambda and qwen3moe both hosts unmeasured/unkeyed, 752 of 768 cells absent, the NO-GO inherited dogfood). NOT DONE HERE (so the PR carries Refs, not Closes): the autopilot T-1 / check_publish_preflight T-4 call sites (aprender-f0, #3717/#3708), and "on 0.69.1 it passes with N/N cells named", which needs the producer (#3712 cells[]) and the model fixes to land. Also fixes a collision that existed independently: receipts.rs refused every schema but apr-model-ladder-receipt/v1 by name, so the first committed v2 receipt would have turned `pv lint --gate shapes` exit 3 on every PR.' diff --git a/docs/roadmaps/entries/PMAT-3717.yaml b/docs/roadmaps/entries/PMAT-3717.yaml new file mode 100644 index 0000000000..e13a1228ba --- /dev/null +++ b/docs/roadmaps/entries/PMAT-3717.yaml @@ -0,0 +1,24 @@ +- id: PMAT-3717 + github_issue: 3717 + item_type: task + title: '#3712 done_when 3 (child): the model-matrix gate runs at T-1 on BOTH hosts before any tag, receipts committed in the bump, re-read at T-4 by the same judge' + status: in_progress + priority: critical + assigned_to: aprender-f0 + created: 2026-09-21T15:46:37Z + updated: 2026-09-21T16:30:00Z + spec: null + acceptance_criteria: + - 'Issue #3717 done_when 3 (verbatim from #3712), verbatim: "**It runs BEFORE the tag**, as a T-1 step in `scripts/release/autopilot.sh` on BOTH hosts (gx10 over the operator-authorized lambda→gx10 SSH). A NO-GO stops the train with no tag cut. The per-host receipts are committed in the bump PR (#3708), and `check_publish_preflight.sh` re-reads the same receipts at T-4."' + - 'cop requirement 1, verbatim: "**red-turning tests**: a case table driving the real autopilot `models` step and the T-4 R7 with stubs. An unreachable host, a failed build, a missing receipt, a red receipt, or a judge decline (exit 2) each STOP before the tag (and FAIL at R7). A green pair passes. A mutant per row."' + - 'cop requirement 2 (instrument isolation; its LOCATION clause is replaced by the amendment below), verbatim: "**instrument isolation on gx10**: build in a PER-RELEASE target dir on gx10''s /mnt/nvme-raid0 (e.g. `targets/rel-`), delete the previous release''s dir first (the disk is at 90%), and NEVER `~/src/aprender/target`, a shared cache under an old checkout (the #3600 class: a stale instrument agrees with itself). Before measuring, prove `apr --version` = ` ()` on each host, and record it in the receipt."' + - 'cop requirement 3, verbatim: "the judge is `check_model_ladder.sh --version V --receipts DIR`, the same function at T-1 and T-4 (the frozen interface with #3712''s measuring side)"' + - 'cop requirement 4 (measured on the 0.69.1 train; this PR says Refs #3717), verbatim: "proof on the 0.69.1 train: the autopilot STATUS shows `MODELS GO` on both hosts before `TAGGED`"' + - 'AMENDMENT on the issue (comment 2026-09-21T15:50:22Z), verbatim: "**Instrument-isolation location amended (cop, 2026-09-21), replacing the location clause:** build in a PER-RELEASE target dir on gx10, derived at run time and never a literal in scripts/release/: `$HOME/.cache/aprender-release/rel-` on the remote; delete every other `rel-*` there first; NEVER `~/src/aprender/target` or any dir under it. The rest of the item stands (fresh target, previous dirs deleted, `apr --version` = " ()" proved on each host, and the receipt''s apr_version checked against it). Why: on gx10 /mnt/nvme-raid0 is the same filesystem as / (measured), and check_release_scripts_derive_identity.sh R1 refuses absolute /mnt paths in scripts/release/. Added: refuse up front with a named ENV reason when the remote free space is below the measured size of a fresh cuda release target."' + - 'GPU-SHARING REQUIREMENT added on the issue (comment 2026-09-21T16:09:54Z, the cop''s ruling relayed verbatim), verbatim: "Ruling, as you recommend: **one owner**. The LOCK lives in model_ladder.sh, per apr call (62''s row B), which protects every caller. Your T-1 wrapper runs the ladder under `choom -n 1000 --` ONLY (oom_score_adj is inherited across fork) and takes NO flock. Builds stay outside any lock. Your table asserts "runs under choom -n 1000 and the wrapper takes no flock"; 62''s asserts "every apr call runs under the lock"."' + phases: [] + subtasks: [] + estimated_effort: null + labels: + - kind:code + notes: 'P0 on the 0.69.1 release path (cop, aprender-04). This is #3712''s done_when 3, split out so that aprender-62 (done_when 1+2, the measuring side: scripts/model_ladder.sh, scripts/check_model_ladder.sh, the ladder contract) and aprender-f0 do not mint the same fragment. Fixtures and stubs only in CI; nothing here ran against the live train or a publish token. STACKED on PMAT-3708 (branch PMAT-3708-dogfood-one-rule, 629143eeb): the judged diff is base 629143eeb to this head. SCOPE: the new scripts/release/models_t1.sh; scripts/release/autopilot.sh (a new `models` step); scripts/check_publish_preflight.sh (a new R7); the new scripts/check_release_models_t1.sh; and this fragment. "The per-host receipts are committed in the bump PR (#3708)" is already delivered by #3708: prepare_bump.sh --ship refuses unless check_model_ladder.sh --version is green in the bump tree, and refuses gitignored receipts. HOW IT IS MET. (1) BEFORE THE TAG, ON BOTH HOSTS: autopilot STEPS gains `models` between `dogfood` and `tag`. It runs scripts/release/models_t1.sh $AP/models-t1 and STOPs with "T-1 model matrix NO-GO … nothing is tagged" on any non-zero exit. On GO it says "MODELS GO at on lambda and gx10", which is the STATUS line cop requirement 4 asks for before TAGGED. models_t1.sh runs two legs IN PARALLEL: lambda locally in the release worktree, and gx10 over `ssh -o BatchMode=yes gx10 "bash -s"`, the lambda->gx10 SSH the operator authorized. Each leg builds `cargo build --release -p apr-cli --bin apr --features cuda --locked` (binary-release.yml''s command) from the release commit. It then PROVES the binary before measuring (`apr --version` must equal "apr ()"), runs `scripts/model_ladder.sh --host --out ` (the frozen interface), and brings gx10''s receipt back. Each receipt''s existing apr_version field must equal that same line. Then `scripts/check_model_ladder.sh --version --receipts $AP/models-t1` judges both. An unreachable host (ssh rc 255), a failed build, a binary that is not the release, a missing receipt, a red cell and a judge DECLINE (rc 2) are each NO-GO. Nothing is retried, and there is no exemption. (2) INSTRUMENT ISOLATION ON gx10, per the AMENDMENT: a PER-RELEASE dir `$HOME/.cache/aprender-release/rel-`, derived on the remote at run time. It holds a detached worktree at the release commit, and CARGO_TARGET_DIR is /target. Every other rel-* is deleted first, and ~/src/aprender/target is never used (only `git -C ~/src/aprender fetch origin main` and `worktree add` touch that checkout). No host path is a literal in scripts/release/ (check_release_scripts_derive_identity.sh PASS). The worktree is removed after measuring; the target stays for a same-version re-run and is deleted by the next release. DISK: the leg REFUSES up front, as ENV ("MODELS-LEG gx10 ENV: … refused before building", models_t1.sh exit 2), when the filesystem''s free space plus an existing same-version dir is below NEED_KIB = 5,524,629 KiB. MEASURED 2026-09-21 on lambda at 629143eeb with a fresh CARGO_TARGET_DIR: the target is 5,285,126,283 bytes (244 s, nice 19), the binary printed "apr 0.69.0 (629143eeb)", and a worktree checkout is 363,372 KiB. gx10 is aarch64 and was not measured. A read-only probe of gx10 (bash 5.2.21, git 2.43.0): df -Pk reports 116,078,068 KiB free, so it "would-build". (3) T-4 RE-READ, THE SAME JUDGE: check_publish_preflight.sh gains R7, rule_r7(), called by the full gate. It runs `check_model_ladder.sh --version ` in the tagged tree, over the receipts COMMITTED by the bump. rc 0 -> "ok R7"; rc 2 -> "FAIL R7 the model-matrix judge DECLINED (rc 2), and a decline is not a pass"; any other -> "FAIL R7 model matrix NOT green", quoting the judge''s FAIL lines; a missing judge -> "FAIL R7 no model-matrix judge". The seam is PUBLISH_PREFLIGHT_LADDER_JUDGE. The PASS line now ends "dogfood GO, model matrix green". The self-test gains r7_model_matrix_red_refuses, r7_judge_decline_refuses and r7_judge_absent_refuses. Every fixture commits a judge stub that refuses any version but 1.2.3, so all_rules_hold also proves the argument: 28/28. #3715 (the pv SHACL shape, aprender-97) will become the gate of record behind the same call sites. That design is agreed on #3712 (comments 5763316076 and 5763344947): T-1 passes MC, and T-4 passes the measured sha after a `git diff --quiet HEAD -- . '':!evidence''` proof. It is NOT in this diff. RED-TURNING TABLE (cop requirement 1): scripts/check_release_models_t1.sh, a scripts/check_*.sh with no bare cargo token, so guard_tree''s --no-cargo set runs it. It drives the REAL autopilot `models` step, and so the real models_t1.sh, through `autopilot.sh 9.9.9 99 models models`. The fixture: a local bare origin (parent 9.9.8, release 9.9.9); an ssh stub that runs the gx10 script locally under a fake gx10 HOME whose ~/src/aprender sits at the OLD parent with a stale rel-9.9.7 beside it; a build stub that bakes `apr --version` from the tree it ran in; and committed stubs for model_ladder.sh and the judge. T-1 ROWS: green-pair (MODELS GO; gx10 measured by "apr 9.9.9 ()" although its checkout sat at the parent; built into rel-9.9.9, not ~/src/aprender/target; rel-9.9.7 deleted); gx10-unreachable; build-fails; missing-receipt; red-cell; judge-decline; stale-binary; disk-refusal (exit 2, nothing built). T-4 ROWS: rule_r7 EXTRACTED from check_publish_preflight.sh and run on a tagged tree''s committed receipts (cargo-free): r7-green, r7-red, r7-missing (names gx10.json), r7-decline, r7-no-judge. MUTANTS, one per row, each killed: shared-target -> "gx10 did not build into the per-release dir"; no-255-reason, build-ignored, no-receipt-ok -> the STOP no longer names the cause; red-is-go and decline-is-go -> "autopilot exited 0"; no-version-proof -> the pre-measure NOT-THE-RELEASE is gone (the receipt''s apr_version check still stops it, which is defence in depth); no-disk-check -> "autopilot exited 0"; autopilot-no-die -> "autopilot exited 0"; r7-no-version -> "decline: no --version given"; r7-red-is-go and r7-decline-is-go -> "rc=0, want 1"; r7-no-fail-lines -> the refusal no longer names gx10.json; r7-no-judge-ok -> "rc 127", not the no-judge refusal. That the full gate CALLS rule_r7 is proved by the preflight self-test, which needs a Rust-package fixture and runs at ci.yml:1978. Ad hoc wiring mutant (the gate without its rule_r7 call): "BROKE r7_model_matrix_red_refuses / r7_judge_decline_refuses / r7_judge_absent_refuses", 25/28. The table''s first full run caught a fixture bug of mine: the red receipt was written before its env var was exported, so r7-red was RED and its mutant was "killed" for the wrong reason. It was fixed by passing the red host positionally, and the mutant now dies with "rc=0, want 1". ROUND 2 CHANGE (after the round-1 receipt at d00665a7f), the GPU-SHARING requirement: both legs run `choom -n 1000 -- bash scripts/model_ladder.sh …`. oom_score_adj is inherited across fork, so the ladder and every apr it starts are the OOM victim, never the CI pool (the gx10 15:56Z OOM killed 18 CI containers). The wrapper takes NO flock: the lock is model_ladder.sh''s own, per apr call (#3712 row B), and a second flock on /tmp/apr-gpu.lock here would deadlock it. That is the one-owner ruling. Builds run under neither. New row oom-victim: stubs for choom and flock record what wrapped each process, and the row requires "ladder host= oom=1000 flock=0" on both hosts, "build host= oom=none flock=0", and no flock call at all. New mutants: no-choom (gx10''s ladder without choom) -> "gx10 ladder did not run under choom -n 1000 alone: ladder host=gx10 oom=none flock=0"; wrapper-flock (lambda''s ladder under `flock /tmp/apr-gpu.lock choom …`) -> "… ladder host=lambda oom=1000 flock=1". MEASURED at this commit: `bash scripts/check_release_models_t1.sh` rc 0, "PASS 30 row(s)"; `bash scripts/check_publish_preflight.sh --selftest` "28/28 rows"; `bash scripts/check_release_autopilot_dogfood_close.sh` "PASS 17 row(s)" and `bash scripts/check_release_bump_pr_body.sh` "PASS 12 row(s)", both unchanged by the new step; check_bashrs_gate "PASS 365 file(s) … 0 SEC/DET/IDEM errors", after a SEC010 on a `cd /` in the gx10 heredoc was removed; check_guards_are_wired, check_no_pipe_into_grep_q and check_release_scripts_derive_identity all PASS. `bash scripts/guard_tree.sh --no-cargo` at 38f8b5945: "79 checks, 0 failed", with "PASS scripts/check_release_models_t1.sh [run]". Cop requirement 4 (the real 0.69.1 train, MODELS GO on both hosts before TAGGED) is measured on that train, hence Refs #3717.' diff --git a/docs/roadmaps/entries/PMAT-3718.yaml b/docs/roadmaps/entries/PMAT-3718.yaml new file mode 100644 index 0000000000..27b80c1fd5 --- /dev/null +++ b/docs/roadmaps/entries/PMAT-3718.yaml @@ -0,0 +1,20 @@ +- id: PMAT-3718 + github_issue: 3718 + item_type: task + title: 'apr run --json: prompt_tokens + completion_tokens + finish_reason; serve usage and finish_reason never report a cut as a stop (#3718, RAH #3716)' + status: in_progress + priority: high + assigned_to: aprender-a8 + created: 2026-09-21T16:20:00Z + updated: 2026-09-21T16:20:00Z + spec: null + acceptance_criteria: + - 'Issue #3718 done_when 1, verbatim: "`apr run --json` emits `prompt_tokens` (the post-template token count actually fed to the model) and `completion_tokens`, and the numbers match the tokenizer (a case row checks a fixed prompt''s count)"' + - 'done_when 2, verbatim: "the same fields are in `apr serve`''s `/v1/chat/completions` `usage` object (OpenAI shape)"' + - 'done_when 3, verbatim: "a prompt truncated or refused for length says so in the JSON (`truncated: true` or an error), never a silent cut"' + phases: [] + subtasks: [] + estimated_effort: null + labels: + - kind:code + notes: 'SCOPE. The three done_when rows quoted above. `prefill_ms` is deliberately NOT emitted by this row. The cop''s lane message asked for it, but it is not in #3718''s done_when, and it is owned elsewhere: aprender-a8 and aprender-3e agreed one definition (wall-clock over the forward passes on the post-template prompt, up to the first token''s logits; load, h2d, the F2 validate and tokenization excluded; null, never 0, where a path does not split prefill). It rides #3606''s StageTimings and 3e''s #3596 timer (`run_qwen35_generate_dispatch_timed`, not on main). Wiring it into this JSON is a follow-up once those land. RUN (done_when 1, 3). New `realizar::infer::run_inference_report(config) -> (InferenceResult, RunReport)`; `run_inference` is a thin wrapper over it. It is a separate entry point so that none of the ~75 `InferenceResult` literals change. `RunReport { finish_reason, context_length }` is None on a path that does not report it (APR, SafeTensors), never a default. The GGUF path computes it where the stop set and the budget are known. `FinishReason::from_decode(generated, stop, budget)`: a last token in the stop set is Stop; otherwise a run that filled its budget is Length; a shorter run ended on a stop token its loop consumed. This rule holds for both loop shapes, traced: the qwen35 CPU/GPU, MoE and wgpu loops PUSH the stop token, the dense CPU (`generate_with_cache`) and CUDA (`decode_blocking`) loops break BEFORE pushing, and every loop''s only other non-error exit is its budget. `decode_budget` clamps to `context_length - prompt` only on the dense CPU loop, the one loop that does (`effective_max_tokens`). `FinishReason` is declared once, in `infer::run_report` (not behind the `server` feature); `api::types::FinishReason` is now a re-export. apr-cli: `RunUsage { prompt_tokens, completion_tokens, finish_reason, context_length }` on `RunResult`, emitted by `build_final_json`, so it appears in both `--json` and the `--stream` final event. `prompt_tokens` = `input_token_count`, the post-chat-template count including BOS that `prepare_tokens` fed the model. `completion_tokens` is the engine count, never the word-count stand-in `tokens_generated` falls back to. No path truncates a prompt (traced: `prepare_tokens` has no truncation). An over-long prompt is REFUSED (ContextLimitExceeded -> error exit), which is done_when 3''s "or an error"; the completion-side cut is now `finish_reason: length`, where before it was silent. SERVE (done_when 2, 3). `/v1/chat/completions` `usage` was already OpenAI-shaped with the post-template count on the realizar backends (`tokenize_chat_prompt`: template, encode, `len`). Three silent cuts are fixed. (a) The dense CPU backend (`try_quantized_backend`) judged finish against the REQUESTED max_tokens after `effective_max_tokens` had clamped the loop to the context room, so a context-clamped cut said "stop"; it is now judged against the budget the loop ran with, both streaming and not. (b) A prompt that cannot fit was a 500 (non-stream) or a 200 SSE carrying an error event that still ended "stop". It is now a 400 before generation on both, via the existing `generation_error_status`, whose doc already said callers map it to 400. (c) The APR CPU fallback (apr-cli handler_apr_cpu_completion.rs) hardcoded "stop" while capping max_tokens at 4096. It now uses `from_decode` against that cap with the loop''s real stop set (token 0, pushed; `is_eos_token`). Its `run_apr_cpu_inference` also returned the PROMPT as the completion when nothing was generated (`else { &output_tokens[..] }`); that now yields empty. MEASURED on lambda, CPU, Qwen3.5-0.8B-Q4_K_M, pinned `apr 0.69.0 (3fadee114)` via scripts/apr_bin.sh. `apr run -i <"What is 2+2? Answer with one number."> --chat --json`: prompt_tokens 24, context_length 262144. The same templated string through llama.cpp 41fc758 `llama-tokenize --ids` gives 24 ids, identical id-for-id to apr''s `--verbose` encode (with and without llama.cpp''s BOS policy). That is done_when 1''s case row against an independent tokenizer. max-tokens 4 and 1024: completion 2, finish stop (the model ends on EOS). A 20-line-poem prompt at max-tokens 6: completion 6, finish length. DONE_WHEN 1 CASE ROW, automated (added after the round-2 quorum FAIL, which correctly said a manual measurement is not a case row): infer::run_report::a_fixed_text_prompt_counts_the_tokens_the_tokenizer_feeds builds the pygmy GGUF with a SentencePiece vocabulary (` ▁Hello ▁world !` + filler; bos 1, eos 2) through the new `test_factory::build_executable_pygmy_gguf_with` metadata hook, runs the TEXT prompt "Hello world!" through `run_inference_report` (the `apr run --prompt` path: prepare_tokens_gguf, the GGUF tokenizer, BOS policy) and asserts the fed ids are [1, 3, 4, 5] and input_token_count (which becomes prompt_tokens) is 4. The expectation is derived by hand from the vocabulary, not from apr''s encoder. Mutant: prepare_tokens_gguf reports tokens.len()-1 -> the row goes RED. The post-template count on a real model is the llama.cpp row above (24 == 24, id for id). TESTS. infer::run_report: a case table over both loop shapes, the clamped-cut pin, budget arithmetic, and four rows through `run_inference_report` on the pygmy GGUF (context 32): a context-clamped cut is Length with 28 = 32 - 4 tokens, a max_tokens cut is Length, a stop token under budget is Stop (0 tokens, dense loop consumed it), and an over-long prompt is refused naming both lengths. api::tests::usage_finish_3718 runs through the real router on a quantized AppState: clamped cut -> length with usage prompt+completion == context, clamped stream -> terminal chunk length, prompt > context -> 400 on stream and non-stream, stop under budget -> stop (both), unclamped -> cut at the request budget. apr-cli run_tests_usage_3718: JSON carries the four fields, null (not 0 or "stop") when unreported, stream final event carries them. MUTANTS, each run and each RED: serve clamp reverted (2 red), 400 reverted to the old path (1), run-report clamp off (1), from_generation always Length (4), never Length (7). SUITES at 3fadee114 + rustfmt: `cargo test -p aprender-serve --lib` 15913 passed; `cargo test -p apr-cli --lib` 7301 passed; `cargo test -p aprender-contracts --lib` 1684 passed; clippy -D warnings on aprender-serve and apr-cli clean; fmt clean. KNOWN, NOT CHANGED. completion_tokens counts the pushed stop token on the qwen35/MoE/wgpu loops and not on the dense loops, because that is what each loop returns (measured: qwen35 `tokens: [17, 248046]`, completion 2). The CUDA backends were not exercised on hardware in this row: the report logic is backend-agnostic and clamps only on the dense CPU loop. Streaming chat does not apply stop STRINGS on the true-streaming path; that is pre-existing and not a length cut. Found while measuring and routed separately: `--prompt P --chat` double-templates (24 -> 56 prompt_tokens), which is #3743 = #3672 in batch-1.' diff --git a/docs/roadmaps/entries/PMAT-3726.yaml b/docs/roadmaps/entries/PMAT-3726.yaml new file mode 100644 index 0000000000..2f7b38990f --- /dev/null +++ b/docs/roadmaps/entries/PMAT-3726.yaml @@ -0,0 +1,21 @@ +- id: PMAT-3726 + github_issue: 3726 + item_type: task + title: 'apr''s byte-level BPE tokenizer is non-canonical (greedy longest-match, no pre-tokenizer, no merge ranks): every GGUF prompt reaches the model as ids it never saw; corrupts copy tasks (#3693) and every non-ASCII user' + status: in_progress + priority: critical + assigned_to: aprender-c3 + created: 2026-09-21T16:20:00Z + updated: 2026-09-21T17:25:00Z + spec: null + acceptance_criteria: + - 'Issue #3726 done_when 1, verbatim: "apr''s token ids are **identical to llama.cpp''s** (the pinned comparator, `llama-tokenize` / `--verbose-prompt`) for every Q4_K model family in the inventory (qwen35, qwen3, qwen2 / qwen2.5, qwen3moe) on a corpus of real consumer briefs (#3716), source code, and non-ASCII sets (CJK, accented Latin, emoji, box-drawing, zero-width chars), and the round-trip decode(encode(x)) == x"' + - 'done_when 2, verbatim: "the byte-level BPE byte→unicode map (GPT-2 bytes_to_unicode) and the pre-tokenizer regex match the model''s `tokenizer.ggml.pre` / HF definition. Root cause named on the issue."' + - 'done_when 3, verbatim: "a **tokenizer-parity gate** (a scripts/check_*.sh, wired) that fails on any id mismatch against the pinned llama.cpp, plus a mutant (reintroduce the id-0 byte fallback and it goes RED). It becomes a required cell per model in the release shape (#3715)."' + - 'done_when 4, verbatim: "re-measure #3693 after the fix: if the corruption disappears, #3693 closes as this; if not, the remainder stays on #3693"' + phases: [] + subtasks: [] + estimated_effort: null + labels: + - kind:code + notes: 'Filed by the cop (aprender-04) from infra-c8''s measurement; owner aprender-c3, who reached the same root cause from the forward side (#3693). ROOT CAUSE: `GGUFModel::encode` (crates/aprender-serve/src/gguf/token.rs) was documented and implemented as greedy longest-match over each whole segment, with no pre-tokenizer and no merge ranks. `prepare_tokens_gguf` (the `apr run` path), `apr parity` and serve''s batch path all call it. Measured on apr 0.69.0 (225b2a9ab) against the pinned llama-tokenize (d1d3c3396) for "The quorum forbids it; cited refutes!!!": apr gave `Ġquo|rum Ġforbid|s … Ġrefute|s`, llama.cpp `Ġqu|orum Ġforb|ids … Ġref|utes`. On arbiter''s r891 prefix apr gave 253 ids and llama.cpp 248, with 16 differing spans. The lane''s "quoorum" is `Ġquo` (apr''s odd split of the prompt) joined to the canonical continuation `orum`. FIX (this row): new crates/aprender-serve/src/gguf/byte_level_bpe.rs, a port of llama.cpp''s reference: `unicode_regex_split_custom_qwen2`/`_qwen35` line for line (selected by tokenizer.ggml.pre; the fleet inventory has exactly these two: qwen2 covers qwen2/qwen2.5/qwen3/qwen3moe, qwen35 covers qwen35/qwen35moe), GPT-2 bytes_to_unicode, ranked merges from tokenizer.ggml.merges with a priority queue (lowest rank, then leftmost, as llm_tokenizer_bpe_session), special tokens by token_type CONTROL/USER_DEFINED/UNKNOWN longest first (tokenizer_st_partition). `GGUFModel::encode` routes every byte-level vocabulary through it. A byte-level file with an unimplemented pre-tokenizer keeps the greedy path and says so once on stderr, naming #3726. The old `<|...|>` special-token rule also missed USER_DEFINED ``/``. The gate follows the #3076 shape: scripts/check_tokenizer_parity.sh (PR-time case table, run by guard_tree) and scripts/tokenizer_parity.sh (model-backed, apr vs the pinned llama-tokenize, declared in [package.metadata.dogfood] for the pre-publish phase), reading ids from a new `apr tokenize encode MODEL (-p TEXT | -f FILE)`. That command reads only the GGUF metadata, calls the production GGUFModel::encode, and prints the path taken and decode(encode(x)) == x. Corpus: evidence/tokenizer-parity/corpus/ (public aprender source, a review brief over a public aprender commit, non-ASCII / whitespace / identifier / chat-template sets). MEASURED (receipts in evidence/tokenizer-parity/): lambda, release mode at 9026b70a5, 147/147 (21 GGUFs x the 7 committed files of that time, pinned llama-tokenize d1d3c3396 resolved by the default path); lambda at 9026b70a5 with the 4 real consumer briefs, 210/210 (21 x 10: the 6 committed files after the corpus was trimmed to keep review briefs small, a 25 KB shell script dropped and the review brief cut to 9 KB, plus the 4 briefs); gx10 (aarch64) at 9026b70a5 with the briefs and the 7-file corpus, 132/132 (12 x 11; the pinned build prints d1d3c339 and tp_commit_matches accepts a 7+ hex prefix). Every pair: identical ids, canonical path, decode(encode(x)) == x. MUTANT (done_when 3): bytes >= 0x80 -> id 0 reintroduced; the gate goes RED, 14 of 22 pairs FAIL (the 8 that pass are the pure-ASCII files, as that mutation predicts), first mismatch at the first non-ASCII byte (mutant-id0-lambda.txt). GUARDS: guard_tree --no-cargo 82/82 at 9026b70a5 after three fixes it found: complexity (the splitter is a Splitter with one method per alternative; GGUFModel::encode fell from cognitive 60 to 41), shell lint (SC2075, SEC011), dogfood coverage (surface_audit.csv rows re-mapped through a line diff, one new row, denominator 837 -> 838 in contracts/apr-dogfood-coverage-v1.yaml and the apr-dogfood skill). done_when 4 (#3693 re-measure): an A/B of the fixed binary (9026b70a5) against the control (262dfb2f6, this branch''s parent, which already carries #3672''s template-once fix) on the two lane prompts at temperature 0 on CUDA, queued under the GPU lock on both hosts; recorded on #3693 and #3726 when it lands. OUT OF SCOPE, filed: aprender-core''s BpeTokenizer and realizar''s apr/tokenizer (the APR/HF paths) are #3742; SentencePiece GGUFs keep the greedy path and the gate reports them UNCOVERED, never PASS.' diff --git a/docs/roadmaps/entries/PMAT-3733.yaml b/docs/roadmaps/entries/PMAT-3733.yaml new file mode 100644 index 0000000000..6e6b1ec353 --- /dev/null +++ b/docs/roadmaps/entries/PMAT-3733.yaml @@ -0,0 +1,19 @@ +- id: PMAT-3733 + github_issue: 3733 + item_type: task + title: 'apr inspect --json drops every qwen35.* metadata key: the GGUF reader parsed only an arch-prefix allowlist; every header key now reaches inspect (#3733)' + status: in_progress + priority: critical + assigned_to: aprender-a8 + created: 2026-09-21T17:05:00Z + updated: 2026-09-21T17:05:00Z + spec: null + acceptance_criteria: + - 'Issue #3733 done_when, verbatim: "`apr inspect --json ` emits every KV the header carries (or at least every `.*` key) for any arch, including one inspect has never heard of. A test runs a fixture GGUF with an unknown arch prefix and fails if any header key is missing from `.metadata`."' + - 'Cop comment on #3733 (2026-09-21T16:25:54Z), verbatim: "done_when: `apr inspect --json` emits EVERY GGUF metadata key the file carries, with no arch-prefixed allowlist (the 46 KVs on the 4B). A case row checks the count against the header''s KV count, plus a mutant (drop one prefix and it goes RED). Also trace the RosettaStone path, since 62 didn''t."' + phases: [] + subtasks: [] + estimated_effort: null + labels: + - kind:code + notes: 'ROOT CAUSE (traced, the RosettaStone path included): `apr inspect` sends every GGUF to `run_rosetta_inspect` (inspect.rs:162), then `RosettaStone::inspect_gguf` (rosetta/validate_inspect.rs:419) copies `load_gguf_raw(..).raw_metadata` unfiltered, and `load_gguf_raw` (gguf/api.rs) built that map from `GgufReader::metadata`. `GgufReader::from_bytes` parsed a key only if it started with `tokenizer.`, `general.`, `llama.`, `qwen2.`, `qwen3.`, `phi.`, `mistral.` or `gpt2.` (reader_parsing.rs) and length-skipped everything else, so `qwen35.*` (which does not start with `qwen3.`) and `quantize.*` never existed after the parse. No filter was in inspect or rosetta. FIX: nothing is skipped any more. Keys outside the allowlist are decoded into a new `GgufReader::display_only_metadata`, and `raw_metadata` is `metadata` chained with it. DELIBERATELY NOT `keep_all` on this path: the config accessors (`context_length()`, `num_layers()`, `hidden_size()`...) read `metadata`, and `load_gguf_raw` feeds them into the `apr import` model config, so widening `metadata` would give every unlisted architecture a config the import path has never been given. That is a behaviour change outside this row. A test pins it: `reader.context_length()` stays None on the qwen35 fixture while `display_only_metadata` holds the key. `from_file_full` (the shard merge) is unchanged and keeps everything in `metadata`. `skip_metadata_value` lost its only caller and is deleted; its ten builder tests now also assert that the key lands in `display_only_metadata`. HARDENING, which parsing instead of skipping makes necessary: `read_metadata_array` took its element count from the file, called `Vec::with_capacity(count)` and indexed `data[..]` directly for i32/f32 arrays. The span is now checked against the bytes left before any allocation, and the checked `read_i32_le`/`read_f32_le` are used. A hostile count returns FormatError "overruns the file" instead of panicking. The case row covers element types 4, 5, 6 and 8. CLI: `output_rosetta_json` is split into `rosetta_json(path, report) -> Value` plus the print, so the --json document is testable. It emits `report.metadata` unfiltered. MEASURED on lambda. Pre-fix instrument: `apr 0.68.2 (97093d2a5)`. It is a valid baseline because `git log v0.68.2..52f43da71` touches none of reader_parsing.rs, api.rs, validate_inspect.rs or inspect.rs. Qwen3.5-4B-Q4_K_M: .metadata|length = 24, qwen35.* keys = 0, qwen35.context_length = null, which reproduces #3733. Post-fix instrument: `apr 0.69.0 (336b7230b)`, resolved and HEAD-checked by `scripts/apr_bin.sh` under CARGO_TARGET_DIR=/mnt/nvme-raid0/targets/a8-3718. The header KV count comes independently from python `gguf` (GGUFReader.fields minus GGUF.*): 0.8B 46/46, 2B 46/46, 4B 46/46, 9B 46/46, 27B 49/49 (shown/header), each with all 18 qwen35.* keys, context_length 262144, full_attention_interval 4, block_count 24/24/32/32/64, head_count_kv 2/2/4/4/4, key_length 256. Control on an arch that was already on the allowlist: Qwen3-1.7B 32/32, context_length 40960, unchanged. TESTS: api_tests_all_keys_3733.rs covers a fixture whose header carries qwen35.*, quantize.*, an unheard-of prefix `zzz_never_heard_of.*`, general.* and tokenizer.*. It asserts load_gguf_raw and RosettaStone::inspect both show every header key with count == header length, the accessor firewall, and the hostile-count rows. inspect_tests adds rosetta_json_emits_every_metadata_key. Suites: `cargo test -p aprender-core --lib -- format::gguf format::rosetta` 645 passed; `cargo test -p apr-cli --lib -- inspect` 140 passed. MUTANTS, each run and each RED: (1) skip restored (display-only insert removed): 13 red, including both header-count rows and all 10 builder rows; (2) the `.chain(display_only_metadata)` union dropped: both header-count rows red; (3) the cop''s mutant, one prefix dropped (`quantize.*` not kept): both header-count rows red; (4) bounds check removed: the hostile-count row red; (5) a `qwen35.` filter in rosetta_json: rosetta_json_emits_every_metadata_key red. KNOWN, NOT CHANGED: arrays whose element type is 8/16/64-bit (types 0,1,2,3,7,10,11,12) still decode to an empty ArrayUint32 and so display `[len=0]`. This is pre-existing, and the shard merge also depends on that variant, so it is out of scope. Census of all 21 GGUFs in /home/noah/models: array element types are STRING (59) and INT32 (30) only, so no fleet file reaches it.' diff --git a/docs/roadmaps/entries/PMAT-3747.yaml b/docs/roadmaps/entries/PMAT-3747.yaml new file mode 100644 index 0000000000..30be763a50 --- /dev/null +++ b/docs/roadmaps/entries/PMAT-3747.yaml @@ -0,0 +1,25 @@ +- id: PMAT-3747 + github_issue: 3747 + item_type: task + title: '#3568 PR 1/4: the TokenConstraint hook and the llguidance engine behind it — no generation loop wired' + status: in_progress + priority: high + assigned_to: aprender-0e + created: 2026-09-21T17:21:12Z + updated: 2026-09-21T17:25:00Z + spec: null + acceptance_criteria: + - 'Issue #3747 done_when, verbatim: "`constrain::TokenConstraint` {mask, accept, is_complete} exists, and `ConstraintError` names every refusal (SchemaInvalid, SchemaUnsupported, DeadEnd, Rejected, Vocab, NotCompiled), each rendered with its name first"' + - 'verbatim: "`load_schema` takes the schema inline OR as `@path` and refuses non-JSON, non-schema JSON and unreadable paths as SchemaInvalid, before any token"' + - 'verbatim: "llguidance 1.8.0 sits behind the trait, as an optional dependency behind a feature that apr-cli''s `inference` enables; `cargo deny check` (licenses + advisories) stays green with its full dependency tree"' + - 'verbatim: "`GGUFModel::constraint_vocab()` builds each token''s bytes from the SAME per-token decoding `decode()` uses; control/unused tokens are special and can never be constrained text"' + - 'verbatim: "case table: RAH''s quorum-lane schema and rmedia''s three schemas (#3716) compile exactly, accept their real documents, refuse violations at the byte, and allow nothing after one complete value; an approximation is refused; without the feature a constraint is refused (NotCompiled), never ignored"' + - 'verbatim: "mutants: a no-op mask, a dropped special marker, flexible whitespace, allowed approximations and a never-refusing accept each turn the case table RED"' + - 'verbatim: "a measured per-token mask overhead on the Qwen3.5-9B vocabulary (the backend-independent part; the CUDA host-logits download is PR 3''s measurement)"' + - 'NOT THIS PR, verbatim from #3747: "PR 2: apr run''s CPU paths, `--json-schema`/`--grammar`, exit codes + `removed_by`, `SchemaWithThinking`, the independent second reader, `finish_reason`/`prompt_tokens`. PR 3: the CUDA paths and their overhead. PR 4: `apr serve` after #3571. The lenient-extraction removal (§5.0 row 4) is infra''s, after PR 2."' + phases: [] + subtasks: [] + estimated_effort: null + labels: + - kind:code + notes: 'Parent #3568 (design note issuecomment-5763921678, ruled by the cop 2026-09-21): PR 1 of 4. Refs #3568 and Closes #3747. WHAT LANDS. crates/aprender-serve/src/constrain/mod.rs: TokenConstraint; ConstraintError with Display `SchemaInvalid: …`, `SchemaUnsupported: …`, `ConstraintDeadEnd: …`, `ConstraintRejected: …`, `ConstraintVocab: …`, `StructuredOutputNotCompiled: …`; load_schema (inline | @path); ConstraintVocab + validate(); ConstraintEnv (index once per model) -> json_schema / lark; doc: cross-field facts no JSON Schema expresses stay the consumer''s to check (rmedia, #3716). constrain/llg.rs (feature structured-output): special tokens carry TokTrie''s marker byte so no grammar can emit them as text. JSON is COMPACT unless the caller set its own x-guidance: llguidance''s default flexible whitespace ran away in this case table ({ then 63 bytes of whitespace). A boolean schema is normalised: `true` -> `{}`, compact; `false` -> SchemaInvalid (it admits no document). An approximation (grammar warnings, reachable only through a caller''s lenient/coerce_one_of x-guidance) is SchemaUnsupported, never used quietly. Engine compile errors map to SchemaUnsupported with the engine''s message verbatim: llguidance''s errors are untyped, and a substring sort would be a text match standing in for a type. A failed compute_mask (NoExtensionBias) and an empty mask are both DeadEnd. crates/aprender-serve/src/gguf/token.rs: GGUFModel::constraint_vocab() and push_token_bytes, which decode() now calls too. control/unused token_type (3, 5) are special, else the GH-320 <|…|> rule; SentencePiece''s ▁ is mapped on BYTES so a lone <0xNN> survives. test_factory gains add_i32_array. examples/constrain_mask_overhead.rs (required-features structured-output). DEPENDENCY. aprender-serve: llguidance 1.8.0 optional, default-features off, [lark, rayon, ahash] (referencing dropped: it duplicated referencing 0.28/0.29; a $ref schema is then refused by name). Feature structured-output = [dep:llguidance]; apr-cli inference adds realizar/structured-output. NOTE for reviewers: `realizar` IS aprender-serve. Root Cargo.toml has `realizar = { path = "crates/aprender-serve", package = "aprender-serve" }`, aprender-serve''s [lib] name is `realizar`, and apr-cli''s existing cuda feature enables `realizar/cuda` the same way. `cargo check -p apr-cli` rc 0 and the example (`use realizar::constrain`) builds rc 0. A round-1 lane read this as a mismatch and was refuted by the build. cargo deny check licenses / advisories / bans / sources: all rc 0; duplicate-crate warnings 64 on main -> 66, the +2 being strum/strum_macros 0.28 via derivre (llguidance''s regex engine). MEASURED at this commit. cargo test -p aprender-serve --lib --features structured-output -- constrain:: constraint_vocab: 24 passed. Without the feature: 12 passed, including StructuredOutputNotCompiled. Consumer schemas (RAH quorum-lane; rmedia quiz, outline, key-terms/reflection) compile exactly, accept real documents byte by byte to completion, allow nothing after one complete value, and refuse 7 violations at the byte (enum, minimum, pattern x2, additionalProperties:false, const, minLength). RAH row: a model intending the 2026-09-21 NULL vote {"verdict":"PASS","summary":"ok","finding":[]} is held until the required `findings` exists; unconstrained it writes the NULL vote. MUTANTS (8, each RED, re-run on this commit): mask a no-op (7 rows), special marker dropped (1, the in-string special row), flexible whitespace (3), approximations allowed (1), accept never refusing (2), token_type ignored (1), gpt2 bytes read as UTF-8 (1), boolean `true` left flexible (1). OVERHEAD, examples/constrain_mask_overhead, release, 3 runs, a real 124-token RAH verdict tokenized by the model''s own tokenizer and accepted end to end: Qwen3.5-9B-Q4_K_M (vocab 248,320) index 559-606 ms once per model, compile 0.67-0.79 ms, mask p50 155-163 us, p95 207-223 us, max 262-291 us per token; Qwen3-1.7B (151,936) p50 104-105 us. Against RAH''s measured decode of 1.0-2.9 tok/s that is under 0.05 percent. cargo fmt --all -- --check rc 0 (the example is formatted; round 1 caught it unformatted); clippy -p aprender-serve --lib --features structured-output -D warnings rc 0.' diff --git a/docs/roadmaps/entries/PMAT-3756.yaml b/docs/roadmaps/entries/PMAT-3756.yaml new file mode 100644 index 0000000000..befe1c0940 --- /dev/null +++ b/docs/roadmaps/entries/PMAT-3756.yaml @@ -0,0 +1,19 @@ +- id: PMAT-3756 + github_issue: 3756 + item_type: task + title: 'Parity lane refuses on intel and mini: the pinned llama.cpp (d1d3c3396) has no llama-bench/llama-server there' + status: in_progress + priority: high + assigned_to: aprender-04 + created: 2026-09-21T18:00:00Z + updated: 2026-09-21T18:00:00Z + spec: null + acceptance_criteria: + - 'Issue #3756 done_when, verbatim: "done_when: `llama_bin.sh` resolves the pin on intel and mini (evidence committed like `evidence/parity/pin-bump-d1d3c3396/RESOLVE-{lambda,gx10}.txt`), and `host_receipt.sh intel|mini` writes a `parity` block"' + - 'THIS PR delivers the FIRST clause only (llama_bin.sh resolves the pin on intel and mini, with the evidence committed). The second clause (host_receipt.sh writes a parity block) needs #3731, which is not on main; this PR says Refs #3756, not Closes.' + phases: [] + subtasks: [] + estimated_effort: null + labels: + - kind:code + notes: 'SCOPE: scripts/llama_bin.sh (one line: the commit-token extraction), evidence/parity/pin-bump-d1d3c3396/RESOLVE-{intel,mini}.txt, and this fragment. STACKED on #3563(A) (branch PMAT-3563-llama-pin-resolves-both-hosts @ 8fe36c080, receipt committed), whose canonical-location resolver it uses; fold #3563(A) first. HOST PROVISIONING (the cop''s fleet row; the operator: "you are traffic cop and have ssh, you fix anything you need"; done before the 0.69.1 train starts, not inside it): intel: `cmake --build ~/src/llama.cpp-d1d3c3396/build --target llama-bench llama-server llama-tokenize llama-cli -j 8` under nice 19 / ionice idle (incremental; the existing cache already matched build_flags_intel, GGML_NATIVE=ON, GGML_CUDA=OFF), BUILD_EXIT=0. mini: `git clone https://github.com/ggml-org/llama.cpp ~/src/llama.cpp-d1d3c3396 && git checkout d1d3c3396aa13a5f239109a822666c4870490ad5 && cmake -B build -DGGML_METAL=ON` (build_flags_mini verbatim), then the same four targets, BUILD_EXIT=0. DEFECT FOUND BY THE PROOF (fixed here): on mini, #3563(A)''s resolver refused the correct build as wrong_build. Its commit extraction `sed -nE ''s/…/p;t;s/…/p''` is one expression, and BSD sed (macOS /usr/bin/sed) reads everything after `t` as a LABEL, so it failed with "undefined label" and extracted nothing. Measured pre-fix on mini, rc 1: `sed: 2: "s/.*commit ([0-9a-f]{7, ...": undefined label '';s/.*\(([0-9a-f]{7,40})\).*/\1/p''` then `FAIL llama.cpp named but NOT the declared build (wrong_build)`. FIX: the same three commands as separate -e arguments. PROBE: 4 inputs (commit d1d3c3396, paren abcdef1, a 6-digit commit, no commit) give byte-identical output under GNU sed 4.8 (lambda) and BSD sed (mini): d1d3c3396 / abcdef1 / empty / empty. MEASURED AFTER: `bash scripts/check_llama_pin.sh` rc 0, 43 ok rows, both mutants still killed; live `env -u LLAMA_BENCH_PATH LLAMA_PIN_HOST= bash scripts/llama_bin.sh` rc 0 on intel (`version: 0.4.1-dev (build 2744, commit d1d3c3396)`) and on mini (`version: 0.4.1-dev (build 10987, commit d1d3c3396)`), in the committed RESOLVE files. REMAINDER: the llama.cpp builds live in user-space trees under ~/src on each host; declaring them in forjar (so a re-provisioned host rebuilds them) is not in this PR.' diff --git a/docs/roadmaps/entries/PMAT-3770.yaml b/docs/roadmaps/entries/PMAT-3770.yaml new file mode 100644 index 0000000000..6700410307 --- /dev/null +++ b/docs/roadmaps/entries/PMAT-3770.yaml @@ -0,0 +1,17 @@ +- id: PMAT-3770 + github_issue: 3770 + item_type: task + title: 'batch-1 combination break: check_release_bump_pr_body.sh fixture ladder receipts meet the current #3712 judge' + status: in_progress + priority: critical + assigned_to: null + created: '2026-09-21T00:00:00Z' + updated: '2026-09-21T00:00:00Z' + spec: null + acceptance_criteria: [] + phases: [] + subtasks: [] + estimated_effort: null + labels: + - kind:code + notes: 'ISSUE (#3770): folding #3721 + #3738 into release/0.69.1-batch-1 (ded8a932a) turned scripts/check_release_bump_pr_body.sh RED on 4 rows (epic-and-refs, prs-only, ladder-missing-gx10, ladder-ignored) + the drop-refusal mutant, all ''STOP model-ladder receipts for 9.9.9 are not green in the bump tree''. The table runs the REAL scripts/check_model_ladder.sh on fabricated receipts that were the pre-#3712 shape (v1, no inventory; a fixture ladder with no inventory block; a seed tree without the producer the judge''s GPU-lock audit reads and without the cells module the judge imports). done_when (the cop, 2026-09-21): fixtures green under the current judge; the two refusal rows name their intended reasons; every other table that fabricates ladder receipts swept in the same commit; guard_tree --no-cargo on the branch shows only the two known complexity REDs. WHAT THIS DOES: write_receipt writes apr-model-ladder-receipt/v2 (inventory = the fixture rung''s file, candidates = that file read as a Q4_K member, the rung row names its file); the fixture ladder COPIES the real contract''s `inventory` block (never retyped, so it follows the tree''s inventory shape: row A''s patterns today, #3763''s candidates + member_dtype after it folds) and its rung names a file; the seed tree gets real copies of scripts/model_ladder.sh and scripts/lib/model_ladder_cells.py. SWEEP: the other tables that reference the ladder judge were run and need nothing: check_publish_preflight.sh --selftest (R7 rows all ok; it uses a judge STUB) and check_release_models_t1.sh --self-test (rc 0; stubs). MEASURED: check_release_bump_pr_body.sh --self-test PASS 12 rows (all 4 formerly-red rows ok; ladder-missing-gx10 names evidence/dogfood/models/9.9.9/gx10.json and ladder-ignored names ''are gitignored''; mutants drop-keep-open, drop-refusal, drop-ladder, drop-ignored all killed); the same table with #3763''s judge, producer and contract swapped in: PASS 12 rows.' diff --git a/docs/roadmaps/entries/PMAT-3792.yaml b/docs/roadmaps/entries/PMAT-3792.yaml new file mode 100644 index 0000000000..a5b47af572 --- /dev/null +++ b/docs/roadmaps/entries/PMAT-3792.yaml @@ -0,0 +1,19 @@ +- id: PMAT-3792 + github_issue: 3792 + item_type: task + title: 'shapes_gate complexity ratchet: pure refactor of run_shapes_gate_with, no behaviour change' + status: in_progress + priority: critical + assigned_to: aprender-97 + created: 2026-09-21T21:58:00Z + updated: 2026-09-21T21:58:00Z + spec: null + acceptance_criteria: + - 'Issue #3792 done_when 1, verbatim: "`run_shapes_gate_with` is no longer RED in `bash scripts/check_complexity_ratchet.sh`, via a PURE refactor with no behaviour change: the same answers, in the same order."' + - 'done_when 2, verbatim: "`cargo test -p aprender-contracts --lib` is green, and `pv lint contracts --gate shapes` passes with 8/8 pc_extract controls fired."' + phases: [] + subtasks: [] + estimated_effort: null + labels: + - kind:code + notes: 'The cop (aprender-04) requested this, verbatim: "Please land a pure-refactor commit on a branch OFF release/0.69.1-batch-1 (not your old branch) that brings it under threshold with no behaviour change. Proof: `bash scripts/check_complexity_ratchet.sh` rc 0, `cargo test -p aprender-contracts --lib` green, and `pv lint contracts --gate shapes` 8/8 controls fired." SCOPE: this ticket IS the refactor. It is NOT #3715''s feature, which was already folded at ded8a932a; the diff against ded8a932a is only this refactor plus this fragment and receipt. The ratchet was RED on release/0.69.1-batch-1 (ded8a932a) vs main a9502d992: run_shapes_gate_with cyclomatic 13 cognitive 28 (the limit is 25). The fix moves the preamble (collect -> empty -> arming -> family, same order), the family ordering, the verdict rule and the two report maps into prepare / order_by_family / needs_receipts / verdict_of / by_shape / by_entity_type. MEASURED at 2a40dda85: the ratchet no longer lists run_shapes_gate_with (rc stays 1 only for crates/aprender-serve/src/constrain/tests.rs::generate_intent, not in this diff and not this ticket''s); cargo test -p aprender-contracts --lib 1701 passed; pv lint contracts --gate shapes Pass, 8/8 pc_extract fired; ont_release_readiness / ont4b / ont4c1 / ont4c3 CLI targets 34/11/12/10 passed; clippy -D warnings clean. A prior round under PMAT-3715 had a lane FAIL on SCOPE because it judged this delta against the #3715 feature ticket; the cop ruled that this refactor gets its own ticket.' diff --git a/docs/roadmaps/roadmap.yaml b/docs/roadmaps/roadmap.yaml index 997bd80d0e..fb9250e123 100644 --- a/docs/roadmaps/roadmap.yaml +++ b/docs/roadmaps/roadmap.yaml @@ -19457,6 +19457,26 @@ roadmap: estimated_effort: null labels: [] notes: null +- id: PMAT-3543 + github_issue: 3543 + item_type: task + title: 'The post-publish dogfood CANNOT pass: version-unpublished is not phase-aware and the banner still says pre-release' + status: in_progress + priority: high + assigned_to: aprender-f0 + created: 2026-09-20T08:17:54Z + updated: 2026-09-21T16:50:00Z + spec: null + acceptance_criteria: + - 'Issue #3543 defect 1 (the issue has no done_when list; its defects and its falsifier are the acceptance), verbatim: "1. **`version-unpublished` is not phase-aware.** It is a pre-publish gate ("this version must be absent from the index"). In the post-publish phase its precondition is inverted by construction: a successful publish makes it RED forever. `--phase` is parsed and validated (dogfood.sh:42-53) and 13 sites read `DOGFOOD_PHASE`, but this row is not one of them. Post-publish it should assert the opposite — the version IS on the index, resolvable, and `cargo install` of it works — which is the receipt the train actually wants."' + - 'defect 2, verbatim: "2. **The banner lies about which phase ran** (dogfood.sh:269 prints `dogfood pre-release:` unconditionally). Anyone reading the log believes they are looking at the pre-release gate set."' + - 'the issue''s falsifier, verbatim: "**Falsifier for the fix:** run `--phase post-publish` at a version that IS published and require GO; run it at a version that is NOT published and require RED on the new post-publish assertion. Both directions, or the row is decoration again."' + phases: [] + subtasks: [] + estimated_effort: null + labels: + - kind:code + notes: 'Release-tooling lane (cop, aprender-04), on the 0.69.1 release path. Its own branch off main 225b2a9ab: dogfood.sh is in none of the #3699/#3708/#3717 stack. SCOPE: scripts/dogfood.sh, the new scripts/check_dogfood_version_row.sh, and this fragment. SUPERSEDES open PR #3202 (PMAT-1098-dogfood-post-publish-phase, 2026-09-13), which the cop will close with these reasons. It does NOT meet this issue: (a) it patches only the dry-run branch, while main has since moved pre-publish onto a direct index.crates.io lookup (dogfood.sh:387-441 at 225b2a9ab); (b) it makes post-publish PASS unconditionally, so the falsifier''s second direction (a version NOT published must be RED) would read "not yet published" and PASS; (c) it leaves the banner. HOW IT IS MET. DEFECT 1: the sparse-index lookup is ONE function, index_version_state CRATE VERSION, which prints "present", "absent", "crate-absent" or "unknown ". Its body is the pre-publish code moved verbatim: the lowercased shard path, HTTP status read apart from the body, and a body that is not the index being UNKNOWN, never absent. version_row PHASE CRATE VERSION STATE decides the row. pre-publish gives version-unpublished, PASS only when absent or crate-absent (unchanged). post-publish gives a NEW row, version-published, PASS only when present AND installable (next), FAIL when absent ("… is NOT in the crates.io index — this phase runs after the cascade, so the version was not published"), crate-absent, or unknown. mark_version_row PHASE marks whichever row version_row decides. POST-PUBLISH ALSO INSTALLS (round 2): defect 1''s own text says post-publish must assert "the version IS on the index, resolvable, and `cargo install` of it works". Round 1''s lanes 1 and 2 FAILed an index-only row on that clause, and they were right. When the index carries the version, published_install_check runs `cargo install --version = --locked --root --target-dir ` with the registry token unset, exactly as a user installs. Every binary that lands must report the version in `--version`. A failed install (which is also where "resolvable" is decided: a version cargo cannot resolve, e.g. a yanked one, fails it), a binary reporting another version, or no binary at all turns version-published FAIL: "… is in the crates.io index but is NOT installable: …". No install is attempted for a version the index does not carry. The pre-publish branch calls it as before. The post-publish phase still runs the publish dry-run, because row 10 (publish-dry-run) reads its exit code and is what discharges the pre-publish DEFER. Its version row is then mark_version_row post-publish, and the pre-publish row is NOT emitted in that phase. The full phase is unchanged: the dry-run path, already exists -> FAIL. DEFECT 2: `phase_banner "$DOGFOOD_PHASE" "$CRATE" "$VERSION"` prints "══ dogfood : v ══". The only other mention of "dogfood pre-release" in scripts/ is a comment in dogfood-use.sh, which nothing parses. FALSIFIER, as a wired table: scripts/check_dogfood_version_row.sh, a scripts/check_*.sh with no bare cargo token, so guard_tree''s --no-cargo set runs it. It runs the REAL "2 + 10" section of dogfood.sh, extracted from its header to the next section header, so the call sites run as well as the helpers. It stubs `mark` (records rows), the publish dry-run, and curl (serving a fixture sparse index: present / absent / 404 / an HTML 200 / down). ROWS: pre-present FAIL; pre-absent PASS; pre-404 PASS; pre-down FAIL UNKNOWN; post-present version-published PASS (in the index, installed with `--version =9.9.9 --locked`, the binary reports 9.9.9) with NO version-unpublished row; post-install-fails FAIL; post-install-wrongver FAIL; post-install-nobin FAIL; post-absent FAIL with no install attempted; post-404 FAIL; post-html FAIL UNKNOWN; post-dry-run (the dry-run still runs, DRC 0); full-unchanged (already exists -> version-unpublished FAIL, no index call); banner ("══ dogfood post-publish: aprender v9.9.9 ══", printed from DOGFOOD_PHASE); index-path (crate "Aprender" is looked up at index.crates.io/ap/re/aprender). That is both directions in both phases, as the falsifier asks. MUTANTS, one per row, each killed: pre-present-pass, pre-absent-fail, pre-404-fail, down-is-absent, post-phase-blind (post-publish emitting the pre-publish row), post-absent-pass (#3202''s shape), post-404-pass, html-is-absent, no-post-call (the `mark_version_row post-publish` call deleted), post-skips-dry-run, full-uses-index, banner-fixed ("pre-release"), index-case (no lowercasing -> "looked up: curl https://index.crates.io/Ap/re/Aprender"); install-ignored, version-unchecked, nobin-ok, install-on-absent, unlocked (the install without --locked). The table''s first full run caught a fixture bug of mine: CRATE was exported before the per-row env, so index-path never saw the uppercase name and passed for the wrong reason until its mutant survived. MEASURED at this commit: `bash scripts/check_dogfood_version_row.sh` rc 0, "PASS 33 row(s)" (15 rows, 18 mutants); bashrs on dogfood.sh and the table, 0 errors (an `eval` in the banner row was replaced by sourcing the extracted line, after check_bashrs_gate''s SEC001); check_bashrs_gate, check_guards_are_wired and check_no_pipe_into_grep_q all rc 0. NOT RUN: a real `dogfood.sh --phase post-publish` against crates.io, which would include a real `cargo install aprender --version = --locked`, a full release build. The next train''s post-publish step is that measurement, hence Refs #3543.' - id: PMAT-3545 github_issue: 3545 item_type: task @@ -19506,6 +19526,25 @@ roadmap: estimated_effort: null labels: [] notes: null +- id: PMAT-3563 + github_issue: 3563 + item_type: task + title: 'Two llama.cpp comparator pins are live in-tree at once — 216 refs to d1d3c3396, 124 to 39173bcac; some committed artifacts measure against a superseded oracle' + status: in_progress + priority: critical + assigned_to: aprender-f0 + created: 2026-09-20T08:51:31Z + updated: 2026-09-21T17:00:00Z + spec: null + acceptance_criteria: + - 'Issue #3563 done_when, as the cop added it (comment 2026-09-21T16:35:16Z), verbatim: "done_when (added): exactly one pin in-tree; `llama_bin.sh` resolves and PROVES it on lambda and gx10 (a version+sha line) or fails naming why; every comparator consumer reads it"' + - 'THE SPLIT, recorded on the issue (comment 2026-09-21T16:49:53Z, "Split (cop ruling, aprender-04, 2026-09-21, order A → C → B). This issue now delivers (A) only"): this PR delivers clause 2 only, verbatim: "- **(A), this issue:** "`llama_bin.sh` resolves and PROVES it on lambda and gx10 (a version+sha line) or fails naming why". With nothing named, the resolver takes the pinned build at its CANONICAL location, `${LLAMA_PIN_SRC_ROOT:-$HOME/src}/llama.cpp-`. That path is derived from the pin, not searched for and not a host literal, and the build found there is verified by the same oracle-version, commit and cmake-cache checks as a named one. When it is absent the resolver FAILs naming the path and the declared build line. d1d3c3396 is built at that path on lambda (llama-bench and llama-tokenize added to the existing build) and on gx10 (a fresh build with `build_flags_gx10`)."; clause 3 is #3740 and clause 1 is #3741' + phases: [] + subtasks: [] + estimated_effort: null + labels: + - kind:code + notes: 'P0 for the 0.69.1 path (cop, aprender-04): four 0.69.1 rows need ONE resolvable pinned comparator (#3726 tokenizer parity, #3725, #3596, #3693). SPLIT (cop ruling, recorded on the issue, comment 5764188590): this issue delivers (A) only, "llama_bin.sh resolves and PROVES it on lambda and gx10 (a version+sha line) or fails naming why". (C) "every comparator consumer reads it" is #3740; (B) "exactly one pin in-tree" (classify the 142 refs to 39173bcac and the 3 to 60b06ab9a, plus the citation guard) is #3741. Its own branch off main 52f43da71 (the merge-base; used as the quorum base). SCOPE: scripts/llama_bin.sh, scripts/check_llama_pin.sh, two evidence files under evidence/parity/pin-bump-d1d3c3396/, and this fragment. MEASURED BEFORE THE CHANGE: the tree has ONE pin, d1d3c3396 (scripts/llama_pin.toml build_commit). gx10''''s "39173bcac" was only its stale ~/src/aprender checkout (de0e3e182, 2026-08-28). On lambda, `bash scripts/llama_bin.sh` printed "FAIL … (no_binary_named) … reported: ": the resolver never searches (NEVER PATH), $LLAMA_BENCH_PATH was unset, and the pinned build sat unnamed at ~/src/llama.cpp-d1d3c3396, with llama-cli and llama-server but NO llama-bench (the resolver''''s candidate) and no llama-tokenize. ~/src/llama.cpp there is 60b06ab9a, whose llama-cli dies "undefined symbol: llama_memory_breakdown_print". gx10 had NO d1d3c3396 build (~/src/llama.cpp-pin is the old 39173bcac). HOW (A) IS MET. (1) THE CANONICAL LOCATION: with nothing named, llama_bin.sh takes ${LLAMA_PIN_SRC_ROOT:-$HOME/src}/llama.cpp-/build/bin/llama-bench. That path is DERIVED FROM THE PIN, not a PATH search and not a host literal, and the build found there passes the same oracle-version, commit and cmake-cache checks as a named one. A named $LLAMA_BENCH_PATH always wins. When it is absent, the executable mode FAILs naming the canonical path and the pinned commit to build ("fails naming why"). (2) THE ABBREVIATION LENGTH, found by running the resolver on gx10: gx10''''s correct build prints "commit d1d3c339" (8 hex digits; git abbreviates to what is unique in THAT clone), lambda''''s "commit d1d3c3396". The old test (the 9-digit pin as a substring of the version line) refused gx10''''s correct build as wrong_build. The commit token is now read out of either --version shape ("commit " or "()") and matched when one abbreviates the other at git''''s minimum of 7 hex digits (llama_commit_match). Like the build number, which llama_pin.toml already says is not an identity, the length is not. (3) THE BUILDS: on lambda, `cmake --build build --target llama-bench llama-tokenize` in ~/src/llama.cpp-d1d3c3396 (incremental, BUILD_EXIT=0; cache GGML_CUDA=ON with no CMAKE_CUDA_ARCHITECTURES, matching build_flags_lambda). On gx10, a fresh clone of ~/src/llama.cpp detached at d1d3c3396aa13a5f239109a822666c4870490ad5 into ~/src/llama.cpp-d1d3c3396, `cmake -B build -DGGML_CUDA=ON -DCMAKE_CUDA_ARCHITECTURES=121` (build_flags_gx10 verbatim), `cmake --build build -j 16`: BUILD_EXIT=0, 285M, llama-bench/cli/server/tokenize/completion, cache GGML_CUDA=ON CMAKE_CUDA_ARCHITECTURES=121. The disk was checked first (174G free, 81%). No GPU lock was needed for a build. PROOF, committed as evidence/parity/pin-bump-d1d3c3396/RESOLVE-lambda.txt and RESOLVE-gx10.txt: `env -u LLAMA_BENCH_PATH bash scripts/llama_bin.sh` rc 0 on BOTH hosts. lambda: "ok llama.cpp pinned: …/llama.cpp-d1d3c3396/build/bin/llama-bench / build: version: 0.4.1-dev (build 10987, commit d1d3c3396) / cmake: GGML_CUDA=ON CMAKE_CUDA_ARCHITECTURES=unset / pin expires 2026-12-01". gx10: "… build: version: 0.4.1-dev (build 2423, commit d1d3c339) / cmake: GGML_CUDA=ON CMAKE_CUDA_ARCHITECTURES=121 …" (run from a scratch copy of this branch''''s llama_bin.sh + llama_pin.toml, since gx10''''s own checkout is stale). CASE TABLE, scripts/check_llama_pin.sh (ci-wired as before). Every row now runs against a FIXTURE canonical root, never the operator''''s $HOME. New rows: nothing named with the canonical build present -> rc 0; the canonical build at the WRONG commit -> wrong_build; the canonical build''''s cmake disagrees -> cmake_mismatch; a NAMED path wins over the canonical build -> wrong_build; UNPINNED never falls back to a canonical build -> unpinned; an oracle abbreviating SHORTER (7) and LONGER (10) than an 8-digit pin -> rc 0; 6 hex digits (below git''''s minimum) -> wrong_build; a different commit of the same length -> wrong_build. MUTANTS, each killed: canonical-lookup (the path pointed nowhere, as before) -> "expected rc=0, got rc=1 (no_binary_named)"; substring-match (the pre-#3563 test) -> "oracle abbreviates SHORTER than the pin expected rc=0, got rc=1 (wrong_build)". MEASURED at this commit: `bash scripts/check_llama_pin.sh` rc 0. #3726''''s owner (aprender-c3) was told to use the pinned llama-tokenize at the canonical path on both hosts, not ~/src/llama.cpp.' - id: PMAT-3567 github_issue: 3567 item_type: task @@ -19560,7 +19599,7 @@ roadmap: title: 'apr serve routes the Qwen3.5 hybrid to Qwen35Model instead of the dense path' status: in_progress priority: medium - assigned_to: null + assigned_to: aprender-c7 created: 2026-09-21T00:00:00Z updated: 2026-09-21T00:00:00Z spec: null @@ -19570,7 +19609,9 @@ roadmap: estimated_effort: null labels: - kind:code - notes: 'ACCEPTANCE (hand-entered from gh#3571 done_when; see paiml-mcp-agent-toolkit#1414). Layer 1 of a surface no test had ever traversed: apr serve could not load the flagship architecture at all, and each assumption was visible only once the one above it cleared. Done when: handler_gpu_completion.rs branches on is_qwen35 and calls Qwen35Model::create_base_model rather than falling through to the dense OwnedQuantizedModel path, which is the wrong architecture for a Gated DeltaNet hybrid. Scope is deliberately ONE LAYER: land what works rather than hold the whole stack behind the last unknown. STOP: do not claim the serve path is end-to-end green on Qwen3.5 from this row alone — the layers above it are separate rows.' + notes: 'ACCEPTANCE (cop rulings, moved from acceptance_criteria at the 0.69.1 fold: check_roadmap_diff_additive forbids editing an existing entry''s acceptance_criteria, and notes is a lifecycle field): Cop ruling (aprender-04, 2026-09-21, after an external user hit #3571 on main), unit (1), verbatim: "FIRST, the smallest safe unit: no serve request handler may serve a model whose transformer stack resolved to 0 layers, on any arch and both routes. Refuse at load, naming the arch and #3571, with a case row plus a mutant. Stop at its receipt; I fold it with priority." | Cop condition (a), verbatim: "Siblings. List every other `apr serve` load entry point (APR and SafeTensors starts, and any CLI-side serve path in aprender-serve `cli`). In the body, say for each one whether it can reach a 0-layer model: guarded, or unreachable and why. A fix to one copy makes the other copies look reviewed." | Cop condition (b), verbatim: "(1) must not block (2). Qwen3.5''s generic stack is legitimately 0 layers, so when (2) lands, qwen35 must route to Qwen35Session BEFORE the check, or the check must test the stack the chosen route actually executes. Add a row to (2) proving qwen35 serve passes the refusal. (1) alone refuses qwen35 on main, which is correct until (2)." || ACCEPTANCE (hand-entered from gh#3571 done_when; see paiml-mcp-agent-toolkit#1414). Layer 1 of a surface no test had ever traversed: apr serve could not load the flagship architecture at all, and each assumption was visible only once the one above it cleared. Done when: handler_gpu_completion.rs branches on is_qwen35 and calls Qwen35Model::create_base_model rather than falling through to the dense OwnedQuantizedModel path, which is the wrong architecture for a Gated DeltaNet hybrid. Scope is deliberately ONE LAYER: land what works rather than hold the whole stack behind the last unknown. STOP: do not claim the serve path is end-to-end green on Qwen3.5 from this row alone — the layers above it are separate rows. + + UNIT (1), the zero-layer refusal (aprender-c7). Measured on #3571 (issuecomment-5763718338) at release/0.70-batch-1 262dfb2f6, which has the fix of #3678: the qwen35 base reached the dense servers with 0 layers — CPU route HTTP 200, content "", 1024 tokens of "\n" in 6.0 s; GPU route HTTP 500. On main it stops earlier at the refusal, so this unit is what keeps main honest when batch-1 lands. THE CHECK: `zero_layer_refusal(arch, layers)` (pure; ModelLoadFailed, exit 6, naming the arch, the 0 and #3571; the apr run/apr chat hint only for qwen35) inside `build_serve_model`, the one GGUF loader of apr serve. EVERY apr serve LOAD ENTRY POINT (condition a) — found by sweeping crates/apr-cli/src/commands/serve for from_mapped / from_apr / from_apr_file / SafetensorsToAprConverter / spawn_apr_q4k_inference_thread; ALL GUARDED: [1] GGUF --gpu, [2] GGUF --gpu --batch, [3] GGUF CPU — start_gguf_server takes its model from build_serve_model BEFORE the route is chosen. [4] GGUF --backend wgpu (try_start_wgpu_backend) and [5] the CPU fallback start_gguf_server_cuda rebuilds when CUDA init fails — both built their own with from_mapped; both now load through build_serve_model (one GGUF loader, one check). [6] APR CPU transformer (AprTransformer::from_apr_file), [7] SafeTensors single-file and [8] sharded (SafetensorsToAprConverter::convert / convert_sharded): the helper on t.config.num_layers where the transformer is in hand. [9] APR GPU (start_apr_server_gpu), [10] APR CPU quantized (try_apr_quantized_cpu), [11] SafeTensors GPU (start_safetensors_server_gpu), [12] SafeTensors CPU quantized (start_safetensors_server_cpu_quantized): the helper on the OwnedQuantizedModel before its Model ready line. [13] APR Q4K pool GPU (start_apr_q4k_server_gpu) parses its layer count on the scheduler thread after the upload begins, so the count the file DECLARES is read first (AprV2Model::load is mmap-based) and 0 is refused before the spawn; a file declaring none is refused by name inside the pool path (parse_apr_q4k_config: APR metadata missing num_layers). [14] aprender-serve cli (serve_model / prepare_serve_state): UNREACHABLE from apr — `git grep realizar::cli:: -- crates/apr-cli/src` has no hit — and it builds GGUF models with from_mapped, which refuses the qwen35 hybrid by name. REPO-LAYOUT FACTS a reviewer needs (round 1 lanes 1-3 misread them as compile errors; each checked at the head): handlers.rs:1642 is `include!("handler_gpu_completion.rs")`, handlers.rs:1643 `include!("server.rs")`, and handler_apr_cpu_completion.rs:869 `include!("handlers_include_01.rs")` — so build_serve_model and zero_layer_refusal, defined in handler_gpu_completion.rs, ARE items of the `handlers` module, callable unqualified from handlers.rs and handlers_include_01.rs, and reachable from the sibling module as `super::handlers::zero_layer_refusal` (pub(super)). Proof it compiles: `cargo test -p apr-cli --lib --features cuda zero_layer_refusal` builds apr-cli and passes both tests. CONDITION (b): on main this refuses qwen35 serve outright (correct until unit 2). Unit 2 (branch PMAT-3571-serve-qwen35-session) returns to the resident Qwen35Session BEFORE build_serve_model is called and adds the row proving qwen35 serve passes. TESTS: zero_layers_is_refused_by_name_and_any_layer_is_admitted (the case table: qwen35/0 and llama/0 refused, 1/28/24 admitted); serve_refuses_the_qwen35_base_at_load (real Qwen3.5-0.8B through build_serve_model; SKIPs where the file is absent — lambda/gx10 carry it, CI does not). MUTANT: deleting the check in build_serve_model turns serve_refuses_the_qwen35_base_at_load RED ("a 0-layer stack reached the serve routes"), measured.' - id: PMAT-3574 github_issue: 3574 item_type: task @@ -20007,6 +20048,23 @@ roadmap: labels: - kind:code notes: 'ACCEPTANCE (quoted from issue #3673 "What lands"): 1. scripts/batch_fold.sh — fold receipted PR branches into one release/-batch branch, git merge --no-ff in the given order: a conflict ONLY in the generated set {docs/roadmaps/roadmap.yaml, contracts/census.json, contracts/contracts.nt, contracts/shapes.ttl, the CONTRACT_COUNT blocks of README.md} is taken and marked for regeneration; a conflict ANYWHERE else — including README prose outside a count block — aborts that branch''s merge and reports SKIP with the paths, never resolved by picking a side; --regen regenerates the generated set ONCE with the tree-built pv (scripts/pv_bin.sh), asserts the fixed points (aggregate --check, pv extract --check, readme_sync.sh --check), and commits the regeneration; exit 0 all folded; 1 any SKIP; 2 usage/git/regeneration failure; a folded-but-unregenerated generated set is never silent; --self-test case table, wired into CI. 2. APR-RELEASE-001 §13 — the batching procedure as the default path (cop prompt §7): assembly, generated-set regeneration, left-out rules, what needs a new quorum (only content new to the batch), constituent disarm/close, a red batch, and the review-lane VOID rule while paiml-implement#317 is open. AUTHOR NOTES (not criteria): the case table lives in scripts/check_batch_fold.sh (guard-tree runs it) because a --self-test case inside batch_fold.sh made check_guards_are_wired.sh RED; §13 required amending hard rule 9 ("No hand-squash of batches") and the matching §8 stop, which the new default contradicts.' +- id: PMAT-3676 + github_issue: 3676 + item_type: task + title: 'CI diet: aprender itself burns 99 intel runner-hours/day — coverage on tag/nightly only, path-filter book jobs, perf/model suites nightly, mutants only on .rs diffs' + status: planned + priority: critical + assigned_to: null + created: 2026-09-21T09:50:38Z + updated: 2026-09-21T09:50:38Z + spec: null + acceptance_criteria: [] + phases: [] + subtasks: [] + estimated_effort: null + labels: + - kind:code + notes: 'ACCEPTANCE (quoted from issue #3676 "Proposal" + "done_when"; operator 08:38Z: "YES, coverage on tags release only"; Noah approved all five workflow items directly to aprender-b3): 1. Coverage (ci / coverage) on the release tag and the nightly only, never on pull_request or merge_group. COV_FLOOR enforcement moves to the tag gate (T-2) and the nightly, where a regression still blocks the release. 2. Book Chapter Examples and mdBook: path-filtered. They run when book/** or the crates a chapter imports change, and nightly always. 3. The perf gate case table and the model-tests falsification suites move to nightly. The PR path keeps a smoke row. 4. mutants runs only when the diff has .rs. 5. Every moved job still runs on the tag. check_workflow_path_filters.sh and the gate aggregator are updated so a skipped job is a pass only when the skip reason is the documented filter. done_when: intel runner-minutes per 24 h drop by >= 40% at equal PR throughput (measured after merge); the tag gate still runs coverage with COV_FLOOR, and a planted coverage drop fails the tag (control); a book-only PR runs the book jobs, a crate-only PR not imported by a chapter does not (both rows); gate accepts skipped only for jobs whose filter says skip, any other skipped state fails (mutation). COP ADDENDUM (aprender-04, relaying rmedia-88): before removing coverage from ci.yml, enumerate every consumer of the coverage number and prove each still has a producer, with a guard asserting the chain. (A), THE ci / gate FIX (cop ruling 2026-09-21 ~15:00Z; batch-1 #3707 left this row out): the first spelling, `skip_coverage: true` on sovereign-ci@70e51ec, made `ci / gate` RED on every PR: that reusable''s gate counts a skipped coverage job as a mandatory failure ("coverage did not succeed (result: skipped)"). This PR''s own CI was red on it (run 35585906113, cancelled unread) and so was batch-1. The fix: pin sovereign-ci to paiml/.github f713290c86fcd70d6f26a0faab14e67e6713586f (#74 `coverage_on` + #77), pass `coverage_on: tag`, add `push: tags: [v*]` to ci.yml (the reusable''s second required edit), and remove coverage-nightly''s v* tag trigger so each tag is measured once, by ci.yml. The same pin also brings #73''s roadmap-fragment-parity, which would have refused every aprender PR (aprender still commits roadmap.yaml). #77 keys it on the BASE tree: NOT-ACTIVE while roadmap.yaml is tracked, and it arms by itself when #3709 migrates. scripts/check_coverage_has_producers.sh now asserts R3 (the nightly runs on schedule, NOT on v* tags) and R5 (ci.yml has coverage_on: tag, no live skip_coverage: true, and on: push tags [v*]), with 16/16 self-test rows including "skip_coverage: true instead of coverage_on -> RED". On f7b25a6c6 the guard reports R3 and R5 RED; after the change it PASSES. Also in this change (cop, found by aprender-f0): ci.yml runs `check_pr_closes_issue.sh --self-test` beside `--body`, because guard_tree skips it as wired-with-args and its case table ran in no job, and the ci.yml comment that claimed otherwise is corrected. PROOF REQUIRED before re-folding: this PR''s own CI green, with the gate logging `coverage: NOT MEASURED` and `roadmap-fragment-parity: NOT-ACTIVE`.' - id: PMAT-3681 github_issue: 3681 item_type: task @@ -20062,6 +20120,27 @@ roadmap: labels: - kind:code notes: 'ACCEPTANCE (hand-entered from issue #3685 done_when, verbatim; minted on the PR branch so the AD-04 quorum can run). (1) apr parity whose quant type the CUDA capability gate refuses (PMAT-785) prints ONE stderr line parity: REFUSED architecture= quant= reason= and exits 12, never 5; exit 5 stays reserved for a comparison that ran and failed. (2) a RED-turning test: a refused quant type gives 12 plus the mark; a supported one never takes that path; mutation-proved (map the refusal back to 5 and the test fails). (3) bash scripts/check_model_parity.sh --manifest --apr on this host: the qwen2.5-0.5b-instruct row is UNMEASURED-TOOL (report), every Q4_K_M row still PASS, and the must-RED twin still turns C14 RED. (4) scripts/dogfood.sh --phase pre-publish model-parity row is not FAIL on this host. Blocks the 0.69.0 T-2 preflight and the release-commit dogfood; own small PR, not folded into #3669.' +- id: PMAT-3691 + github_issue: 3691 + item_type: task + title: 'apr hex / apr trace still say "Invalid APR format" for GGUF and SafeTensors failures — 6 more InvalidFormat sites after #3661' + status: in_progress + priority: medium + assigned_to: aprender-37 + created: 2026-09-21T14:25:00Z + updated: 2026-09-21T14:25:00Z + spec: null + acceptance_criteria: + - 'Issue #3691 done_when 1, verbatim: "Each of the six sites uses `CliError::invalid_model_file(path, context, &e)` (added by #3661), or a variant naming the format it already knows. Exit code stays 4."' + - 'done_when 2, verbatim: "A row per command (`hex`, `trace`, and the SafeTensors header reader) on a truncated GGUF / malformed SafeTensors asserts that the message names the format and does not say "Invalid APR"."' + - 'done_when 3, verbatim: "The remaining `CliError::InvalidFormat(` sites (130 at `d0771c5c5`, including these six) are classified: APR-only input (keep) versus possibly-non-APR (move). The classification goes in the PR body."' + - 'The six sites, verbatim from the issue (line numbers at `d0771c5c5`): hex.rs:323 and hex.rs:350 `InvalidFormat(format!("Failed to parse GGUF: {e}"))`; safe_tensors_header.rs:180 `InvalidFormat(format!("Invalid SafeTensors header UTF-8: {e}"))`; safe_tensors_header.rs:182 `InvalidFormat(format!("Invalid SafeTensors JSON: {e}"))`; trace_likely_has_repeated.rs:267 `InvalidFormat(format!("Failed to parse GGUF: {e}"))`; trace_likely_has_repeated.rs:332 `InvalidFormat(format!("Failed to inspect SafeTensors: {e}"))`.' + phases: [] + subtasks: [] + estimated_effort: null + labels: + - kind:code + notes: 'Dispatched by the cop, aprender-04, to aprender-37 (2026-09-21 ~14:25Z). BASE: `0517e75a7` (PMAT-3661), not origin/main. #3691 reuses `CliError::invalid_model_file`, which #3661 added, and #3661 is not on main yet: it is folded into the unpushed `release/0.70-batch-1`. This branch is one commit on top of it, so the batch folds only this delta. HOW EACH done_when IS MET. (1) The six sites use `CliError::invalid_model_file(path, context, &e)`: hex.rs `parse_gguf` and `get_gguf_tensor_f32` (context "Failed to parse GGUF"), and trace_likely_has_repeated.rs `trace_gguf` ("Failed to parse GGUF") and `trace_safetensors` ("Failed to inspect SafeTensors"). Each has a path and an `AprenderError`, so the helper names the format from the file''s magic bytes and strips the core FormatError prefix. The two SafeTensors header-reader sites (UTF-8, JSON) have neither a path nor an `AprenderError`: `parse_safetensors_header(bytes)` gets the whole file, which `hex::run` already read, and a `Utf8Error` / `serde_json::Error`. They use a new sibling constructor, `CliError::invalid_model_bytes(bytes, message)`. It follows the same rule, reading the format from the bytes'' magic and never assuming it, through a new `hex::detected_format_of(bytes)`; `detected_model_format(path)` now calls that function after reading its 9 bytes. That is the done_when''s "variant naming the format it already knows". Every moved site keeps exit code 4: `InvalidModelFile` is on the same arm as `InvalidFormat`, and each row asserts `exit_code_value() == 4`. SCOPE, with one rule and its exclusions named. Besides the six, four more sites move: the rest of "the SafeTensors header reader", the unit done_when 2 names. Those are `parse_safetensors_header`''s size check ("SafeTensors file too small") and length check ("SafeTensors header length exceeds file size"), plus the "SafeTensors header is not a JSON object" check in each of its two callers (`run_safetensors`, `slice_safetensors`). The issue''s grep matched `format!(` calls only, so it missed these four string literals. Leaving them would keep printing "Invalid APR format: SafeTensors …" from the same reader. That makes 10 sites in 4 files. NOT moved, deliberately: hex.rs:353, `get_gguf_tensor_f32`''s tensor read. It also fires on `apr hex --slice … --tensor NAME` with a NAME that is not in the file (core says "Tensor ''NAME'' not found in GGUF"), so moving it would call a healthy file "Invalid GGUF file". It needs a split, not a move, and is classified below. The other 116 sites are classified, not changed. `CliError::InvalidFormat` itself is unchanged. TESTS (done_when 2). New `commands/model_file_error_tests_3691.rs`, declared as a `#[cfg(test)]` child of `hex` so it can call the private reader. It has one row per moved site, plus a second-caller row and an APR control. Each row renders `error: {e}` exactly as main.rs prints it and checks four things, the same as #3661''s table: the expected prefix; no "Invalid APR" unless the file is APR; no core "Invalid model format" anywhere; and exit 4. Rows that a file can reach call the real `hex::run` / `trace::run`: a truncated GGUF (hex, `hex --slice`, trace); SafeTensors with an overrunning header length, a non-UTF-8 header, and a cut-off header `{"a":` (hex, `hex --slice`, trace); and #3661''s 4 KiB noise named `.safetensors` (trace, by extension). Three sites cannot be reached through `hex::run`, because its `detect_format` picks SafeTensors only for 9+ bytes with `{` at byte 8: the 4-byte size check, and "not a JSON object" in both callers (a header starting with `{` is an object or not JSON at all). Their rows call the private function directly with the bytes that would reach them: 4 bytes, and a `[1]` header. The expected messages are: GGUF rows "Invalid GGUF file: Failed to parse GGUF: Unexpected EOF reading u64 at offset 24"; hex SafeTensors rows "Invalid SafeTensors file: "; the 4-byte and array rows "Invalid model file: …", because their magic matches no format; trace on the cut-off header "Invalid SafeTensors file: Failed to inspect SafeTensors: …"; trace on noise "Invalid model file: Failed to inspect SafeTensors: …"; and the APR control (a truncated .apr through `run_apr`, unchanged) "Invalid APR format: Failed to read APR: …". The fixtures are #3661''s, made `pub(super)` in model_file_error_tests_3661.rs instead of copied; that file''s own test is unchanged and still passes. CLASSIFICATION (done_when 3). It is in the commit message and goes in the PR body. There are 126 sites at `0517e75a7`: 125 `CliError::InvalidFormat(` lines plus one function-pointer use (dispatch_run.rs:242 `.map_err(CliError::InvalidFormat)`, which the `(` grep misses). The issue''s 130 was `git grep ''InvalidFormat(''` at `d0771c5c5`: 128 sites, the variant declaration, and its exit-code arm. #3661 moved 3. Classes, decided by the input that reaches each site: MOVED here, 10; MOVE, model file (may be GGUF, SafeTensors, or unrecognised, so "Invalid APR format" can be false), 33; MOVE, not a model file (CLI arguments, JSON/CSV/corpus inputs, HF responses, serialising our own reports, IO), 58; KEEP, APR-only input, 15; TEST, asserting on the variant, 10. A script checked that every one of the 126 appears in exactly one class. STOP LIST: no change to `InvalidFormat`''s Display or to any of the 116 unmoved sites; no new error variant; no edit outside apr-cli except this fragment and the regenerated roadmap.yaml; no push and no arm, because the cop folds this into the batch. MEASURED at this commit: `cargo test -p apr-cli --lib`: 7295 passed, 0 failed, 12 ignored (#3661''s 7294 plus this table). `cargo test -p apr-cli --lib model_file_error_tests`: 2 passed, this table and #3661''s, whose fixtures it now shares. `cargo clippy -p apr-cli --lib -- -D warnings`: rc 0. `--tests` fails only on 2 findings in the untouched pre-existing `commands/nf4_classifier.rs`, the same state #3661 recorded; none are in a touched file. `cargo fmt --all -- --check`: rc 0. Rustfmt does not reach safe_tensors_header.rs through its `include!`, so its changed lines were formatted by hand, and a whole-file rustfmt was not applied because it rewrites untouched lines. `scripts/check_complexity_ratchet.sh`: "a877fa056 vs … none new, none grown". `bash scripts/guard_tree.sh --no-cargo`: 76 checks, 0 failed. `check_roadmap_fragment_required.sh`, `roadmap_fragments.py aggregate --check`, `check_roadmap_sorted.sh` and `check_roadmap_ids_unique.sh`: all PASS. MUTANTS, run one at a time against this table, each RED: S01–S10, each moved site reverted alone to its pre-fix `CliError::InvalidFormat(…)`, fails exactly its own row. For example, S01 renders "error: Invalid APR format: Failed to parse GGUF: Invalid model format: Unexpected EOF reading u64 at offset 24", the issue''s string. With all 10 reverted, the 12 non-control rows fail (29 checks) and the APR control still passes. H1, `invalid_model_bytes` hardcoded to "SafeTensors" (the assumed format), fails the 3 "Invalid model file" rows (4 bytes, array header ×2). H2, `detected_format_of` always returning "model", fails the 8 rows that expect GGUF or SafeTensors. Sources were restored and checked by md5 before the full runs. NOT DONE: a run of a freshly built `apr` binary. The host was at load 55–64 on 48 cores, and a release build of the facade would have added to it. The table renders the same `error: {e}` line main.rs prints and asserts the exit-code mapping directly.' - id: PMAT-3692 github_issue: 3692 item_type: task @@ -20083,6 +20162,48 @@ roadmap: labels: - kind:code notes: 'ANDON, found by aprender-62 as first responder on #3689: its guard-tree (job 106309701767, intel-clean-room-3) failed "76 checks, 1 failed" on this guard alone, "ENV: no TOML reader on this interpreter (python 3.10 …)". #3644 (batch #3669, a877fa056) put the guard in guard-tree''s --no-cargo population while its own header said the intel hosts have python 3.10.12 and no reader. The batch''s guard-tree passed only because it ran on gx10-pool1. The fleet fix is OUT: infra declared the fleet python-free for automation (infra#708, 02:05Z, "do not propose installing tomli"). So the cop ruled (~11:28Z) that the fix is the guard, folded onto #3689 for one CI cycle. SCOPE: this commit touches only scripts/check_pathonly_devdeps_unused_in_src.sh plus this fragment. #3689''s PMAT-3685 parity change is judged by its own receipt. HOW IT IS MET. scan() returns a third code, 3, for fleet state, in exactly two ways: `command -v` finds no interpreter (reason=no-interpreter), or the scanner prints its own "ENV: no TOML reader on this interpreter" line AND exits 3 (reason=no-toml-reader; the scanner''s sys.exit changed 2 -> 3). The pattern is keyed on the PAIR, so either one alone stays ENV. In both cases scan() prints `UNMEASURED runner=… reason=… … fleet state, not a pass (#3692)` to stderr, where it survives the callers'' command substitution; guard_tree captures both streams. selftest() passes 3 through; main() returns 0 on 3 from the case table or the tree scan, before any verdict or OK line. Every other failure path is unchanged: ENV rc 2. `--selftest` in fleet state also exits 0. DEATH ROWS, 5 -> 7: FORCE_NO_TOML -> 3; no interpreter -> 3 (was 2); /bin/false -> 2; an interpreter that exits 3 printing a traceback but NOT the no-reader line -> 2 (new: the exit code alone does not buy an UNMEASURED); the working interpreter -> 0; the WHOLE guard under FORCE_NO_TOML -> rc 0 with "UNMEASURED runner=intel-probe reason=no-toml-reader" and no "missed hit_", "false positive" or "OK: no NEW" line (was: rc 2); the WHOLE guard under /bin/false -> rc 2 (new). MEASURED at this commit: `bash scripts/check_pathonly_devdeps_unused_in_src.sh --selftest` rc 0, "PASS: case table (4 must-match, 7 must-not-match, 7 death rows)"; a bare run on this box''s python3 (which has a reader) rc 0, "OK: no NEW src/ reference to a publish-stripped dev-dep (3 baselined pair(s), #3306)", so the real verdict is unchanged where a reader exists. REAL PYTHON 3.10: /usr/bin/python3.10 is 3.10.12; run with -S (no site-packages) it cannot import tomli and has no tomllib, which is the intel host''s shape with a real interpreter. The whole guard under it gives rc 0, "UNMEASURED runner=py310-probe reason=no-toml-reader -- no TOML reader on this interpreter (python 3.10, need tomllib >= 3.11 or tomli) …". The PRE-FIX guard (main) under the same interpreter gives rc 2, "ENV: the scanner did not finish … ENV: the case table could not be measured (scan rc=2)", which is #3689''s failure reproduced. MUTANT, every ENV mapped to UNMEASURED (scan''s ENV branch `return 2` -> `return 3`): selftest rc 1 with three death rows RED, "rc=3 (wanted 2) interpreter exits 1 with no output", "rc=3 (wanted 2) interpreter exits 3 without the no-reader line", "rc=0 (wanted 2) the whole guard under a dead interpreter". GUARD_TREE UNDER THE SAME REAL PYTHON 3.10 (PATHONLY_GUARD_PYTHON=<3.10 -S stub> bash scripts/guard_tree.sh --no-cargo): rc 0, "76 checks, 0 failed", with "PASS scripts/check_pathonly_devdeps_unused_in_src.sh [run]" and beneath it "~ UNMEASURED runner=py310-probe reason=no-toml-reader -- …", surfaced by #3651. That is the job #3689 lost on intel-clean-room-3.' +- id: PMAT-3693 + github_issue: 3693 + item_type: task + title: 'Qwen3.5-9B Q4_K_M on CUDA at ~12k-token context corrupts identifiers it quotes from the prompt and runs at 1.0 tok/s: discriminate model weakness from long-context CUDA divergence' + status: in_progress + priority: critical + assigned_to: aprender-c3 + created: 2026-09-21T15:55:00Z + updated: 2026-09-21T15:55:00Z + spec: null + acceptance_criteria: + - 'Issue #3693 done_when 1, verbatim: "**discriminate first, then fix.** The same prompt, same seed, temperature 0: CPU (`--no-gpu`) vs CUDA, token by token. Identical corruption means model/quant weakness (document it, then compare a Q8_0/F16 9B); divergence means a CUDA path defect."' + - 'done_when 2, verbatim: "`apr parity` (or `--per-op`) at ≥ 8k positions, CPU vs CUDA, with the first position where cosine drops below the C14 floor named. C14''s corpus gains a long-context row, so the gate covers the context length the product is used at."' + - 'done_when 3, verbatim: "decode rate at 12k context measured separately from prefill, with `--json` timing fields. 1.0 tok/s end-to-end has to be attributed (prefill vs decode, and KV-cache growth)."' + - 'done_when 4, verbatim: "if CUDA diverges: the fix plus a falsifier that goes RED on the 12k-position corpus"' + phases: [] + subtasks: [] + estimated_effort: null + labels: + - kind:code + notes: 'Routed by the cop (aprender-04) at 2026-09-21 ~15:45Z onto the 0.69.1 path (#3710 bar, context row: "each of the four at the largest prompt the three consumers actually send, derived from their measured traffic, never picked here", with RAH''s review lane measured at ~44 KB, about 12k tokens). The input is paiml-implement-c9''s prompt at /mnt/nvme-raid0/agent-wt/gtmp-318/apr-lane/lane.json.prompt.txt on lambda (44667 bytes, sha256 ed8b3007…). Model /home/noah/models/Qwen3.5-9B-Q4_K_M.gguf, sha256 03b74727…, identical on lambda and gx10. Binary: the 0.69.0 release build, `apr 0.69.0 (225b2a9ab)`, at /mnt/nvme-raid0/targets/ladder-069/release/apr on each host. Measurements are recorded on the issue as they land. DISCRIMINATION RESULT (aprender-c3, 2026-09-21): not a CUDA defect and not model weakness. (a) The gdn_decode_attention multi-pass path (>4096 positions) is correct: a new device test at the default cap, seq_len up to 12,288, passes against the CPU port. (b) RoPE is ruled out: rope.dimension_sections [11,11,10,0] make IMROPE equal NeoX for text. (c) On arbiter''s r891 prompt, apr CPU and apr CUDA BOTH lose to the pinned llama.cpp at copy positions from about 250 tokens (pos 252: llama.cpp 17114 `maps` at logprob -0.001, apr 2117 `map`). (d) The cause both share is the tokenizer: apr''s GGUF encoder is greedy longest-match, so the prompt reaches the model as ids it never saw. That is #3726, which carries the fix. This row closes as #3726 if the re-measure after the fix shows no corruption; any remainder stays here. The 1.0 tok/s is serial prefill (qwen35 CUDA has no batched prefill), owned by #3596.' +- id: PMAT-3694 + github_issue: 3694 + item_type: task + title: 'parity_receipt_denominator.sh needs a python-free classify() — then drop its UNMEASURED path' + status: in_progress + priority: high + assigned_to: aprender-0e + created: 2026-09-21T14:52:00Z + updated: 2026-09-21T14:52:00Z + spec: null + acceptance_criteria: + - 'Issue #3694 done_when 1, verbatim: "`parity_receipt_denominator.sh` classifies record / legacy / other with no python3 on PATH, by a real parse, and its self-test runs every row on a python-free runner."' + - 'done_when 2, verbatim: "The exit-3 UNMEASURED path and `ont4c3_parity_receipts`''s acceptance of exit 3 are removed, so a runner without the tool can no longer pass the test unmeasured."' + - 'Issue body, verbatim: "The predicate stays an independent implementation, deliberately not the Rust extractor''s, and it must still PARSE the JSON rather than substring-match it (a substring test standing in for a parse is refused). One option is a tiny parse in a tool the runners do carry."' + - 'Cop dispatch (aprender-04 to aprender-0e, 2026-09-21), verbatim: "done_when: the real corpus still gives `PASS 7 receipt(s)`; a stub PATH with NO python3 gives the same PASS; the git-refusal row (#3682) is unchanged; a mutant goes RED. No jq either unless you prove the runner has it (clean-rooms have none, and yoga''s is jq 1.6)."' + phases: [] + subtasks: [] + estimated_effort: null + labels: + - kind:code + notes: 'Follow-up to #3695 (the stop-gap that made a python-free runner UNMEASURED exit 3) under the cop ruling on #3689. SCOPE: scripts/parity_receipt_denominator.sh, crates/aprender-contracts-cli/tests/ont4c3_parity_receipts.rs, and this fragment. HOW IT IS MET. classify() is POSIX awk, run under LC_ALL=C: a strict RFC 8259 parser, iterative (an explicit container stack, so no recursion limit), that validates UTF-8 inside strings (no overlongs, no surrogates, nothing past U+10FFFF) and keeps only the ROOT object''s keys, each value''s kind and the decoded text of a string value. The rule is unchanged: record iff the root is an object whose schema is the string apr-parity-receipt/v2; legacy iff not, and metrics is an array or parity is present; anything else, including any syntax error, is other. A later duplicate key wins, as in json.load and serde_json. A raw NUL byte is decided in bash (other), because busybox awk holds C strings and would truncate the document. It is still an independent implementation: no code shared with extract:parity-receipt. The one deliberate difference from the python it replaces: NaN / Infinity are not JSON (RFC 8259), and serde_json refuses them too, so they are other. REMOVED: PY_BIN / PARITY_PYTHON, the command -v check and its UNMEASURED return 3, the --self-test UNMEASURED early exit, and the Rust test''s exit-3 acceptance. There is no exit 3 any more. A missing awk, or one that answers nothing, is ENV exit 2, RED (seam PARITY_AWK). SELF-TEST, 9 rows (was 7): the 5 original rows unchanged, including the #3682 git-refusal row; NEW "a PATH with no python3 (only git awk sort grep head tr wc) measures the same PASS", which also lists the script''s dependencies; NEW "a missing classifier is ENV exit 2 naming it, never a count of 0 or UNMEASURED", replacing the #3695 no-interpreter row; "a classifier that answers nothing is ENV exit 2" (#3695, kept); NEW a 35-row classification table (escaped slash, a \u-escaped key, quotes and brackets inside strings, schema nested or only in prose, a root array, a truncated document, trailing content, a trailing comma, single quotes, NaN, a leading zero, a raw tab, a bad escape, latin-1, a UTF-8 surrogate, a NUL, a BOM, duplicate keys in both orders, and legacy metrics[] / parity / an escaped metrics key). MEASURED at this commit. `bash scripts/parity_receipt_denominator.sh` rc 0 "PASS 7 receipt(s) under evidence/parity/**, and evidence/parity/EXPECTED_RECEIPTS says 7." --self-test rc 0, 9 ok, under gawk 5.1, mawk 1.3.4-20200120 and busybox awk (PARITY_AWK). THE YOGA SHAPE, a PATH of every /usr/bin and /bin executable except python* (2832 tools, command -v python3 finds nothing): the real corpus gives the same rc 0 PASS 7, --print gives 7, and --self-test gives 9 ok. THE RUNNER IMAGE: sovereign-ci is FROM rust:1.95-slim-bookworm (infra machines/intel/sovereign-ci/Dockerfile). In debian:bookworm-slim plus git (mawk 1.3.4.20200120-3.1, python3 NONE, jq NONE) the self-test gives 9 ok and the real corpus gives PASS 7 rc 0; in rust:1.95-slim (trixie, mawk 1.3.4.20250131-1, python3 NONE, jq NONE) the same. awk is Priority: required in Debian, and 52 of 131 scripts/check_*.sh already run it on every runner. DIFFERENTIAL against the old python classify(), over all 118 tracked files under evidence/parity/** plus 55 adversarial documents including a 3000-deep nesting: identical under gawk and mawk except NaN and -Infinity (the deliberate difference above); 7 records in both. The 35 table rows agree with the python rule on 34; the 35th is the NaN row. MUTANTS, each run against --self-test, all RED: a substring classifier (grep for the schema) gives rc 1 with 23 FAIL rows; the python classify() restored gives rc 1 ("a python-free PATH gave exit 2 … python3: command not found"); the first duplicate key wins gives rc 1 on both duplicate rows; UTF-8 validation removed gives rc 1 (latin-1, surrogate); trailing content accepted gives rc 1; the schema-kind check removed gives rc 1 ({"schema":"","schema":2} -> record); the silent-classifier arm restored to fall through as other (#3695) gives rc 1, "the tree holds 0"; the NUL pre-check removed gives rc 1 under busybox awk and rc 0 under gawk, so that guard is load-bearing only for C-string awks. THE RUST TEST: `cargo test -p aprender-contracts-cli --test ont4c3_parity_receipts` 9 passed with python3 on PATH, and 9 passed on the yoga-shape PATH plus ~/.cargo/bin (no python3). PARITY_AWK=/bin/true fails it, "exit Some(2)" plus the ENV line. MUTANT: the pre-fix script (225b2a9ab) on the python-free PATH fails the new test with "exit Some(3)" and the UNMEASURED line, so the exit-3 acceptance is gone. bashrs lint 0 errors; check_shell_lint_ratchet.sh PASS (ratcheted); cargo fmt --all -- --check rc 0. Not in scope: #3697 (the other guards that need python3), which is the next row in this lane.' - id: PMAT-3695 github_issue: 3695 item_type: task @@ -20124,6 +20245,67 @@ roadmap: labels: - kind:code notes: 'Found by aprender-7a seat-filling #3657; the cop ruled it a small 0.70 row stacked on #3657''s 6fd241645, which is not touched (this commit is its child). SCOPE: scripts/release/publish_strict.sh (4 lines before the mapfile) and this fragment. HOW IT IS MET: awk lists every line of publish-order.txt that is empty or whitespace-only, by number; any hit stops with `die "blank line in publish-order.txt: an empty crate name is not a crate"` before ORDER is built, so no empty string reaches the set checks or bash''s associative-array subscript. publish_strict.sh has no --self-test on purpose (one would make it a guard that must be workflow-invoked), so the proof is end to end. MEASURED in a private clone (its own refs; tag v9.9.9 in the clone only; RELEASE_AP in scratch; --plan reads the sparse index and uploads nothing). BEFORE, at 6fd241645, a blank line at line 36: rc 1, "scripts/release/publish_strict.sh: line 47: EXPECT: bad array subscript" and "STOP publish: in the publish order but not in the universe: " (an empty name). AFTER, at this commit: the clean file gives rc 0 and 74 rows; a blank line at 36 gives rc 1, "STOP publish: blank line 36 in publish-order.txt: an empty crate name is not a crate", with 0 subscript errors; a whitespace-only line at 3 plus a blank line (now at 37) gives rc 1, "STOP publish: blank line 3, 37 in publish-order.txt: an empty crate name is not a crate". `bash scripts/check_release_scripts_derive_identity.sh`: PASS (no literal count, no identity literal).' +- id: PMAT-3697 + github_issue: 3697 + item_type: task + title: 'guard-tree python dependence, batch 1 of #3697: 4 guards python-free or UNMEASURED (t2_preflight_verdict, no_hosted_runners, nextest_ci_profile_no_fail_fast, roadmap_ids_unique) + scripts/lib/python_fleet_state.sh' + status: in_progress + priority: high + assigned_to: aprender-0e + created: 2026-09-21T15:20:00Z + updated: 2026-09-21T16:40:00Z + spec: null + acceptance_criteria: + - 'Issue #3697, verbatim: "**What lands (0.70):** per guard, one of: - (a) remove the python dependency (the root fix, the direction of #3694: no python in automation). For YAML reads, prefer `pv`/bashrs/Rust readers the repo already builds. - (b) where a python reader stays, classify its absence as fleet state: `UNMEASURED … reason=`, exit 0, surfaced by guard_tree (#3651), never red. That is the ruling applied to fleet-pv (#3669''s andon) and to pathonly (#3692)."' + - 'Issue #3697, verbatim: "My own `check_nextest_ci_profile_no_fail_fast.sh` (#3626) is on the no-python3 list. It reads TOML through a purpose-built fallback on 3.10, but with no interpreter at all it reports ENV rc 2, which guard_tree counts as FAIL: same class, (b) applies."' + - 'Cop dispatch (aprender-04 to aprender-0e, 2026-09-21), verbatim: "Each guard becomes python-free, or UNMEASURED per the #3692 ruling. One PR per 3–4 guards so the reviews stay small." and "same proof shape (no-python3 PATH + debian:bookworm-slim)".' + - 'NOT in this batch (it lands with the last batch, when it can be green): the issue''s "one guard-tree row: `guard_tree.sh --no-cargo` under a `python3 -S` stub and a no-python3 stub. It must end `0 failed`". Today 18 other guards still fail it.' + phases: [] + subtasks: [] + estimated_effort: null + labels: + - kind:code + notes: 'SCOPE: scripts/lib/python_fleet_state.sh (new, sourced), scripts/check_t2_preflight_verdict.sh, scripts/check_no_hosted_runners.sh, scripts/check_nextest_ci_profile_no_fail_fast.sh, scripts/check_roadmap_ids_unique.sh, and this fragment. The rest of #3697 (18 guards, and the stub-shaped guard-tree row) is later batches. (a) PYTHON-FREE. t2_preflight_verdict: the self-test''s mutant splice (replace decide() with the pre-#3561 absence-as-consent body) is awk over a quoted-heredoc mutant, cut from the `decide() {` line through the first line that is exactly `}`; a subject with no such close is ENV. The spliced file is BYTE-IDENTICAL to the python splice''s output under gawk 5.1, mawk 1.3.4 and busybox awk. no_hosted_runners: the scan is awk; POSIX ERE has no \b, so the token''s boundaries are spelled `(^|[^A-Za-z0-9_])…([^A-Za-z0-9_]|$)`. DIFFERENTIAL vs the python scan (cwd-relative paths, same baseline semantics, sorted stale list): identical stdout+rc on the real 21 workflows (with and without a baseline), on a 611-violation adversarial corpus (every token shape x 11 prefixes x 12 suffixes), and on two baseline corpora, under all three awks, EXCEPT a token touching a non-ASCII letter (e.g. ubuntu-latesté): python (Unicode \w) passed it, awk (ASCII word chars) flags it. 0 violations lost, 111 added, all with é. Rows 13-15 pin both boundaries and the ASCII choice (rows 13/14 agree with python; row 15 is the deliberate difference). (b) UNMEASURED. scripts/lib/python_fleet_state.sh: py_fleet_state [module...] probes the interpreter (PY_FLEET_PYTHON, default python3) by importing each module; only ModuleNotFoundError is `not installed`. rc 0 ok; rc 3 fleet state with ONE stderr line `UNMEASURED runner= reason=no-interpreter interpreter= guard=` (absent, or exec 126/127) or `reason=no-module module=`; rc 2 ENV for an interpreter that exists and dies or answers nothing. Reason names: the tree already says `no-interpreter` (#3692 pathonly, #3695 denominator), so that is kept over the issue''s example `no-python`, and `no-module module=yaml` names the module rather than coining one reason per module. Sourced, option-neutral (check_sourced_libs_option_neutral.sh OK: 9 libraries). Its case table py_fleet_state_self_test (5 stub rows + 2 real-interpreter rows that print UNMEASURED where there is none) runs inside each consuming guard''s --self-test, because a scripts/tests file would be dark without a workflow edit. nextest_ci_profile_no_fail_fast: judge() probes first; no interpreter -> rc 3 -> the run exits 0 with the UNMEASURED line and no PASS line; /bin/false still ENV rc 2; the old row `no interpreter at all -> ENV rc=2` is now `-> UNMEASURED rc=3` plus a run-level row; the reader table (both readers) is UNMEASURED where python is absent. roadmap_ids_unique: scan() needs python3 + PyYAML; either missing -> UNMEASURED exit 0; a dying python3 -> ENV; the 5 YAML rows are UNMEASURED where there is no reader; the missing-file row and two new fleet-state rows run everywhere. MEASURED, guard_tree.sh --no-cargo on 225b2a9ab + this batch, each shape a python3 stub first on PATH: normal python3 76 checks, 0 failed (unchanged); python3.10 -S 14 -> 12 FAIL rows (roadmap_ids_unique s+r gone); no python3 (stub exits 127) 29 -> 22 FAIL rows (all 7 rows of these 4 guards gone), and under PASS guard_tree surfaces the UNMEASURED lines (#3651). debian:bookworm-slim (sovereign-ci''s FROM base) + git, python3 NONE, jq NONE, mawk 1.3.4.20200120-3.1: t2_preflight_verdict self-test rc 0 (2 ok) run rc 0; no_hosted_runners self-test rc 0 (15 ok) run rc 0 `OK: 21 workflow files`; nextest self-test rc 0 (9 ok, 3 UNMEASURED) run rc 0; roadmap_ids_unique self-test rc 0 (8 ok, 3 UNMEASURED) run rc 0 with the UNMEASURED line. MUTANTS, each against a scratch copy, all RED: py_fleet_state mapping every failure to UNMEASURED -> roadmap_ids_unique self-test rc 1 (`a python3 that dies -> ENV exit 2, never UNMEASURED`, plus the helper''s crash rows); no_hosted_runners leading boundary dropped -> rc 1 (row 13 xubuntu-latest); t2 splice made a no-op -> rc 1 (`the pre-#3561 mutant PASSED the case table`). bashrs: check_shell_lint_ratchet.sh PASS (errors only may not rise).' +- id: PMAT-3699 + github_issue: 3699 + item_type: task + title: 'prepare_bump.sh --ship opens a bump PR whose body fails §6 R-2 (check_pr_closes_issue): the CHANGELOG''s context refs (#EPIC, ''refs #N'') have no keep-open line — hit on 0.68.2 (#3498) and 0.69.0 (#3698)' + status: in_progress + priority: high + assigned_to: aprender-f0 + created: 2026-09-21T13:03:24Z + updated: 2026-09-21T15:20:00Z + spec: null + acceptance_criteria: + - 'Issue #3699 done_when 1 AS AMENDED on the issue by its author (comment 2026-09-21T15:17:48Z, "done_when 1 amended (issue author, cop, 2026-09-21), replacing the original wording"), verbatim: "`prepare_bump.sh --ship` derives every `#N` the §6 R-2 guard owes a reason for (`check_pr_closes_issue.sh --list-owed`: not under a closing keyword, not proven a PR or a closed issue; an unresolvable ref counts) and writes ONE `keep-open:` line naming them (\"cited by the CHANGELOG for context; each closes via its own PR; the release EPIC closes at T-4\")"' + - 'done_when 2, verbatim: "before `gh pr create` it runs `scripts/check_pr_closes_issue.sh --body ` and refuses to open the PR on rc != 0"' + - 'done_when 3, verbatim: "a case-table row: a CHANGELOG with an EPIC ref and a `refs #N` produces a body the guard PASSes; a mutant that drops the keep-open line goes RED"' + phases: [] + subtasks: [] + estimated_effort: null + labels: + - kind:code + notes: 'Assigned by the cop (aprender-04) as the first item of the release-tooling lane, to land before the 0.70 bump. Fixtures and stubs only: nothing here ran against the live 0.69 train, its state dir, or a publish token. SCOPE: scripts/release/prepare_bump.sh, scripts/check_pr_closes_issue.sh, the new scripts/check_release_bump_pr_body.sh, and this fragment. HOW IT IS MET. (1) --ship builds the body BEFORE the cargo checks, the commit and the push (it used to be built after the push). It asks the guard for the refs that owe a reason through a new `check_pr_closes_issue.sh --list-owed --body FILE`, which prints the output of the SAME classify_refs() the judge uses (refactored out of check_body_text, behaviour unchanged), and appends ONE line `keep-open: #A #B -- cited by the CHANGELOG for context; each closes via its own PR; the release EPIC closes at T-4`, whose reason is done_when 1''s parenthetical, verbatim. The epic-and-refs row pins the whole line. When nothing is owed there is no line. WHICH REFS (done_when 1 as amended; the ORIGINAL read "derives every `#N` in the generated body that isn''t under a closing keyword", and quorum round 2''s lane 1 FAILed on that literal reading, which is why the issue author amended it): the guard''s own set, so references PROVEN to be pull requests or closed issues are left out, exactly as the judge leaves them out, and an unresolvable ref is kept (fails closed). Measured on the real 0.69.0 body: 61 distinct #N, of which the guard owes 5 (#3080 #3545 #3571 #3602 #3658), which is the same 5 the CI step flagged and aprender-62 named by hand. Naming all 61 would list merged PRs in a keep-open line. THE REASON TEXT is the ticket''s own. Quorum round 1 FAILed an earlier wording that described the close MECHANISM instead (lane 1, gemini-3.1-pro-high: the reason must be done_when 1''s text), and the ticket''s text wins. Its last clause is a statement about WHEN, and it holds today: the epic is closed at T-4, by hand (0.68.2''s #3477, closer=null, 2026-09-20T08:23Z; the cop is closing 0.69.0''s #3080 right after the crates.io publish). #3708, same owner and queued before the 0.70 bump that first emits this line, makes the close step do it mechanically: autopilot.sh close (lines 301-307) comments on the epic, then requires 0 open milestone items, and closes nothing. The reason carries no `#`, so the line cannot add a closing reference (a line that closed the epic on the bump''s merge would be the #3400 landmine). (2) `bash check_pr_closes_issue.sh --body pr_body.md || die "the bump PR body fails §6 R-2; nothing committed, pushed or opened"` runs before the cargo checks, the commit, the push and `gh pr create`. (3) scripts/check_release_bump_pr_body.sh is a scripts/check_*.sh, so guard_tree.sh runs it bare in guard-tree with no workflow edit (the check_t2_preflight_verdict.sh pattern). It runs the REAL prepare_bump.sh --ship end to end in a throwaway fixture: a local bare origin, a bump clone on release-9.9.9 with the CHANGELOG section uncommitted, a T-2 GO receipt, stubs for gh (records calls, captures --body-file), cargo and the arm helper, and ref kinds from the guard''s PR_CLOSES_REF_KIND_CMD seam. ROWS: guard-case-table (runs check_pr_closes_issue.sh --self-test, see DARK TABLE below); epic-and-refs (EPIC #9002 + `refs #9005` + PR #9001 + `closes #9006` -> PR opened, body PASSes the guard, ONE keep-open line naming exactly #9002 #9005, not the PR and not the closed-by-keyword ref); no-keep-open (that body minus its keep-open line -> guard rc 1); adds-no-close (the keep-open line alone -> --list-owed lists every ref it names); prs-only (no line, still PASS); landmine (`no-close: #9005` -> non-zero, no `gh pr create`, no branch on origin). COMMITTED MUTANTS, run on every bare run: drop-keep-open (the printf line removed) is killed by epic-and-refs, "exited 1: STOP the bump PR body fails §6 R-2"; drop-refusal is killed by landmine, "exited 0 on a landmine body". A missing subject or mutation anchor is ENV rc 2, never a pass. The guard table gains list-owed (#9001 pr, #9002 open, #9003 unresolvable, #9004 closed, #9005 under Closes -> exactly "#9002 #9003") and list-owed-empty-body (rc 2). MEASURED at this commit: `bash scripts/check_release_bump_pr_body.sh` rc 0, "PASS 8 row(s)"; `bash scripts/check_pr_closes_issue.sh` rc 0, "self-test OK: 21 case(s)". The first run of the new table caught a real bug in this change: the keep-open line read "#9005-- cited", because $(...) strips the trailing newline. LIVE REPLAY: the real CHANGELOG [0.69.0] section through the real prepare_bump.sh --ship, with gh `api` passed to the real gh and only `pr create` stubbed. rc 0, 122 api calls (two passes over 61 refs), 1 pr create. Keep-open line "keep-open: #3080 #3545 #3571 #3602 #3658 -- cited by the CHANGELOG for context; each closes via its own PR; the release EPIC closes at T-4". The unmodified guard with LIVE resolution on the opened body: "PASS: non-closing ref(s) #3080 #3545 #3571 #3602 #3658 allowed by a keep-open reason." Zero closing references in the body. AD HOC MUTANTS, not committed, each RED: a reason ending "the release EPIC closes #9002" turns 2 rows RED (epic-and-refs, adds-no-close: "names '#9002 #9005 #9002 ' but only '#9005 ' are still owed"); naming every #N instead of --list-owed turns 3 rows RED (epic-and-refs, adds-no-close, prs-only); moving the refusal after the push turns landmine RED, "the branch was pushed before the refusal"; a --list-owed that prints every ref turns 3 rows of the WIRED table RED (guard-case-table, epic-and-refs naming #9006, prs-only). DARK TABLE, pre-existing and found here: `bash scripts/guard_tree.sh --no-cargo` prints `skipped: scripts/check_pr_closes_issue.sh -- wired-with-args in ci.yml`, and ci.yml line 1398 runs that guard only with --body. So its case table ran in NO CI job, even though the ci.yml comment at lines 1368-1373 still says guard_tree runs it bare. Its new list-owed rows would have been dark too. The guard-case-table row gives that table a caller inside guard-tree with no workflow edit. The stale ci.yml comment is left for a workflow-authorized change. GUARD_TREE: `bash scripts/guard_tree.sh --no-cargo` at fc5fdee22 (7 rows then): "77 checks, 0 failed", with "PASS scripts/check_release_bump_pr_body.sh [run]". OUT OF SCOPE, observed: a CHANGELOG bullet whose PR title carries a closing keyword ("fix: closes #N") would still close #N when the bump merges. The guard PASSes that, correctly by its own rule, and nothing here changes it.' +- id: PMAT-3703 + github_issue: 3703 + item_type: task + title: 'aprender-verify-ml test_pipeline_generate_small asserts wall-clock (generation_time_ms > 0) — failed v0.69.0''s clean-room B2 on a fast gx10 runner' + status: in_progress + priority: critical + assigned_to: aprender-fd + created: 2026-09-21T14:20:00Z + updated: 2026-09-21T15:22:00Z + spec: null + acceptance_criteria: + - 'Issue #3703 done_when 1, verbatim: "the assertion tests what the code promises (the stat is populated and monotone), never elapsed wall time. E.g. drop the timing clause, or record µs and assert `generation_time_us` is set, not `> 0`"' + - 'done_when 2, verbatim: "a sweep of every test in the crates clean-room B2 runs (`--lib` of the published crates) for `as_millis() > 0` / `elapsed ... > ` / `Instant`-based assertions, with each hit fixed or listed with its reason. `scripts/check_no_timing_in_required.sh` extended to see this form (it passed while this sat in a required gate)"' + - 'done_when 3, verbatim: "mutation proof: a test build with an instantaneous generator must still PASS"' + phases: [] + subtasks: [] + estimated_effort: null + labels: + - kind:code + notes: 'Handed to aprender-62 by the cop after infra-c8 was redirected by the operator (#3703 was unstarted). #3702, filed a minute earlier for the same defect, was closed as a duplicate. THE FAILURE: v0.69.0 clean-room (paiml/infra run 35608872676, gx10-pool1, "Run clean-room gates") GATE B2: aprender-verify-ml data::pipeline::tests::test_pipeline_generate_small, "assertion failed: stats.generation_time_ms > 0 || stats.total_generated < 10". `generation_time_ms` is `elapsed.as_millis() as u64` and the test asks for count(10), so a host that generates 10 programs in under 1 ms reads 0. It did not reproduce on lambda''s debug build (0/40 runs failed there), which is why PR CI and the merge group were green. DONE_WHEN 1: the timing clause is replaced by what generate() promises, `assert_eq!(stats.total_generated, programs.len())`, which holds by construction (`total_generated: all_programs.len()`). No clock is asserted. DONE_WHEN 3, measured: with the generator made instantaneous (`generation_time_ms` forced to 0), the fixed test PASSES ("1 passed"), and the OLD clause under the same mutant FAILS with exactly the clean-room message, "assertion failed: stats.generation_time_ms > 0 || stats.total_generated < 10". So #3703 reproduces deterministically on any host. DONE_WHEN 2, the sweep. Scope: every `crates/*/src/**.rs` (the `--lib` surface). The #3703 shape is an INTEGER duration (`.as_millis()`, `.as_micros()`, `.as_secs()`, or a name ending _ms/_us/_secs) asserted `> 0`, `>= 1` or `!= 0`, which truncates to 0 on a fast host. A float `> 0.0` on `as_secs_f64()` does not truncate. 104 hits. FIXED, 3, each a measured duration with nothing guaranteeing it is positive: pipeline.rs:438 (above), aprender-orchestrate bug_hunter/tests_patterns.rs `hunt_result.duration_ms > 0` (dropped; result.is_ok() and the parsed claims already prove the hunt ran) and aprender-train finetune/instruct_trainer.rs `result.total_time_ms > 0` (dropped; the epoch count is asserted). LISTED WITH ITS REASON, 101, in the new scripts/wallclock_assert_baseline.txt, each line read and classified: 76 declared render budgets (`budget()` returns BrickBudget::uniform(16) and similar; constants), 9 configured values (timeouts, windows, intervals, chunk sizes, options), 7 epoch timestamps (`timestamp_ms` from SystemTime), 1 cost-model estimate, 4 fixture inputs (health_tests Duration::from_millis(100), process_tracer from_micros(100), stats u64::MAX/2, format_chat latency_us=5000), and 4 real measurements taken AFTER a sleep of at least the truncation unit (cache/tests.rs sleeps 2 ms; falsification/types.rs sleeps 1 ms; brick_tracer sleeps 10 us and 100 us). THE GUARD: check_no_timing_in_required.sh PART 4 scans with that pattern (comment lines excluded by the same matcher) and ships a 15-row case table (7 must-match, including the exact pipeline line; 8 must-not-match, including `> 0.0`, `as_secs_f64() > 0.0`, `>= 10`, a comment, and a name that only contains `_ms`). A pattern that disagrees with its table is refused before any scan result is trusted. THE LEDGER IS KEYED ON THE ASSERTION TEXT, not on `path:line`: each line is ``, so the 101 hits are 32 recorded texts. A coordinate key was the first version (handed over at 0221c7c1b) and was measured to be a trap. With the ledger at a comparand (BASELINE_RATCHET_BASE_REF=HEAD), one unrelated comment line inserted at the top of widgets/border.rs made PART 4 report border.rs:755 NEW and :754 stale. Re-pointing the entry, which is the remedy it printed, was then REFUSED by the set-aperture ratchet ("the owning guard is unchanged in this diff"), so any unrelated edit above any of the 101 lines in ~50 files would have turned guard-tree red with no in-branch fix. The ratchet is now `keyed` (classified in check_baseline_ratchets.sh): no text may appear and no count may rise. PART 4 fails on NEW (a text the tree holds more often than recorded, so a COPY of a recorded line fails too), on STALE (a recorded count above the tree''s: lower it in the commit that fixed the line, which the ratchet allows), on a malformed ledger line, and on a scan of 0 files. GUARD MUTANTS, measured on this tree with BASELINE_RATCHET_BASE_REF=HEAD: M0 unmutated rc 0; M1 the #3703 line restored rc 1 (NEW, pipeline.rs:442); M2 the trap above, an unrelated insertion over a recorded line, rc 0; M3 a copy of `assert!(budget.total_ms > 0);` added in a new place rc 1 (count rises); M4 every data line removed from the ledger with the hits still in the tree rc 1; M5 a recorded count raised rc 1 (stale, and the ratchet reports RAISED); M6 a new key appended rc 1 (NEW KEY); M7 a malformed ledger line rc 1; M8 the `_ms` alternative dropped from the pattern rc 1 (4 case-table rows flip, scan not trusted); M9 the scan made to answer 0 files rc 1; M10 M4 under `NR == FNR` instead of `FILENAME == ARGV[1]` rc 0. M10 is the proof that the awk file test is load-bearing: with an emptied ledger `NR == FNR` reads every hit as a recorded entry and passes them all, so the guard uses FILENAME. ONE MORE HIT, OUTSIDE THE INTEGER SHAPE AND FIXED: aprender-train monitor/inference/tests/inference_monitor_tests.rs test_monotonic_ns_non_zero asserted `monotonic_ns() > 0` where monotonic_ns() is `START.get_or_init(Instant::now).elapsed().as_nanos()`. When that test is the first caller in the process, the call initializes the epoch and reads the gap between two back-to-back clock reads, which is 0 on a coarse timer. Its sleep ran BEFORE that first call, so it guarded nothing. The test now initializes, sleeps 1 us, and asserts the second read is greater than the first; the sleep, not host speed, carries the assertion. bashrs on the guard: 0 errors (the shell lint ratchet counts errors). The added warnings are the parser reading awk array syntax, `$''\t''` and `[![:space:]]` as shell. SCOPE AMENDMENT, the issue author (cop) on #3703, 2026-09-21T14:49:28Z, verbatim: "done_when 2''s sweep is satisfied by *listing* each remaining hit with its reason. The slow-host `elapsed < N` upper-bound class is the named remainder, tracked in **#3705**, and #3703''s ledger may list those hits with reason `#3705`. The PR for #3703 may therefore say `Closes #3703` + `Refs #3705`." THE LISTING: docs/audits/pmat-3703-timing-sweep.md, a snapshot at 8c08dd8dd with the commands that regenerate it (they print 58 + 45 = 103 lines). 58 upper bounds carry reason `#3705`. The 45 other lower bounds on a duration each carry a reason: 10 are sleep-guarded, 8 are tautologies, 8 are not clock readings, 17 are nanosecond or float readings over real work (0 only inside one clock tick; the closest is trueno-ublk adaptive_batch.rs:494, which times two add_items calls), 1 is a hidden upper bound routed to #3705 (tuner_tests_callbacks.rs:191, `remaining >= 999 s`), and 1 is `#[ignore]`d. It is listed rather than put in the PART 4 ledger because the ledger''s data lines are ENFORCED by text and count, and a coordinate list there would read as enforced while drifting. OUT OF THIS ROW, and stated rather than stretched: the SLOW-host twin, an upper bound on elapsed time (`elapsed < N`), with 59 candidate lines in the same sweep, is filed as #3705 (0.70). done_when 2''s "`Instant`-based assertions" reads that class in; this row covers the fast-host truncation class that failed the release, and a reviewer should judge #3705 as the named remainder.' - id: PMAT-3704 github_issue: 3704 item_type: task @@ -20145,3 +20327,249 @@ roadmap: labels: - kind:code notes: "Refs #3577 (ONT-4c3; its ledger binding is infra's row, bound after this merges) and #3624 (the by_entity_type sibling, not closed here). R-3 of ONT-001: pc_extract is one planted defect per registered extractor, every run." +- id: PMAT-3708 + github_issue: 3708 + item_type: task + title: 'autopilot''s INHERITED dogfood GO writes no receipt that check_publish_preflight.sh R5 accepts — v0.69.0 stopped at the publish preflight 43 min after tagging' + status: in_progress + priority: critical + assigned_to: aprender-f0 + created: 2026-09-21T14:42:13Z + updated: 2026-09-21T15:45:00Z + spec: null + acceptance_criteria: + - 'Issue #3708 done_when 1, verbatim: "ONE rule, enforced at both ends. EITHER autopilot never inherits and runs the real dogfood at T-1 in `$AP/wt` (pre-tag, where a NO-GO costs no public tag), OR R5 accepts the inherited receipt under exactly the conditions autopilot checked (parent GO receipt present, bump diff = version surface only, parent is the release commit''s first parent), and both read the same file. Pick one; don''t keep both."' + - 'done_when 2 AS AMENDED on the issue by its author (comment 2026-09-21T15:37:51Z, "done_when 2 amended (issue author, cop, 2026-09-21), replacing the original, which assumed the inherit option. The option picked is: NEVER inherit."), verbatim: "a case table, wired as a scripts/check_*.sh so guard-tree runs it (the check_t2_preflight_verdict.sh pattern), that drives the real scripts/release/autopilot.sh dogfood step with stubs: a parent GO receipt plus a version-only bump diff still runs dogfood.sh (never inherits); a receipt R5 refuses (NO-GO, other commit, other version, absent) STOPs at T-1 before any tag; a GO receipt passes R5 at T-1, and `check_publish_preflight.sh` R5 at T-4 reads the same receipt file with the same function. Plus the close step: the epic as the milestone''s only open item is closed, then the milestone; any other open item refuses both. A mutant for each."' + - 'done_when 3, verbatim (measured at the NEXT train; this PR says Refs #3708, not Closes): "measured on the 0.70 train: no STOP between TAGGED and CASCADE for this reason"' + - 'The issue body''s close-step finding, verbatim: "Also found the same day (aprender-f0): the `close` step requires the milestone to have 0 open items, but nothing closes the release EPIC, so `close` dies on its own epic. 0.68.2''s was closed by hand. Same fix owner."' + - 'Obligation ADDED on the issue by its author (comment 2026-09-21T15:24:47Z, "Obligation added (issue author, cop, 2026-09-21)"), verbatim: "`prepare_bump.sh --ship` refuses to open the bump PR unless per-host model-ladder receipts for the NEW version are committed on the bump branch: `evidence/dogfood/models//{lambda,gx10}.json`, written by `scripts/model_ladder.sh` with an apr built from the bump tree. 0.68.2 did this by hand (#3498); 0.69.0 missed it, and that made `check_model_ladder` unmeasurable until after the tag. The script may REQUIRE the receipts without producing them; gx10''s is measured over the operator-authorized lambda→gx10 SSH. Case row: missing gx10.json refuses, naming the file; mutant."' + phases: [] + subtasks: [] + estimated_effort: null + labels: + - kind:code + notes: 'P0 on the 0.69.1 critical path (cop, aprender-04). Fixtures and stubs only: nothing ran against the live 0.69 train, its state dir, GitHub writes or a publish token. STACKED on PMAT-3699 (branch PMAT-3699-bump-pr-body-keep-open, 0cb1e4bd6), because both change scripts/release/prepare_bump.sh. The judged diff is base 0cb1e4bd6 to this head. SCOPE: scripts/release/autopilot.sh, scripts/check_publish_preflight.sh, scripts/release/prepare_bump.sh, scripts/check_release_bump_pr_body.sh (extended), the new scripts/check_release_autopilot_dogfood_close.sh, and this fragment. PICK (done_when 1 says pick one): autopilot NEVER inherits. The parent''s T-2 GO is at the OLD version, so rows keyed on the version (check_model_ladder) were never measured at the release version. And the T-4 recovery ran the same ~35 min dogfood anyway, after the tag was public. The inherit branch is DELETED, not flagged off, so there is no second rule to keep in sync. HOW EACH IS MET. (1) The dogfood step runs `scripts/dogfood.sh --phase pre-publish` in $AP/wt (the release worktree), then `check_publish_preflight.sh --receipt-only`. That is a new mode running R2 plus R5 only (R3 cannot hold before the tag). R5 is now ONE function, rule_r5(), which the full gate calls at T-4 and --receipt-only calls at T-1, both on the newest $AP/wt/.dogfood/receipt-*.json: the same file and the same function. A refusal is `STOP T-1 R5 refused …`, before the tag step. (2) The case table is scripts/check_release_autopilot_dogfood_close.sh. It is a scripts/check_*.sh because guard_tree.sh''s universe is the flat `git ls-files ''scripts/check_*.sh''`, so a table living only in scripts/release/ would run nowhere (the check_t2_preflight_verdict.sh pattern). It drives the REAL autopilot.sh through its own from/to steps (`dogfood dogfood`, `close close`). The fixture: a local bare origin whose main holds a parent (9.9.8) and a bump touching only Cargo.toml and CHANGELOG.md (9.9.9), a T-2 GO receipt for the parent (exactly the shape the old code inherited on), a real copy of check_publish_preflight.sh, and stubs for gh, cargo metadata and dogfood.sh. ROWS: never-inherits (dogfood.sh RUNS, no dogfood-inherited.receipt, "DOGFOOD GO at (R5 holds at T-1)"); t4-agrees-go (after tagging, the FULL gate prints "ok R5 dogfood receipt receipt-….json: GO for at 9.9.9" for the receipt T-1 accepted); nogo-stops; t1-r5-version (a receipt for 9.9.8 on a 9.9.9 cut STOPs at T-1, and the T-4 gate''s R5 refuses the same receipt, so the two ends agree on refusal too); t1-r5-commit (a receipt naming the parent STOPs at T-1); t1-r5-absent (the dogfood exits 0 and writes no receipt: STOPs at T-1); close-epic-last; close-other-open. MUTANTS, one per claim ("A mutant for each"), run on every bare run, each killed: inherit (the pre-#3708 step verbatim) -> "dogfood.sh never ran: the parent GO was inherited"; drop-nogo-stop -> "it stopped, but not on the NO-GO" (T-1 R5 caught it instead: defence in depth, but the NO-GO row still names the line); drop-t1-r5, run against each of t1-r5-version/-commit/-absent -> "autopilot passed T-1 with a receipt R5 refuses (version|commit|absent)"; t1-reads-another-file (--receipt-only pointed at a receipt dir the T-4 gate does not read) -> never-inherits "STOP T-1 R5 refused", so "the same file" is load-bearing; t1-skips-rule-r5 (--receipt-only no longer calls rule_r5) -> t1-r5-version passes a 9.9.8 receipt, so "the same function" is load-bearing; drop-epic-close -> "STOP milestone 9.9.9 still has 1 open item(s) after the epic"; drop-others-check -> "the epic was closed while #9010 is open". check_publish_preflight.sh --selftest (run by ci.yml:1978) gains 6 --receipt-only rows: an untagged tree with a GO receipt passes (the full gate refuses that tree on R3); NO-GO, stale commit, other version, absent, and a disallowed deferral each refuse with "FAIL R5". (3) is measured at the next train; it cannot be met by this PR, hence Refs. CLOSE STEP: it lists the milestone''s open items EXCEPT the epic. Any remaining one is a STOP naming them, closing neither. Otherwise it closes the epic if it is OPEN, so a re-run after a failure is idempotent, then requires open_issues == 0 and PATCHes the milestone closed. OBLIGATION: `prepare_bump.sh --ship` runs `bash scripts/check_model_ladder.sh --version "$V"` in the bump tree. That is the same judge the dogfood''s check_model_ladder row runs. Its FAIL lines go to stderr, and it refuses before the cargo checks, the commit, the push and `gh pr create`. It also refuses receipts that are present but gitignored (`git ls-files --others --ignored --exclude-standard -- evidence/dogfood/models/$V`), which `git add -A` would silently leave out. check_release_bump_pr_body.sh gives every fixture green receipts, a real copy of check_model_ladder.sh and a MINIMAL fixture ladder (the two required hosts, lambda and gx10, and one rung), so a rung added to the real ladder cannot break this table. New rows: ladder-missing-gx10 (refuses, naming "evidence/dogfood/models/9.9.9/gx10.json"; no push, no PR) and ladder-ignored. New mutants: drop-ladder (judge replaced by `true`) -> "exited 0 (ladder=no-gx10)"; drop-ignored -> "exited 0 (ladder=ignored)". MEASURED at this commit: `bash scripts/check_release_autopilot_dogfood_close.sh` rc 0, "PASS 17 row(s)" (8 behaviours, 9 mutants); `bash scripts/check_release_bump_pr_body.sh` rc 0, "PASS 12 row(s)"; `bash scripts/check_publish_preflight.sh --selftest` rc 0, "25/25 rows"; `bash scripts/check_model_ladder.sh --version 0.68.2` rc 0 on the real ladder and receipts (the judge the gate calls is live); bashrs lint on the 5 changed scripts: 0 errors each; check_release_scripts_derive_identity.sh PASS. GUARD_TREE (`bash scripts/guard_tree.sh --no-cargo`) at the first cut of this commit: "77 checks, 2 failed". check_guards_are_wired: the new table was UNWIRED, because a comment said "cargo metadata" and guard_tree''s textual `(^|[^a-z_-])cargo ` test moved it out of the --no-cargo set; the table never runs cargo (it stubs the metadata call), so the comment was reworded. check_no_pipe_into_grep_q: two `printf | grep -q` sites in the new table (76 vs ceiling 74), now here-strings. Both guards pass after the fix (grep-q "measured 74", guards_are_wired "PASS (ratcheted)"), and `guard_tree.sh --list --no-cargo` lists the table. Full re-run at 3a0513d35 (11-row table): "78 checks, 0 failed", with "PASS scripts/check_release_autopilot_dogfood_close.sh [run]". After the table grew to 17 rows: check_no_pipe_into_grep_q, check_guards_are_wired, and check_bashrs_gate ("PASS 363 file(s) linted … 0 SEC/DET/IDEM errors") re-run, all rc 0. PyYAML (needed by check_model_ladder.sh) is present in guard-tree: check_roadmap_ids_unique.sh and check_roadmap_completion_is_cited.sh already hard-fail without it, and they are green there.' +- id: PMAT-3711 + github_issue: 3711 + item_type: task + title: 'apr qa golden_output: a GPU golden-generation ERROR is reported as a SKIP (note_gpu_golden_skip) — ''GPU broken'' passes the gate; the Q4_K CUDA rule can''t be proven while it does' + status: in_progress + priority: critical + assigned_to: aprender-0e + created: 2026-09-21T15:55:00Z + updated: 2026-09-21T15:55:00Z + spec: null + acceptance_criteria: + - 'Issue #3711 done_when 1, verbatim: "a GPU golden generation error on a host that HAS a CUDA device and a `--features cuda` build is a **FAIL** naming the error, never a skip. SKIP stays only for \"no GPU on this host / no cuda feature\", and it names which."' + - 'done_when 2, verbatim: "case rows: GPU error gives FAIL; no-device gives SKIP-with-reason; GPU wrong answer gives FAIL; GPU right answer gives PASS. Plus a mutant (route the error back to skip and it goes RED)"' + - 'done_when 3, verbatim: "the Q4_K sweep and `scripts/model_ladder.sh` read the new verdict. Re-measure lambda + gx10 with the 0.69.1 binary, and no model is GREEN whose GPU leg errored"' + - 'Cop ruling (aprender-04 to aprender-0e, 2026-09-21), verbatim: "FOLD the sibling in: `run_golden_output_gate_runtime` saying \"GPU hybrid forward\" on a host with no device is the same gate labelling a run by intent. The pass message must name the backend that actually ran (a measured signal, e.g. the device the dispatch reported), plus a case row for cuda-feature-no-device. Same PR, same receipt."' + phases: [] + subtasks: [] + estimated_effort: null + labels: + - kind:code + notes: 'SCOPE: crates/apr-cli/src/commands/output_verification.rs, golden_output.rs, qa_gguf.rs (one line: golden_gate_for passes cpu_only), and this fragment. THE DEFECT IS WORSE THAN THE TITLE. On main, a CUDA init or GPU generation error in validate_gpu_golden_output went to note_gpu_golden_skip (a verbose-only println) and returned Ok(None); the case then judged only the CPU answer and the gate said PASSED, "3 golden test cases passed", not SKIP. Sibling (cop ruling, folded in): run_golden_output_gate_runtime labelled every cuda build "GPU hybrid forward, #3090" and ignored InferenceResult::used_gpu, and the qwen35 GPU dispatch falls back to the CPU on ANY GPU failure (forward_qwen35::run_qwen35_generate_dispatch), so a broken GPU passed on the CPU''s answer with a GPU label. HOW IT IS MET. GpuGoldenLeg {Passed, WrongAnswer, Errored, NotRun(why)}: judge() maps an Err (CUDA init on device 0, or GPU generation) to Errored, and failure() makes Errored a gate FAIL naming the error ("... a broken GPU, not a skip (#3711): "); WrongAnswer is the existing "GPU output failed (CPU passed)" FAIL. gpu_golden_not_run(cuda_feature, device, gguf) is the ONLY source of a skip and names which: "this build has no cuda feature" / "no CUDA device on this host" / "the dense GPU leg judges GGUF only"; the hybrid fail-safe arm is NotRun naming the runtime rung. The pass message names the leg: "(GPU leg judged on CUDA device 0)" or "(GPU leg SKIPPED: )". Runtime rung: golden_output_runtime returns (text, used_gpu); runtime_golden_backend(used_gpu, gpu_not_run) names the backend the dispatch REPORTED, and fails when the GPU should have served (cuda build + device + an architecture the GPU runs, i.e. not cpu_only) and the dispatch reported CPU. cuda_device_present() is one probe for both rungs. CASE ROWS (crates/apr-cli golden_output_tests, cargo test -p apr-cli --lib golden_output_tests: 14 passed, 10 new): GPU generation error -> FAIL naming CUDA_ERROR_ILLEGAL_ADDRESS; CUDA init error -> FAIL naming CUDA_ERROR_OUT_OF_MEMORY; GPU wrong answer -> FAIL; GPU right answer (plain and with a think block) -> PASS, judged; no device -> SKIP naming it, not a failure; each skip names which and a cuda host+GGUF starts the leg; runtime: cuda build + no device -> labelled CPU, never GPU; runtime GPU fallback on a cuda host -> FAIL; runtime GPU served -> labelled GPU from the dispatch; architecture the GPU declines -> CPU pass saying why. MUTANTS, each RED: the error routed back to skip (Err -> NotRun) fails 2 rows (generation error, init error); Errored mapped to no failure fails the same 2; the runtime fallback labelled by intent ("GPU hybrid forward, #3090") fails runtime_gpu_fallback_on_a_cuda_host_fails_the_gate; a no-device run labelled GPU fails 2 runtime rows. LIVE, RTX 4090 sm_89, apr 0.69.0 built from this branch with --features cuda, apr qa --json --offline with the perf/parity gates skipped (the ladder''s flags): qwen2.5-1.5b-instruct-q4_k_m rc 0 "3 golden test cases passed (GPU leg judged on CUDA device 0)"; the same with CUDA_VISIBLE_DEVICES= rc 0 "... (GPU leg SKIPPED: no CUDA device on this host)" [main said "3 golden test cases passed"]; Qwen3.5-0.8B-Q4_K_M rc 0 "... through the runtime entry point (served by GPU: the dispatch reported used_gpu=true)"; the same with no device rc 0 "served by CPU: the dispatch reported used_gpu=false, and the GPU was not expected (no CUDA device on this host)" [main, same run: "(GPU hybrid forward, #3090)" -- the label-by-intent, measured]; and ONE RUN under GPU memory contention from another session: Qwen3.5-0.8B rc 5 FAIL "the GPU should have served this model ... fell back ... A broken GPU, not a pass (#3711)" with stderr "GPU (CUDA) qwen35 path rejected, falling back to CPU: ... CUDA_ERROR_OUT_OF_MEMORY" -- exactly the run main would have passed as GPU. FOUND, NOT CAUSED HERE: qwen2.5-coder-0.5b-instruct-q4_k_m on the RTX 4090 fails the golden gate on BOTH main and this branch, identically: "GPU output failed (CPU passed): golden_output_gpu: gibberish (fragment \"```\\n\" repeats 3+ times)". Reported to the cop for the 0.69.1 Q4_K CUDA rule. DONE_WHEN 3: scripts/model_ladder.sh (qa_row gate()) and scripts/check_model_ladder.sh both require golden_output passed and not skipped, so an Errored GPU leg (passed:false) turns the rung RED with no ladder change; the red-golden case fixture already pins that path. The re-measure of lambda + gx10 with the 0.69.1 binary is the train''s T-2 step and is NOT done here -- this PR is Refs #3711, not Closes. clippy -p apr-cli --lib -D warnings (default features) rc 0; --features cuda: apr-cli has no clippy findings (the -D warnings run stops in aprender-compute on a pre-existing unused GemmOp import, same on main). cargo fmt --all -- --check rc 0.' +- id: PMAT-3712 + github_issue: 3712 + item_type: task + title: Model gate universe = each host's measured Q4_K inventory; the gate refuses an optional Q4_K rung (the contract's last one flips in the 0.69.1 fold, per the done_when 1 amendment); skip/fallback/rc!=0 RED; cells = verb x thinking x context rung + status: in_progress + priority: critical + assigned_to: null + created: '2026-09-21T00:00:00Z' + updated: '2026-09-21T00:00:00Z' + spec: null + acceptance_criteria: [] + phases: [] + subtasks: [] + estimated_effort: null + labels: + - kind:code + notes: 'ISSUE AMENDMENT (#3712 comment by the issue author, quoted verbatim): "done_when 1 amended (issue author, cop aprender-04, 2026-09-21 16:56Z): the guard that refuses `required: false` on a Q4_K rung (case row + mutant) is met by #3721. The contract''s own `qwen3-8b-q4km` rung flips to `required: true` in the 0.69.1 fold, together with aprender-fd''s qwen3-8b fix and a committed green lambda receipt. Flipping it earlier would turn `pv lint contracts/` red on every PR, because the armed ladder-green shape reads the committed 0.68.2 lambda receipt, in which qwen3-8b is red. This changes only the order of the work, not the bar: 0.69.1 does not tag until qwen3-8b is `required: true` and green on both lambda and gx10." ISSUE AMENDMENT 2 (#3712 comment 5764495191 by the issue author, quoted verbatim; lanes judge the cells scope against it): "done_when 5 added (issue author, cop aprender-04, 2026-09-21 17:12Z). Operator, verbatim, 2026-09-21: "lets ensure this release has MORE than enough tokens for our needs (overdo it), and our verbs are dogfooded, i.e chat with and without thinking, ditto run, ditto code, etc," - [ ] The cells matrix. A cell is (Q4_K model, required host, verb ∈ {run, chat, serve, code}, thinking mode derived from the model''s chat template, context rung). Rungs come from `evidence/release/context-rungs.json` up to the model''s declared `context_length`, which is read from the GGUF. The long rungs are owed by qwen35 (always the full set through the declared length) plus one representative per other arch, named in the ladder contract and echoed in the receipt (issue author''s ruling). - [ ] What each cell must do. Where the cell fits the host''s total GPU memory by the declared arithmetic, it must pass on CUDA. Where it doesn''t fit, it must be an honest pre-load refusal naming the arithmetic, never a CPU fallback (operator "a", #3710). - [ ] Coverage. Every rung must pass on at least one required host. The operator''s sentence is the requirement. The cell definition above is the issue author''s reading of it, so a lane judging scope judges against this comment. This is not unrequested scope." ACCEPTANCE (hand-entered from gh#3712; this row = done_when 1 + 2, the GATE side; cop assignment 2026-09-21: worker B). Issue done_when 1, verbatim: "Universe = measured inventory, not a list. The release gate derives the model set from what''s ON each required host (every `*Q4_K*` GGUF/APR under the declared models dir on lambda AND gx10), unioned with the ladder. A model present on a host but missing from the gate''s run is a FAIL naming it. The ladder has no `required: false` for any Q4_K rung; a guard refuses the key on a Q4_K rung (case row + mutant)." Issue done_when 2, verbatim: "Every (model, host) cell is GREEN on CUDA: capability_match passed (not skipped), golden_output passed (not skipped; #3711), `apr run --gpu` rc 0 with no fallback line. One red cell = NO-GO. There is no known, optional or pre-existing exemption, and no threshold like >= N%." WHAT THIS ROW DOES: (a) contracts/model-capability-ladder-v1.yaml gains `ladder.inventory` {dirs, patterns (case-insensitive *q4_k*/*q4k* .gguf/.apr), backends [cuda]}; (b) scripts/model_ladder.sh measures ladder UNION the host''s inventory (every match, depth 1), judges each inventory model on cuda with the SAME measure() as a rung, and writes receipt schema apr-model-ladder-receipt/v2 carrying `inventory` [{file, sha256, bytes}], `file` per row, and `apr_sha` (full 40-hex, for #3715); (c) scripts/check_model_ladder.sh FAILs: `required` not true on a Q4_K rung; a Q4_K rung that does not claim cuda; a ladder with no inventory; a receipt that is not v2 or has no inventory; an EMPTY inventory; an inventory file with no present row (MISSING from the run, named); an inventory-only model not green on cuda (skip, fallback, rc != 0 are RED); (d) case table grows 18 -> 26 (every case red for exactly its own reason, one FAIL line each except the two-host cases), plus 3 self-mutants each killed by its case (q4k-required-false, q4k-without-cuda, inventory-missing); --case with no such case is now RED, and the root is derived from the script path, not git rev-parse (#3581). WHAT IT DOES NOT DO (cop ruling 2026-09-21): qwen3-8b-q4km stays `required: false` in the contract, and the gate refuses that key at T-2, so the release cannot ship on it. Flipping it here would turn pv lint red on EVERY PR: the armed ladder-green shape reads the committed 0.68.2 lambda receipt (golden_output: Empty output), measured rc 0 -> 1 with only that flip. The flip folds into the 0.69.1 batch with the qwen3-8b fix and a committed green lambda receipt. So done_when 1''s `no required: false` clause is NOT met by this row: Refs, not Closes. Done_when 2''s cells turn green only through the model fixes under EPIC #3710; done_when 3 and 4 are aprender-f0''s (#3708). cells[] per verb x context (the widened bar, #3715 proposal) ships with the cop''s widened-bar delta, not here. WHERE THE GATE RUNS (so its rc=1 on the real contract is the release NO-GO, not a PR red): check_model_ladder.sh is declared ONLY in Cargo.toml [package.metadata.dogfood] gates (line 612, the T-2 pre-publish dogfood) and is listed in scripts/unwired_guards_baseline.txt (line 12): no PR workflow runs it, and guard_tree skips it (unwired-baseline). pv lint is a different tool that never runs this script; it reads the contract and PASSES with qwen3-8b at required: false. Measured on this head: guard_tree --no-cargo 0 failed, pv lint contracts/ PASS, cargo test -p aprender-contracts --lib 0 failed. THE GPU LOCK (cop ruling 2026-09-21, after a gx10 global OOM at 15:56:08Z killed CI containers): every apr call in model_ladder.sh goes through apr_locked = flock -E 75 -w ${MODEL_LADDER_LOCK_WAIT:-1800} /tmp/apr-gpu.lock choom -n 1000 --; a lock still held after the wait is an ENV decline (exit 2) naming the holder pid from /proc/locks, never a hang and never a model verdict. T-1 wraps the script in choom only (a second flock outside would deadlock). check_model_ladder.sh audits it statically in the REAL run (a raw "$APR" call is RED) and behaviourally in --self-test (fake apr via --lock-probe: lock held + oom 1000; a held lock declines naming the pid), with four producer mutants each killed: raw-apr-call, no-lock, no-choom, unbounded. FALSIFY-MCL-015. ROW B1 (the judge side of cells; stacked on row A; operator bar raise 2026-09-21: "lets ensure this release has MORE than enough tokens for our needs (overdo it), and our verbs are dogfooded, i.e chat with and without thinking, ditto run, ditto code, etc,"). A cell is (model, host, verb, thinking, context rung). contracts/model-capability-ladder-v1.yaml gains `ladder.cells` {verbs [run, chat, serve, code], long_rungs_for {families [qwen35], representatives: one file per other arch}}. The RUNGS are deliberately NOT in the contract: they live in evidence/release/context-rungs.json (apr-release-context-rungs/v1, landing in #3715), one declaration shared with pv; consumer-max = max over consumers[].max_prompt_tokens. The judge (scripts/lib/model_ladder_cells.py, called from check_model_ladder.sh) derives the owed set from the MEASURED inventory: short rungs for every model up to its context_length, long rungs for qwen35 and each arch''s representative; thinking modes from the template evidence (enable_thinking -> both; always-open -> on; else off, #3723), never the producer''s claim; fit := weights + kv_bytes_per_token * tokens + workspace <= gpu_mem_total_bytes (operator ''a'', #3710; an unmeasured term keeps the cell owed in full). A fitting cell must pass (cuda, no fallback, rc 0, prompt_tokens >= rung, answer_chars > 0, think_closed when on); a non-fitting cell must be `refused` with required_bytes > the host TOTAL (a co-tenant refusal is RED); every owed cell must pass on >= 1 required host; every arch in any inventory has a representative some required host holds. A missing context-rungs file, a consumer without a number, a missing context_length (#3733) and a receipt with no cells are each a named FAIL, never an empty owed set. Anti-shrink floors vs origin/main: the cells block (block, verbs, long-rung families and representatives) AND the rung file evidence/release/context-rungs.json (its rung ids and consumers; git show origin/main, absent there = bootstrap), because the rungs live in that file, not in the cells block. A `pass` on a cell the declared arithmetic says cannot fit is RED (MCL-INV-008: not fits => refused): the arithmetic decides what is owed, so a pass beyond it proves the arithmetic wrong (#3596). The module itself keeps only REQUIRED hosts'' receipts. Case table 26 -> 47; 10 module mutants each killed by its case. The PRODUCER side (row B2: measuring the cells) is not in this row: it needs #3723 (thinking toggle), #3733 (inspect keys) and #3596 (capacity::plan, the pre-load refusal). Until it lands, the real T-2 run is RED on ''receipt carries no cells'', which is the operator''s bar, not a PR red: check_model_ladder.sh runs only in the release dogfood (Cargo.toml [package.metadata.dogfood]) and is listed in scripts/unwired_guards_baseline.txt. MEASURED (answers a round-3 lane claim that qwen35-family GGUFs report general.architecture = qwen2): on lambda, Qwen3.5-0.8B-Q4_K_M.gguf and Qwen3.5-4B-Q4_K_M.gguf report general.architecture = qwen35 (apr inspect --json, and a direct GGUF v3 header read whose keys are qwen35.block_count, qwen35.context_length 262144, qwen35.full_attention_interval 4, ...), so `families: [qwen35]` matches them by arch. The fixture''s q.gguf is arch qwen35 and owes 16 cells in green-cells, every long rung included; a mutant deleting the families check (family-long) is killed by red-cells-missing-cell. 11 cells mutants.' +- id: PMAT-3715 + github_issue: 3715 + item_type: task + title: 'pv SHACL release-readiness shape: the tag is REFUSED unless every (Q4_K model in host inventory × {lambda,gx10} × {run,chat,serve,code} × consumer-context) cell has a Pass receipt — a missing cell is a violation, never a skip' + status: in_progress + priority: critical + assigned_to: aprender-97 + created: 2026-09-21T15:55:00Z + updated: 2026-09-21T15:55:00Z + spec: null + acceptance_criteria: + - 'Issue #3715 done_when 1, verbatim: "**the evidence graph**: `pv extract` ingests the release receipts (per-host model inventory, ladder and matrix receipts from #3712/#3708, dogfood receipt) as nodes: `:Model` (sha256, quant, arch), `:Host`, `:Verb` (run/chat/serve/code), `:ContextRung` (token counts DERIVED from the consumer-traffic records on #3710, never invented), `:Receipt` (verdict, backend, fallback, apr version plus sha)"' + - 'done_when 2, verbatim: "**the shape** `release-readiness-v1`: focus = every `:Model` in the inventory of every required `:Host`. For each (verb, context rung) exactly one `:Receipt` with `verdict=Pass`, `backend=cuda`, `fallback=false`, `apr_sha = release commit`. A missing, skipped, stale-sha or fallback receipt is a violation naming the cell."' + - 'done_when 3, verbatim: "**armed with a red-turning falsifier**: fixtures where each single cell is missing, skipped, falls back, or is stale, and each one turns the gate RED (the ONT-001 rule: a shape that has never gone red isn''t armed)"' + - 'done_when 4, verbatim: "**wired where the decision is made**: autopilot''s T-1 step and `check_publish_preflight.sh` both run `pv lint --gate shapes --shape release-readiness-v1` on the release commit''s evidence, and a violation STOPs before the tag and refuses at T-4"' + - 'done_when 5, verbatim: "**proof**: on the v0.69.0 evidence the shape reports the actual leaks (qwen3-8b lambda, qwen2.5-coder lambda, qwen3moe both hosts, chat/serve/code cells absent). On 0.69.1 it passes with N/N cells named."' + phases: [] + subtasks: [] + estimated_effort: null + labels: + - kind:code + notes: 'KNOWN REMAINDER (cop ruling, verbatim): release_evidence.rs `VERBS` is a hand-coded list; #3745 S2 replaces it with the derived cell set via `Inputs.cells`. 0.69.1 does not tag while it exists. QUORUM: round 3 AGREED at 84289cb7ba38adf96c0a1c5f836eb84ccb8d0646 (gemini-3.1-pro-high, gemini-3.1-pro-low, gpt-oss-120b-medium all PASS; docs/audits/quorum-PMAT-3715.json). Assigned by the cop (aprender-04) to aprender-97 on the 0.69.1 all-hands (EPIC #3710). The receipt FORMAT is shared and was agreed on #3712 before building: with aprender-62 (measuring side: issuecomment-5763332078, -5763540875, -5763596544, memory fields -5763795158) and aprender-f0 (T-1/T-4 wiring, issuecomment-5763344947); kernel-diff with aprender-37/aprender-eb (-5763412455, -5763633743, -5763653200); tokenizer parity on #3726 (-5763745060). SCOPE ADDED ON THE ISSUE by its author (cop), each built: kernel differential per GPU arch (#3715 issuecomment-5763396082); thinking {on,off} x context rungs {4k,8k,20k,60k,consumer-max,declared} (-5763490841, #3710 -5763489958); tokenizer parity per model (#3726 done_when 3); the memory-fit rule, operator decision "a" (#3710 -5763753907); three-state thinking_modes (#3723). WAIVERS: none, cop ruling (c) - a comment-based waiver is forgeable by every agent session (all post as noahgift); the operator-only surface is #3722. DONE HERE: done_when 1 (pv extract --release-* --out ingests inventory, cells, kernel/tokenizer/dogfood receipts as Release/Host/Model/Verb/ContextRung/Cell/CellReceipt/... nodes); done_when 2 (the nine-shape release-readiness-v1 family, armed by --shape); done_when 3 (34 red-turning CLI cases in crates/aprender-contracts-cli/tests/ont_release_readiness.rs, wired as ci/explicit-test-commands.d/450; two mutants caught: skip-as-pass turns 1 red, always-fresh turns 3 red); the pv side of done_when 4 (the invocation and exit contract f0 calls: 0 Pass, 1 Fail naming cells, 2 decline, 3 caller error); the first half of done_when 5 (evidence/release/proof-0.69.0: RED, 795 findings naming qwen3-8b lambda verdict=fail, qwen2.5-coder lambda and qwen3moe both hosts unmeasured/unkeyed, 752 of 768 cells absent, the NO-GO inherited dogfood). NOT DONE HERE (so the PR carries Refs, not Closes): the autopilot T-1 / check_publish_preflight T-4 call sites (aprender-f0, #3717/#3708), and "on 0.69.1 it passes with N/N cells named", which needs the producer (#3712 cells[]) and the model fixes to land. Also fixes a collision that existed independently: receipts.rs refused every schema but apr-model-ladder-receipt/v1 by name, so the first committed v2 receipt would have turned `pv lint --gate shapes` exit 3 on every PR.' +- id: PMAT-3717 + github_issue: 3717 + item_type: task + title: '#3712 done_when 3 (child): the model-matrix gate runs at T-1 on BOTH hosts before any tag, receipts committed in the bump, re-read at T-4 by the same judge' + status: in_progress + priority: critical + assigned_to: aprender-f0 + created: 2026-09-21T15:46:37Z + updated: 2026-09-21T16:30:00Z + spec: null + acceptance_criteria: + - 'Issue #3717 done_when 3 (verbatim from #3712), verbatim: "**It runs BEFORE the tag**, as a T-1 step in `scripts/release/autopilot.sh` on BOTH hosts (gx10 over the operator-authorized lambda→gx10 SSH). A NO-GO stops the train with no tag cut. The per-host receipts are committed in the bump PR (#3708), and `check_publish_preflight.sh` re-reads the same receipts at T-4."' + - 'cop requirement 1, verbatim: "**red-turning tests**: a case table driving the real autopilot `models` step and the T-4 R7 with stubs. An unreachable host, a failed build, a missing receipt, a red receipt, or a judge decline (exit 2) each STOP before the tag (and FAIL at R7). A green pair passes. A mutant per row."' + - 'cop requirement 2 (instrument isolation; its LOCATION clause is replaced by the amendment below), verbatim: "**instrument isolation on gx10**: build in a PER-RELEASE target dir on gx10''s /mnt/nvme-raid0 (e.g. `targets/rel-`), delete the previous release''s dir first (the disk is at 90%), and NEVER `~/src/aprender/target`, a shared cache under an old checkout (the #3600 class: a stale instrument agrees with itself). Before measuring, prove `apr --version` = ` ()` on each host, and record it in the receipt."' + - 'cop requirement 3, verbatim: "the judge is `check_model_ladder.sh --version V --receipts DIR`, the same function at T-1 and T-4 (the frozen interface with #3712''s measuring side)"' + - 'cop requirement 4 (measured on the 0.69.1 train; this PR says Refs #3717), verbatim: "proof on the 0.69.1 train: the autopilot STATUS shows `MODELS GO` on both hosts before `TAGGED`"' + - 'AMENDMENT on the issue (comment 2026-09-21T15:50:22Z), verbatim: "**Instrument-isolation location amended (cop, 2026-09-21), replacing the location clause:** build in a PER-RELEASE target dir on gx10, derived at run time and never a literal in scripts/release/: `$HOME/.cache/aprender-release/rel-` on the remote; delete every other `rel-*` there first; NEVER `~/src/aprender/target` or any dir under it. The rest of the item stands (fresh target, previous dirs deleted, `apr --version` = " ()" proved on each host, and the receipt''s apr_version checked against it). Why: on gx10 /mnt/nvme-raid0 is the same filesystem as / (measured), and check_release_scripts_derive_identity.sh R1 refuses absolute /mnt paths in scripts/release/. Added: refuse up front with a named ENV reason when the remote free space is below the measured size of a fresh cuda release target."' + - 'GPU-SHARING REQUIREMENT added on the issue (comment 2026-09-21T16:09:54Z, the cop''s ruling relayed verbatim), verbatim: "Ruling, as you recommend: **one owner**. The LOCK lives in model_ladder.sh, per apr call (62''s row B), which protects every caller. Your T-1 wrapper runs the ladder under `choom -n 1000 --` ONLY (oom_score_adj is inherited across fork) and takes NO flock. Builds stay outside any lock. Your table asserts "runs under choom -n 1000 and the wrapper takes no flock"; 62''s asserts "every apr call runs under the lock"."' + phases: [] + subtasks: [] + estimated_effort: null + labels: + - kind:code + notes: 'P0 on the 0.69.1 release path (cop, aprender-04). This is #3712''s done_when 3, split out so that aprender-62 (done_when 1+2, the measuring side: scripts/model_ladder.sh, scripts/check_model_ladder.sh, the ladder contract) and aprender-f0 do not mint the same fragment. Fixtures and stubs only in CI; nothing here ran against the live train or a publish token. STACKED on PMAT-3708 (branch PMAT-3708-dogfood-one-rule, 629143eeb): the judged diff is base 629143eeb to this head. SCOPE: the new scripts/release/models_t1.sh; scripts/release/autopilot.sh (a new `models` step); scripts/check_publish_preflight.sh (a new R7); the new scripts/check_release_models_t1.sh; and this fragment. "The per-host receipts are committed in the bump PR (#3708)" is already delivered by #3708: prepare_bump.sh --ship refuses unless check_model_ladder.sh --version is green in the bump tree, and refuses gitignored receipts. HOW IT IS MET. (1) BEFORE THE TAG, ON BOTH HOSTS: autopilot STEPS gains `models` between `dogfood` and `tag`. It runs scripts/release/models_t1.sh $AP/models-t1 and STOPs with "T-1 model matrix NO-GO … nothing is tagged" on any non-zero exit. On GO it says "MODELS GO at on lambda and gx10", which is the STATUS line cop requirement 4 asks for before TAGGED. models_t1.sh runs two legs IN PARALLEL: lambda locally in the release worktree, and gx10 over `ssh -o BatchMode=yes gx10 "bash -s"`, the lambda->gx10 SSH the operator authorized. Each leg builds `cargo build --release -p apr-cli --bin apr --features cuda --locked` (binary-release.yml''s command) from the release commit. It then PROVES the binary before measuring (`apr --version` must equal "apr ()"), runs `scripts/model_ladder.sh --host --out ` (the frozen interface), and brings gx10''s receipt back. Each receipt''s existing apr_version field must equal that same line. Then `scripts/check_model_ladder.sh --version --receipts $AP/models-t1` judges both. An unreachable host (ssh rc 255), a failed build, a binary that is not the release, a missing receipt, a red cell and a judge DECLINE (rc 2) are each NO-GO. Nothing is retried, and there is no exemption. (2) INSTRUMENT ISOLATION ON gx10, per the AMENDMENT: a PER-RELEASE dir `$HOME/.cache/aprender-release/rel-`, derived on the remote at run time. It holds a detached worktree at the release commit, and CARGO_TARGET_DIR is /target. Every other rel-* is deleted first, and ~/src/aprender/target is never used (only `git -C ~/src/aprender fetch origin main` and `worktree add` touch that checkout). No host path is a literal in scripts/release/ (check_release_scripts_derive_identity.sh PASS). The worktree is removed after measuring; the target stays for a same-version re-run and is deleted by the next release. DISK: the leg REFUSES up front, as ENV ("MODELS-LEG gx10 ENV: … refused before building", models_t1.sh exit 2), when the filesystem''s free space plus an existing same-version dir is below NEED_KIB = 5,524,629 KiB. MEASURED 2026-09-21 on lambda at 629143eeb with a fresh CARGO_TARGET_DIR: the target is 5,285,126,283 bytes (244 s, nice 19), the binary printed "apr 0.69.0 (629143eeb)", and a worktree checkout is 363,372 KiB. gx10 is aarch64 and was not measured. A read-only probe of gx10 (bash 5.2.21, git 2.43.0): df -Pk reports 116,078,068 KiB free, so it "would-build". (3) T-4 RE-READ, THE SAME JUDGE: check_publish_preflight.sh gains R7, rule_r7(), called by the full gate. It runs `check_model_ladder.sh --version ` in the tagged tree, over the receipts COMMITTED by the bump. rc 0 -> "ok R7"; rc 2 -> "FAIL R7 the model-matrix judge DECLINED (rc 2), and a decline is not a pass"; any other -> "FAIL R7 model matrix NOT green", quoting the judge''s FAIL lines; a missing judge -> "FAIL R7 no model-matrix judge". The seam is PUBLISH_PREFLIGHT_LADDER_JUDGE. The PASS line now ends "dogfood GO, model matrix green". The self-test gains r7_model_matrix_red_refuses, r7_judge_decline_refuses and r7_judge_absent_refuses. Every fixture commits a judge stub that refuses any version but 1.2.3, so all_rules_hold also proves the argument: 28/28. #3715 (the pv SHACL shape, aprender-97) will become the gate of record behind the same call sites. That design is agreed on #3712 (comments 5763316076 and 5763344947): T-1 passes MC, and T-4 passes the measured sha after a `git diff --quiet HEAD -- . '':!evidence''` proof. It is NOT in this diff. RED-TURNING TABLE (cop requirement 1): scripts/check_release_models_t1.sh, a scripts/check_*.sh with no bare cargo token, so guard_tree''s --no-cargo set runs it. It drives the REAL autopilot `models` step, and so the real models_t1.sh, through `autopilot.sh 9.9.9 99 models models`. The fixture: a local bare origin (parent 9.9.8, release 9.9.9); an ssh stub that runs the gx10 script locally under a fake gx10 HOME whose ~/src/aprender sits at the OLD parent with a stale rel-9.9.7 beside it; a build stub that bakes `apr --version` from the tree it ran in; and committed stubs for model_ladder.sh and the judge. T-1 ROWS: green-pair (MODELS GO; gx10 measured by "apr 9.9.9 ()" although its checkout sat at the parent; built into rel-9.9.9, not ~/src/aprender/target; rel-9.9.7 deleted); gx10-unreachable; build-fails; missing-receipt; red-cell; judge-decline; stale-binary; disk-refusal (exit 2, nothing built). T-4 ROWS: rule_r7 EXTRACTED from check_publish_preflight.sh and run on a tagged tree''s committed receipts (cargo-free): r7-green, r7-red, r7-missing (names gx10.json), r7-decline, r7-no-judge. MUTANTS, one per row, each killed: shared-target -> "gx10 did not build into the per-release dir"; no-255-reason, build-ignored, no-receipt-ok -> the STOP no longer names the cause; red-is-go and decline-is-go -> "autopilot exited 0"; no-version-proof -> the pre-measure NOT-THE-RELEASE is gone (the receipt''s apr_version check still stops it, which is defence in depth); no-disk-check -> "autopilot exited 0"; autopilot-no-die -> "autopilot exited 0"; r7-no-version -> "decline: no --version given"; r7-red-is-go and r7-decline-is-go -> "rc=0, want 1"; r7-no-fail-lines -> the refusal no longer names gx10.json; r7-no-judge-ok -> "rc 127", not the no-judge refusal. That the full gate CALLS rule_r7 is proved by the preflight self-test, which needs a Rust-package fixture and runs at ci.yml:1978. Ad hoc wiring mutant (the gate without its rule_r7 call): "BROKE r7_model_matrix_red_refuses / r7_judge_decline_refuses / r7_judge_absent_refuses", 25/28. The table''s first full run caught a fixture bug of mine: the red receipt was written before its env var was exported, so r7-red was RED and its mutant was "killed" for the wrong reason. It was fixed by passing the red host positionally, and the mutant now dies with "rc=0, want 1". ROUND 2 CHANGE (after the round-1 receipt at d00665a7f), the GPU-SHARING requirement: both legs run `choom -n 1000 -- bash scripts/model_ladder.sh …`. oom_score_adj is inherited across fork, so the ladder and every apr it starts are the OOM victim, never the CI pool (the gx10 15:56Z OOM killed 18 CI containers). The wrapper takes NO flock: the lock is model_ladder.sh''s own, per apr call (#3712 row B), and a second flock on /tmp/apr-gpu.lock here would deadlock it. That is the one-owner ruling. Builds run under neither. New row oom-victim: stubs for choom and flock record what wrapped each process, and the row requires "ladder host= oom=1000 flock=0" on both hosts, "build host= oom=none flock=0", and no flock call at all. New mutants: no-choom (gx10''s ladder without choom) -> "gx10 ladder did not run under choom -n 1000 alone: ladder host=gx10 oom=none flock=0"; wrapper-flock (lambda''s ladder under `flock /tmp/apr-gpu.lock choom …`) -> "… ladder host=lambda oom=1000 flock=1". MEASURED at this commit: `bash scripts/check_release_models_t1.sh` rc 0, "PASS 30 row(s)"; `bash scripts/check_publish_preflight.sh --selftest` "28/28 rows"; `bash scripts/check_release_autopilot_dogfood_close.sh` "PASS 17 row(s)" and `bash scripts/check_release_bump_pr_body.sh` "PASS 12 row(s)", both unchanged by the new step; check_bashrs_gate "PASS 365 file(s) … 0 SEC/DET/IDEM errors", after a SEC010 on a `cd /` in the gx10 heredoc was removed; check_guards_are_wired, check_no_pipe_into_grep_q and check_release_scripts_derive_identity all PASS. `bash scripts/guard_tree.sh --no-cargo` at 38f8b5945: "79 checks, 0 failed", with "PASS scripts/check_release_models_t1.sh [run]". Cop requirement 4 (the real 0.69.1 train, MODELS GO on both hosts before TAGGED) is measured on that train, hence Refs #3717.' +- id: PMAT-3718 + github_issue: 3718 + item_type: task + title: 'apr run --json: prompt_tokens + completion_tokens + finish_reason; serve usage and finish_reason never report a cut as a stop (#3718, RAH #3716)' + status: in_progress + priority: high + assigned_to: aprender-a8 + created: 2026-09-21T16:20:00Z + updated: 2026-09-21T16:20:00Z + spec: null + acceptance_criteria: + - 'Issue #3718 done_when 1, verbatim: "`apr run --json` emits `prompt_tokens` (the post-template token count actually fed to the model) and `completion_tokens`, and the numbers match the tokenizer (a case row checks a fixed prompt''s count)"' + - 'done_when 2, verbatim: "the same fields are in `apr serve`''s `/v1/chat/completions` `usage` object (OpenAI shape)"' + - 'done_when 3, verbatim: "a prompt truncated or refused for length says so in the JSON (`truncated: true` or an error), never a silent cut"' + phases: [] + subtasks: [] + estimated_effort: null + labels: + - kind:code + notes: 'SCOPE. The three done_when rows quoted above. `prefill_ms` is deliberately NOT emitted by this row. The cop''s lane message asked for it, but it is not in #3718''s done_when, and it is owned elsewhere: aprender-a8 and aprender-3e agreed one definition (wall-clock over the forward passes on the post-template prompt, up to the first token''s logits; load, h2d, the F2 validate and tokenization excluded; null, never 0, where a path does not split prefill). It rides #3606''s StageTimings and 3e''s #3596 timer (`run_qwen35_generate_dispatch_timed`, not on main). Wiring it into this JSON is a follow-up once those land. RUN (done_when 1, 3). New `realizar::infer::run_inference_report(config) -> (InferenceResult, RunReport)`; `run_inference` is a thin wrapper over it. It is a separate entry point so that none of the ~75 `InferenceResult` literals change. `RunReport { finish_reason, context_length }` is None on a path that does not report it (APR, SafeTensors), never a default. The GGUF path computes it where the stop set and the budget are known. `FinishReason::from_decode(generated, stop, budget)`: a last token in the stop set is Stop; otherwise a run that filled its budget is Length; a shorter run ended on a stop token its loop consumed. This rule holds for both loop shapes, traced: the qwen35 CPU/GPU, MoE and wgpu loops PUSH the stop token, the dense CPU (`generate_with_cache`) and CUDA (`decode_blocking`) loops break BEFORE pushing, and every loop''s only other non-error exit is its budget. `decode_budget` clamps to `context_length - prompt` only on the dense CPU loop, the one loop that does (`effective_max_tokens`). `FinishReason` is declared once, in `infer::run_report` (not behind the `server` feature); `api::types::FinishReason` is now a re-export. apr-cli: `RunUsage { prompt_tokens, completion_tokens, finish_reason, context_length }` on `RunResult`, emitted by `build_final_json`, so it appears in both `--json` and the `--stream` final event. `prompt_tokens` = `input_token_count`, the post-chat-template count including BOS that `prepare_tokens` fed the model. `completion_tokens` is the engine count, never the word-count stand-in `tokens_generated` falls back to. No path truncates a prompt (traced: `prepare_tokens` has no truncation). An over-long prompt is REFUSED (ContextLimitExceeded -> error exit), which is done_when 3''s "or an error"; the completion-side cut is now `finish_reason: length`, where before it was silent. SERVE (done_when 2, 3). `/v1/chat/completions` `usage` was already OpenAI-shaped with the post-template count on the realizar backends (`tokenize_chat_prompt`: template, encode, `len`). Three silent cuts are fixed. (a) The dense CPU backend (`try_quantized_backend`) judged finish against the REQUESTED max_tokens after `effective_max_tokens` had clamped the loop to the context room, so a context-clamped cut said "stop"; it is now judged against the budget the loop ran with, both streaming and not. (b) A prompt that cannot fit was a 500 (non-stream) or a 200 SSE carrying an error event that still ended "stop". It is now a 400 before generation on both, via the existing `generation_error_status`, whose doc already said callers map it to 400. (c) The APR CPU fallback (apr-cli handler_apr_cpu_completion.rs) hardcoded "stop" while capping max_tokens at 4096. It now uses `from_decode` against that cap with the loop''s real stop set (token 0, pushed; `is_eos_token`). Its `run_apr_cpu_inference` also returned the PROMPT as the completion when nothing was generated (`else { &output_tokens[..] }`); that now yields empty. MEASURED on lambda, CPU, Qwen3.5-0.8B-Q4_K_M, pinned `apr 0.69.0 (3fadee114)` via scripts/apr_bin.sh. `apr run -i <"What is 2+2? Answer with one number."> --chat --json`: prompt_tokens 24, context_length 262144. The same templated string through llama.cpp 41fc758 `llama-tokenize --ids` gives 24 ids, identical id-for-id to apr''s `--verbose` encode (with and without llama.cpp''s BOS policy). That is done_when 1''s case row against an independent tokenizer. max-tokens 4 and 1024: completion 2, finish stop (the model ends on EOS). A 20-line-poem prompt at max-tokens 6: completion 6, finish length. DONE_WHEN 1 CASE ROW, automated (added after the round-2 quorum FAIL, which correctly said a manual measurement is not a case row): infer::run_report::a_fixed_text_prompt_counts_the_tokens_the_tokenizer_feeds builds the pygmy GGUF with a SentencePiece vocabulary (` ▁Hello ▁world !` + filler; bos 1, eos 2) through the new `test_factory::build_executable_pygmy_gguf_with` metadata hook, runs the TEXT prompt "Hello world!" through `run_inference_report` (the `apr run --prompt` path: prepare_tokens_gguf, the GGUF tokenizer, BOS policy) and asserts the fed ids are [1, 3, 4, 5] and input_token_count (which becomes prompt_tokens) is 4. The expectation is derived by hand from the vocabulary, not from apr''s encoder. Mutant: prepare_tokens_gguf reports tokens.len()-1 -> the row goes RED. The post-template count on a real model is the llama.cpp row above (24 == 24, id for id). TESTS. infer::run_report: a case table over both loop shapes, the clamped-cut pin, budget arithmetic, and four rows through `run_inference_report` on the pygmy GGUF (context 32): a context-clamped cut is Length with 28 = 32 - 4 tokens, a max_tokens cut is Length, a stop token under budget is Stop (0 tokens, dense loop consumed it), and an over-long prompt is refused naming both lengths. api::tests::usage_finish_3718 runs through the real router on a quantized AppState: clamped cut -> length with usage prompt+completion == context, clamped stream -> terminal chunk length, prompt > context -> 400 on stream and non-stream, stop under budget -> stop (both), unclamped -> cut at the request budget. apr-cli run_tests_usage_3718: JSON carries the four fields, null (not 0 or "stop") when unreported, stream final event carries them. MUTANTS, each run and each RED: serve clamp reverted (2 red), 400 reverted to the old path (1), run-report clamp off (1), from_generation always Length (4), never Length (7). SUITES at 3fadee114 + rustfmt: `cargo test -p aprender-serve --lib` 15913 passed; `cargo test -p apr-cli --lib` 7301 passed; `cargo test -p aprender-contracts --lib` 1684 passed; clippy -D warnings on aprender-serve and apr-cli clean; fmt clean. KNOWN, NOT CHANGED. completion_tokens counts the pushed stop token on the qwen35/MoE/wgpu loops and not on the dense loops, because that is what each loop returns (measured: qwen35 `tokens: [17, 248046]`, completion 2). The CUDA backends were not exercised on hardware in this row: the report logic is backend-agnostic and clamps only on the dense CPU loop. Streaming chat does not apply stop STRINGS on the true-streaming path; that is pre-existing and not a length cut. Found while measuring and routed separately: `--prompt P --chat` double-templates (24 -> 56 prompt_tokens), which is #3743 = #3672 in batch-1.' +- id: PMAT-3726 + github_issue: 3726 + item_type: task + title: 'apr''s byte-level BPE tokenizer is non-canonical (greedy longest-match, no pre-tokenizer, no merge ranks): every GGUF prompt reaches the model as ids it never saw; corrupts copy tasks (#3693) and every non-ASCII user' + status: in_progress + priority: critical + assigned_to: aprender-c3 + created: 2026-09-21T16:20:00Z + updated: 2026-09-21T17:25:00Z + spec: null + acceptance_criteria: + - 'Issue #3726 done_when 1, verbatim: "apr''s token ids are **identical to llama.cpp''s** (the pinned comparator, `llama-tokenize` / `--verbose-prompt`) for every Q4_K model family in the inventory (qwen35, qwen3, qwen2 / qwen2.5, qwen3moe) on a corpus of real consumer briefs (#3716), source code, and non-ASCII sets (CJK, accented Latin, emoji, box-drawing, zero-width chars), and the round-trip decode(encode(x)) == x"' + - 'done_when 2, verbatim: "the byte-level BPE byte→unicode map (GPT-2 bytes_to_unicode) and the pre-tokenizer regex match the model''s `tokenizer.ggml.pre` / HF definition. Root cause named on the issue."' + - 'done_when 3, verbatim: "a **tokenizer-parity gate** (a scripts/check_*.sh, wired) that fails on any id mismatch against the pinned llama.cpp, plus a mutant (reintroduce the id-0 byte fallback and it goes RED). It becomes a required cell per model in the release shape (#3715)."' + - 'done_when 4, verbatim: "re-measure #3693 after the fix: if the corruption disappears, #3693 closes as this; if not, the remainder stays on #3693"' + phases: [] + subtasks: [] + estimated_effort: null + labels: + - kind:code + notes: 'Filed by the cop (aprender-04) from infra-c8''s measurement; owner aprender-c3, who reached the same root cause from the forward side (#3693). ROOT CAUSE: `GGUFModel::encode` (crates/aprender-serve/src/gguf/token.rs) was documented and implemented as greedy longest-match over each whole segment, with no pre-tokenizer and no merge ranks. `prepare_tokens_gguf` (the `apr run` path), `apr parity` and serve''s batch path all call it. Measured on apr 0.69.0 (225b2a9ab) against the pinned llama-tokenize (d1d3c3396) for "The quorum forbids it; cited refutes!!!": apr gave `Ġquo|rum Ġforbid|s … Ġrefute|s`, llama.cpp `Ġqu|orum Ġforb|ids … Ġref|utes`. On arbiter''s r891 prefix apr gave 253 ids and llama.cpp 248, with 16 differing spans. The lane''s "quoorum" is `Ġquo` (apr''s odd split of the prompt) joined to the canonical continuation `orum`. FIX (this row): new crates/aprender-serve/src/gguf/byte_level_bpe.rs, a port of llama.cpp''s reference: `unicode_regex_split_custom_qwen2`/`_qwen35` line for line (selected by tokenizer.ggml.pre; the fleet inventory has exactly these two: qwen2 covers qwen2/qwen2.5/qwen3/qwen3moe, qwen35 covers qwen35/qwen35moe), GPT-2 bytes_to_unicode, ranked merges from tokenizer.ggml.merges with a priority queue (lowest rank, then leftmost, as llm_tokenizer_bpe_session), special tokens by token_type CONTROL/USER_DEFINED/UNKNOWN longest first (tokenizer_st_partition). `GGUFModel::encode` routes every byte-level vocabulary through it. A byte-level file with an unimplemented pre-tokenizer keeps the greedy path and says so once on stderr, naming #3726. The old `<|...|>` special-token rule also missed USER_DEFINED ``/``. The gate follows the #3076 shape: scripts/check_tokenizer_parity.sh (PR-time case table, run by guard_tree) and scripts/tokenizer_parity.sh (model-backed, apr vs the pinned llama-tokenize, declared in [package.metadata.dogfood] for the pre-publish phase), reading ids from a new `apr tokenize encode MODEL (-p TEXT | -f FILE)`. That command reads only the GGUF metadata, calls the production GGUFModel::encode, and prints the path taken and decode(encode(x)) == x. Corpus: evidence/tokenizer-parity/corpus/ (public aprender source, a review brief over a public aprender commit, non-ASCII / whitespace / identifier / chat-template sets). MEASURED (receipts in evidence/tokenizer-parity/): lambda, release mode at 9026b70a5, 147/147 (21 GGUFs x the 7 committed files of that time, pinned llama-tokenize d1d3c3396 resolved by the default path); lambda at 9026b70a5 with the 4 real consumer briefs, 210/210 (21 x 10: the 6 committed files after the corpus was trimmed to keep review briefs small, a 25 KB shell script dropped and the review brief cut to 9 KB, plus the 4 briefs); gx10 (aarch64) at 9026b70a5 with the briefs and the 7-file corpus, 132/132 (12 x 11; the pinned build prints d1d3c339 and tp_commit_matches accepts a 7+ hex prefix). Every pair: identical ids, canonical path, decode(encode(x)) == x. MUTANT (done_when 3): bytes >= 0x80 -> id 0 reintroduced; the gate goes RED, 14 of 22 pairs FAIL (the 8 that pass are the pure-ASCII files, as that mutation predicts), first mismatch at the first non-ASCII byte (mutant-id0-lambda.txt). GUARDS: guard_tree --no-cargo 82/82 at 9026b70a5 after three fixes it found: complexity (the splitter is a Splitter with one method per alternative; GGUFModel::encode fell from cognitive 60 to 41), shell lint (SC2075, SEC011), dogfood coverage (surface_audit.csv rows re-mapped through a line diff, one new row, denominator 837 -> 838 in contracts/apr-dogfood-coverage-v1.yaml and the apr-dogfood skill). done_when 4 (#3693 re-measure): an A/B of the fixed binary (9026b70a5) against the control (262dfb2f6, this branch''s parent, which already carries #3672''s template-once fix) on the two lane prompts at temperature 0 on CUDA, queued under the GPU lock on both hosts; recorded on #3693 and #3726 when it lands. OUT OF SCOPE, filed: aprender-core''s BpeTokenizer and realizar''s apr/tokenizer (the APR/HF paths) are #3742; SentencePiece GGUFs keep the greedy path and the gate reports them UNCOVERED, never PASS.' +- id: PMAT-3733 + github_issue: 3733 + item_type: task + title: 'apr inspect --json drops every qwen35.* metadata key: the GGUF reader parsed only an arch-prefix allowlist; every header key now reaches inspect (#3733)' + status: in_progress + priority: critical + assigned_to: aprender-a8 + created: 2026-09-21T17:05:00Z + updated: 2026-09-21T17:05:00Z + spec: null + acceptance_criteria: + - 'Issue #3733 done_when, verbatim: "`apr inspect --json ` emits every KV the header carries (or at least every `.*` key) for any arch, including one inspect has never heard of. A test runs a fixture GGUF with an unknown arch prefix and fails if any header key is missing from `.metadata`."' + - 'Cop comment on #3733 (2026-09-21T16:25:54Z), verbatim: "done_when: `apr inspect --json` emits EVERY GGUF metadata key the file carries, with no arch-prefixed allowlist (the 46 KVs on the 4B). A case row checks the count against the header''s KV count, plus a mutant (drop one prefix and it goes RED). Also trace the RosettaStone path, since 62 didn''t."' + phases: [] + subtasks: [] + estimated_effort: null + labels: + - kind:code + notes: 'ROOT CAUSE (traced, the RosettaStone path included): `apr inspect` sends every GGUF to `run_rosetta_inspect` (inspect.rs:162), then `RosettaStone::inspect_gguf` (rosetta/validate_inspect.rs:419) copies `load_gguf_raw(..).raw_metadata` unfiltered, and `load_gguf_raw` (gguf/api.rs) built that map from `GgufReader::metadata`. `GgufReader::from_bytes` parsed a key only if it started with `tokenizer.`, `general.`, `llama.`, `qwen2.`, `qwen3.`, `phi.`, `mistral.` or `gpt2.` (reader_parsing.rs) and length-skipped everything else, so `qwen35.*` (which does not start with `qwen3.`) and `quantize.*` never existed after the parse. No filter was in inspect or rosetta. FIX: nothing is skipped any more. Keys outside the allowlist are decoded into a new `GgufReader::display_only_metadata`, and `raw_metadata` is `metadata` chained with it. DELIBERATELY NOT `keep_all` on this path: the config accessors (`context_length()`, `num_layers()`, `hidden_size()`...) read `metadata`, and `load_gguf_raw` feeds them into the `apr import` model config, so widening `metadata` would give every unlisted architecture a config the import path has never been given. That is a behaviour change outside this row. A test pins it: `reader.context_length()` stays None on the qwen35 fixture while `display_only_metadata` holds the key. `from_file_full` (the shard merge) is unchanged and keeps everything in `metadata`. `skip_metadata_value` lost its only caller and is deleted; its ten builder tests now also assert that the key lands in `display_only_metadata`. HARDENING, which parsing instead of skipping makes necessary: `read_metadata_array` took its element count from the file, called `Vec::with_capacity(count)` and indexed `data[..]` directly for i32/f32 arrays. The span is now checked against the bytes left before any allocation, and the checked `read_i32_le`/`read_f32_le` are used. A hostile count returns FormatError "overruns the file" instead of panicking. The case row covers element types 4, 5, 6 and 8. CLI: `output_rosetta_json` is split into `rosetta_json(path, report) -> Value` plus the print, so the --json document is testable. It emits `report.metadata` unfiltered. MEASURED on lambda. Pre-fix instrument: `apr 0.68.2 (97093d2a5)`. It is a valid baseline because `git log v0.68.2..52f43da71` touches none of reader_parsing.rs, api.rs, validate_inspect.rs or inspect.rs. Qwen3.5-4B-Q4_K_M: .metadata|length = 24, qwen35.* keys = 0, qwen35.context_length = null, which reproduces #3733. Post-fix instrument: `apr 0.69.0 (336b7230b)`, resolved and HEAD-checked by `scripts/apr_bin.sh` under CARGO_TARGET_DIR=/mnt/nvme-raid0/targets/a8-3718. The header KV count comes independently from python `gguf` (GGUFReader.fields minus GGUF.*): 0.8B 46/46, 2B 46/46, 4B 46/46, 9B 46/46, 27B 49/49 (shown/header), each with all 18 qwen35.* keys, context_length 262144, full_attention_interval 4, block_count 24/24/32/32/64, head_count_kv 2/2/4/4/4, key_length 256. Control on an arch that was already on the allowlist: Qwen3-1.7B 32/32, context_length 40960, unchanged. TESTS: api_tests_all_keys_3733.rs covers a fixture whose header carries qwen35.*, quantize.*, an unheard-of prefix `zzz_never_heard_of.*`, general.* and tokenizer.*. It asserts load_gguf_raw and RosettaStone::inspect both show every header key with count == header length, the accessor firewall, and the hostile-count rows. inspect_tests adds rosetta_json_emits_every_metadata_key. Suites: `cargo test -p aprender-core --lib -- format::gguf format::rosetta` 645 passed; `cargo test -p apr-cli --lib -- inspect` 140 passed. MUTANTS, each run and each RED: (1) skip restored (display-only insert removed): 13 red, including both header-count rows and all 10 builder rows; (2) the `.chain(display_only_metadata)` union dropped: both header-count rows red; (3) the cop''s mutant, one prefix dropped (`quantize.*` not kept): both header-count rows red; (4) bounds check removed: the hostile-count row red; (5) a `qwen35.` filter in rosetta_json: rosetta_json_emits_every_metadata_key red. KNOWN, NOT CHANGED: arrays whose element type is 8/16/64-bit (types 0,1,2,3,7,10,11,12) still decode to an empty ArrayUint32 and so display `[len=0]`. This is pre-existing, and the shard merge also depends on that variant, so it is out of scope. Census of all 21 GGUFs in /home/noah/models: array element types are STRING (59) and INT32 (30) only, so no fleet file reaches it.' +- id: PMAT-3747 + github_issue: 3747 + item_type: task + title: '#3568 PR 1/4: the TokenConstraint hook and the llguidance engine behind it — no generation loop wired' + status: in_progress + priority: high + assigned_to: aprender-0e + created: 2026-09-21T17:21:12Z + updated: 2026-09-21T17:25:00Z + spec: null + acceptance_criteria: + - 'Issue #3747 done_when, verbatim: "`constrain::TokenConstraint` {mask, accept, is_complete} exists, and `ConstraintError` names every refusal (SchemaInvalid, SchemaUnsupported, DeadEnd, Rejected, Vocab, NotCompiled), each rendered with its name first"' + - 'verbatim: "`load_schema` takes the schema inline OR as `@path` and refuses non-JSON, non-schema JSON and unreadable paths as SchemaInvalid, before any token"' + - 'verbatim: "llguidance 1.8.0 sits behind the trait, as an optional dependency behind a feature that apr-cli''s `inference` enables; `cargo deny check` (licenses + advisories) stays green with its full dependency tree"' + - 'verbatim: "`GGUFModel::constraint_vocab()` builds each token''s bytes from the SAME per-token decoding `decode()` uses; control/unused tokens are special and can never be constrained text"' + - 'verbatim: "case table: RAH''s quorum-lane schema and rmedia''s three schemas (#3716) compile exactly, accept their real documents, refuse violations at the byte, and allow nothing after one complete value; an approximation is refused; without the feature a constraint is refused (NotCompiled), never ignored"' + - 'verbatim: "mutants: a no-op mask, a dropped special marker, flexible whitespace, allowed approximations and a never-refusing accept each turn the case table RED"' + - 'verbatim: "a measured per-token mask overhead on the Qwen3.5-9B vocabulary (the backend-independent part; the CUDA host-logits download is PR 3''s measurement)"' + - 'NOT THIS PR, verbatim from #3747: "PR 2: apr run''s CPU paths, `--json-schema`/`--grammar`, exit codes + `removed_by`, `SchemaWithThinking`, the independent second reader, `finish_reason`/`prompt_tokens`. PR 3: the CUDA paths and their overhead. PR 4: `apr serve` after #3571. The lenient-extraction removal (§5.0 row 4) is infra''s, after PR 2."' + phases: [] + subtasks: [] + estimated_effort: null + labels: + - kind:code + notes: 'Parent #3568 (design note issuecomment-5763921678, ruled by the cop 2026-09-21): PR 1 of 4. Refs #3568 and Closes #3747. WHAT LANDS. crates/aprender-serve/src/constrain/mod.rs: TokenConstraint; ConstraintError with Display `SchemaInvalid: …`, `SchemaUnsupported: …`, `ConstraintDeadEnd: …`, `ConstraintRejected: …`, `ConstraintVocab: …`, `StructuredOutputNotCompiled: …`; load_schema (inline | @path); ConstraintVocab + validate(); ConstraintEnv (index once per model) -> json_schema / lark; doc: cross-field facts no JSON Schema expresses stay the consumer''s to check (rmedia, #3716). constrain/llg.rs (feature structured-output): special tokens carry TokTrie''s marker byte so no grammar can emit them as text. JSON is COMPACT unless the caller set its own x-guidance: llguidance''s default flexible whitespace ran away in this case table ({ then 63 bytes of whitespace). A boolean schema is normalised: `true` -> `{}`, compact; `false` -> SchemaInvalid (it admits no document). An approximation (grammar warnings, reachable only through a caller''s lenient/coerce_one_of x-guidance) is SchemaUnsupported, never used quietly. Engine compile errors map to SchemaUnsupported with the engine''s message verbatim: llguidance''s errors are untyped, and a substring sort would be a text match standing in for a type. A failed compute_mask (NoExtensionBias) and an empty mask are both DeadEnd. crates/aprender-serve/src/gguf/token.rs: GGUFModel::constraint_vocab() and push_token_bytes, which decode() now calls too. control/unused token_type (3, 5) are special, else the GH-320 <|…|> rule; SentencePiece''s ▁ is mapped on BYTES so a lone <0xNN> survives. test_factory gains add_i32_array. examples/constrain_mask_overhead.rs (required-features structured-output). DEPENDENCY. aprender-serve: llguidance 1.8.0 optional, default-features off, [lark, rayon, ahash] (referencing dropped: it duplicated referencing 0.28/0.29; a $ref schema is then refused by name). Feature structured-output = [dep:llguidance]; apr-cli inference adds realizar/structured-output. NOTE for reviewers: `realizar` IS aprender-serve. Root Cargo.toml has `realizar = { path = "crates/aprender-serve", package = "aprender-serve" }`, aprender-serve''s [lib] name is `realizar`, and apr-cli''s existing cuda feature enables `realizar/cuda` the same way. `cargo check -p apr-cli` rc 0 and the example (`use realizar::constrain`) builds rc 0. A round-1 lane read this as a mismatch and was refuted by the build. cargo deny check licenses / advisories / bans / sources: all rc 0; duplicate-crate warnings 64 on main -> 66, the +2 being strum/strum_macros 0.28 via derivre (llguidance''s regex engine). MEASURED at this commit. cargo test -p aprender-serve --lib --features structured-output -- constrain:: constraint_vocab: 24 passed. Without the feature: 12 passed, including StructuredOutputNotCompiled. Consumer schemas (RAH quorum-lane; rmedia quiz, outline, key-terms/reflection) compile exactly, accept real documents byte by byte to completion, allow nothing after one complete value, and refuse 7 violations at the byte (enum, minimum, pattern x2, additionalProperties:false, const, minLength). RAH row: a model intending the 2026-09-21 NULL vote {"verdict":"PASS","summary":"ok","finding":[]} is held until the required `findings` exists; unconstrained it writes the NULL vote. MUTANTS (8, each RED, re-run on this commit): mask a no-op (7 rows), special marker dropped (1, the in-string special row), flexible whitespace (3), approximations allowed (1), accept never refusing (2), token_type ignored (1), gpt2 bytes read as UTF-8 (1), boolean `true` left flexible (1). OVERHEAD, examples/constrain_mask_overhead, release, 3 runs, a real 124-token RAH verdict tokenized by the model''s own tokenizer and accepted end to end: Qwen3.5-9B-Q4_K_M (vocab 248,320) index 559-606 ms once per model, compile 0.67-0.79 ms, mask p50 155-163 us, p95 207-223 us, max 262-291 us per token; Qwen3-1.7B (151,936) p50 104-105 us. Against RAH''s measured decode of 1.0-2.9 tok/s that is under 0.05 percent. cargo fmt --all -- --check rc 0 (the example is formatted; round 1 caught it unformatted); clippy -p aprender-serve --lib --features structured-output -D warnings rc 0.' +- id: PMAT-3756 + github_issue: 3756 + item_type: task + title: 'Parity lane refuses on intel and mini: the pinned llama.cpp (d1d3c3396) has no llama-bench/llama-server there' + status: in_progress + priority: high + assigned_to: aprender-04 + created: 2026-09-21T18:00:00Z + updated: 2026-09-21T18:00:00Z + spec: null + acceptance_criteria: + - 'Issue #3756 done_when, verbatim: "done_when: `llama_bin.sh` resolves the pin on intel and mini (evidence committed like `evidence/parity/pin-bump-d1d3c3396/RESOLVE-{lambda,gx10}.txt`), and `host_receipt.sh intel|mini` writes a `parity` block"' + - 'THIS PR delivers the FIRST clause only (llama_bin.sh resolves the pin on intel and mini, with the evidence committed). The second clause (host_receipt.sh writes a parity block) needs #3731, which is not on main; this PR says Refs #3756, not Closes.' + phases: [] + subtasks: [] + estimated_effort: null + labels: + - kind:code + notes: 'SCOPE: scripts/llama_bin.sh (one line: the commit-token extraction), evidence/parity/pin-bump-d1d3c3396/RESOLVE-{intel,mini}.txt, and this fragment. STACKED on #3563(A) (branch PMAT-3563-llama-pin-resolves-both-hosts @ 8fe36c080, receipt committed), whose canonical-location resolver it uses; fold #3563(A) first. HOST PROVISIONING (the cop''s fleet row; the operator: "you are traffic cop and have ssh, you fix anything you need"; done before the 0.69.1 train starts, not inside it): intel: `cmake --build ~/src/llama.cpp-d1d3c3396/build --target llama-bench llama-server llama-tokenize llama-cli -j 8` under nice 19 / ionice idle (incremental; the existing cache already matched build_flags_intel, GGML_NATIVE=ON, GGML_CUDA=OFF), BUILD_EXIT=0. mini: `git clone https://github.com/ggml-org/llama.cpp ~/src/llama.cpp-d1d3c3396 && git checkout d1d3c3396aa13a5f239109a822666c4870490ad5 && cmake -B build -DGGML_METAL=ON` (build_flags_mini verbatim), then the same four targets, BUILD_EXIT=0. DEFECT FOUND BY THE PROOF (fixed here): on mini, #3563(A)''s resolver refused the correct build as wrong_build. Its commit extraction `sed -nE ''s/…/p;t;s/…/p''` is one expression, and BSD sed (macOS /usr/bin/sed) reads everything after `t` as a LABEL, so it failed with "undefined label" and extracted nothing. Measured pre-fix on mini, rc 1: `sed: 2: "s/.*commit ([0-9a-f]{7, ...": undefined label '';s/.*\(([0-9a-f]{7,40})\).*/\1/p''` then `FAIL llama.cpp named but NOT the declared build (wrong_build)`. FIX: the same three commands as separate -e arguments. PROBE: 4 inputs (commit d1d3c3396, paren abcdef1, a 6-digit commit, no commit) give byte-identical output under GNU sed 4.8 (lambda) and BSD sed (mini): d1d3c3396 / abcdef1 / empty / empty. MEASURED AFTER: `bash scripts/check_llama_pin.sh` rc 0, 43 ok rows, both mutants still killed; live `env -u LLAMA_BENCH_PATH LLAMA_PIN_HOST= bash scripts/llama_bin.sh` rc 0 on intel (`version: 0.4.1-dev (build 2744, commit d1d3c3396)`) and on mini (`version: 0.4.1-dev (build 10987, commit d1d3c3396)`), in the committed RESOLVE files. REMAINDER: the llama.cpp builds live in user-space trees under ~/src on each host; declaring them in forjar (so a re-provisioned host rebuilds them) is not in this PR.' +- id: PMAT-3770 + github_issue: 3770 + item_type: task + title: 'batch-1 combination break: check_release_bump_pr_body.sh fixture ladder receipts meet the current #3712 judge' + status: in_progress + priority: critical + assigned_to: null + created: '2026-09-21T00:00:00Z' + updated: '2026-09-21T00:00:00Z' + spec: null + acceptance_criteria: [] + phases: [] + subtasks: [] + estimated_effort: null + labels: + - kind:code + notes: 'ISSUE (#3770): folding #3721 + #3738 into release/0.69.1-batch-1 (ded8a932a) turned scripts/check_release_bump_pr_body.sh RED on 4 rows (epic-and-refs, prs-only, ladder-missing-gx10, ladder-ignored) + the drop-refusal mutant, all ''STOP model-ladder receipts for 9.9.9 are not green in the bump tree''. The table runs the REAL scripts/check_model_ladder.sh on fabricated receipts that were the pre-#3712 shape (v1, no inventory; a fixture ladder with no inventory block; a seed tree without the producer the judge''s GPU-lock audit reads and without the cells module the judge imports). done_when (the cop, 2026-09-21): fixtures green under the current judge; the two refusal rows name their intended reasons; every other table that fabricates ladder receipts swept in the same commit; guard_tree --no-cargo on the branch shows only the two known complexity REDs. WHAT THIS DOES: write_receipt writes apr-model-ladder-receipt/v2 (inventory = the fixture rung''s file, candidates = that file read as a Q4_K member, the rung row names its file); the fixture ladder COPIES the real contract''s `inventory` block (never retyped, so it follows the tree''s inventory shape: row A''s patterns today, #3763''s candidates + member_dtype after it folds) and its rung names a file; the seed tree gets real copies of scripts/model_ladder.sh and scripts/lib/model_ladder_cells.py. SWEEP: the other tables that reference the ladder judge were run and need nothing: check_publish_preflight.sh --selftest (R7 rows all ok; it uses a judge STUB) and check_release_models_t1.sh --self-test (rc 0; stubs). MEASURED: check_release_bump_pr_body.sh --self-test PASS 12 rows (all 4 formerly-red rows ok; ladder-missing-gx10 names evidence/dogfood/models/9.9.9/gx10.json and ladder-ignored names ''are gitignored''; mutants drop-keep-open, drop-refusal, drop-ladder, drop-ignored all killed); the same table with #3763''s judge, producer and contract swapped in: PASS 12 rows.' +- id: PMAT-3792 + github_issue: 3792 + item_type: task + title: 'shapes_gate complexity ratchet: pure refactor of run_shapes_gate_with, no behaviour change' + status: in_progress + priority: critical + assigned_to: aprender-97 + created: 2026-09-21T21:58:00Z + updated: 2026-09-21T21:58:00Z + spec: null + acceptance_criteria: + - 'Issue #3792 done_when 1, verbatim: "`run_shapes_gate_with` is no longer RED in `bash scripts/check_complexity_ratchet.sh`, via a PURE refactor with no behaviour change: the same answers, in the same order."' + - 'done_when 2, verbatim: "`cargo test -p aprender-contracts --lib` is green, and `pv lint contracts --gate shapes` passes with 8/8 pc_extract controls fired."' + phases: [] + subtasks: [] + estimated_effort: null + labels: + - kind:code + notes: 'The cop (aprender-04) requested this, verbatim: "Please land a pure-refactor commit on a branch OFF release/0.69.1-batch-1 (not your old branch) that brings it under threshold with no behaviour change. Proof: `bash scripts/check_complexity_ratchet.sh` rc 0, `cargo test -p aprender-contracts --lib` green, and `pv lint contracts --gate shapes` 8/8 controls fired." SCOPE: this ticket IS the refactor. It is NOT #3715''s feature, which was already folded at ded8a932a; the diff against ded8a932a is only this refactor plus this fragment and receipt. The ratchet was RED on release/0.69.1-batch-1 (ded8a932a) vs main a9502d992: run_shapes_gate_with cyclomatic 13 cognitive 28 (the limit is 25). The fix moves the preamble (collect -> empty -> arming -> family, same order), the family ordering, the verdict rule and the two report maps into prepare / order_by_family / needs_receipts / verdict_of / by_shape / by_entity_type. MEASURED at 2a40dda85: the ratchet no longer lists run_shapes_gate_with (rc stays 1 only for crates/aprender-serve/src/constrain/tests.rs::generate_intent, not in this diff and not this ticket''s); cargo test -p aprender-contracts --lib 1701 passed; pv lint contracts --gate shapes Pass, 8/8 pc_extract fired; ont_release_readiness / ont4b / ont4c1 / ont4c3 CLI targets 34/11/12/10 passed; clippy -D warnings clean. A prior round under PMAT-3715 had a lane FAIL on SCOPE because it judged this delta against the #3715 feature ticket; the cop ruled that this refactor gets its own ticket.' diff --git a/evidence/parity/pin-bump-d1d3c3396/RESOLVE-gx10.txt b/evidence/parity/pin-bump-d1d3c3396/RESOLVE-gx10.txt new file mode 100644 index 0000000000..eb9e5528c8 --- /dev/null +++ b/evidence/parity/pin-bump-d1d3c3396/RESOLVE-gx10.txt @@ -0,0 +1,8 @@ +# scripts/llama_bin.sh on gx10, #3563 (A): nothing named, the canonical location resolves +# measured: 2026-09-21T16:51:53Z host: gx10-a5b5 files: scripts/llama_bin.sh + scripts/llama_pin.toml from aprender tree 52f43da71 + the #3563 change, copied to a scratch dir (gx10's own checkout is stale at de0e3e182) +# command: env -u LLAMA_BENCH_PATH bash scripts/llama_bin.sh +ok llama.cpp pinned: /home/noah/src/llama.cpp-d1d3c3396/build/bin/llama-bench + build: version: 0.4.1-dev (build 2423, commit d1d3c339) + cmake: GGML_CUDA=ON CMAKE_CUDA_ARCHITECTURES=121 + pin expires 2026-12-01 +# rc=0 diff --git a/evidence/parity/pin-bump-d1d3c3396/RESOLVE-intel.txt b/evidence/parity/pin-bump-d1d3c3396/RESOLVE-intel.txt new file mode 100644 index 0000000000..ec65ed461b --- /dev/null +++ b/evidence/parity/pin-bump-d1d3c3396/RESOLVE-intel.txt @@ -0,0 +1,9 @@ +# scripts/llama_bin.sh on intel, #3756: nothing named, the canonical location resolves +# measured: 2026-09-21T17:58:53Z host: mac-server shell: GNU bash, version 5.1.16(1)-release (x86_64-pc-linux-gnu) sed: sed (GNU sed) 4.8 +# files: scripts/llama_bin.sh + scripts/llama_pin.toml from aprender branch PMAT-3756-llama-pin-intel-mini (the #3563(A) resolver + the portable-sed fix), copied to a scratch dir +# command: env -u LLAMA_BENCH_PATH LLAMA_PIN_HOST=intel bash scripts/llama_bin.sh +ok llama.cpp pinned: /home/noah/src/llama.cpp-d1d3c3396/build/bin/llama-bench + build: version: 0.4.1-dev (build 2744, commit d1d3c3396) + cmake: GGML_CUDA=OFF CMAKE_CUDA_ARCHITECTURES=unset + pin expires 2026-12-01 +# rc=0 diff --git a/evidence/parity/pin-bump-d1d3c3396/RESOLVE-lambda.txt b/evidence/parity/pin-bump-d1d3c3396/RESOLVE-lambda.txt new file mode 100644 index 0000000000..7b5bd3b8e3 --- /dev/null +++ b/evidence/parity/pin-bump-d1d3c3396/RESOLVE-lambda.txt @@ -0,0 +1,8 @@ +# scripts/llama_bin.sh on lambda, #3563 (A): nothing named, the canonical location resolves +# measured: 2026-09-21T16:51:53Z host: noah-Lambda-Vector tree: 52f43da71 + the #3563 llama_bin.sh change (uncommitted at measure time) +# command: env -u LLAMA_BENCH_PATH bash scripts/llama_bin.sh +ok llama.cpp pinned: /home/noah/src/llama.cpp-d1d3c3396/build/bin/llama-bench + build: version: 0.4.1-dev (build 10987, commit d1d3c3396) + cmake: GGML_CUDA=ON CMAKE_CUDA_ARCHITECTURES=unset + pin expires 2026-12-01 +# rc=0 diff --git a/evidence/parity/pin-bump-d1d3c3396/RESOLVE-mini.txt b/evidence/parity/pin-bump-d1d3c3396/RESOLVE-mini.txt new file mode 100644 index 0000000000..71658c8fa6 --- /dev/null +++ b/evidence/parity/pin-bump-d1d3c3396/RESOLVE-mini.txt @@ -0,0 +1,9 @@ +# scripts/llama_bin.sh on mini, #3756: nothing named, the canonical location resolves +# measured: 2026-09-21T17:59:01Z host: mini os: Darwin arm64 shell: GNU bash, version 5.3.20(1)-release (aarch64-apple-darwin25.6.0) sed: BSD sed (no --version; /usr/bin/sed) +# files: scripts/llama_bin.sh + scripts/llama_pin.toml from aprender branch PMAT-3756-llama-pin-intel-mini (the #3563(A) resolver + the portable-sed fix), copied to a scratch dir +# command: env -u LLAMA_BENCH_PATH LLAMA_PIN_HOST=mini bash scripts/llama_bin.sh +ok llama.cpp pinned: /Users/noahgift/src/llama.cpp-d1d3c3396/build/bin/llama-bench + build: version: 0.4.1-dev (build 10987, commit d1d3c3396) + cmake: GGML_CUDA=OFF CMAKE_CUDA_ARCHITECTURES=unset + pin expires 2026-12-01 +# rc=0 diff --git a/evidence/release/context-rungs.json b/evidence/release/context-rungs.json new file mode 100644 index 0000000000..c4f107df9a --- /dev/null +++ b/evidence/release/context-rungs.json @@ -0,0 +1,54 @@ +{ + "schema": "apr-release-context-rungs/v1", + "_doc": "aprender#3715. The context rungs release-readiness-v1 requires of every (model, host, verb, thinking) cell. `consumer-max` carries no number: pv COMPUTES it as the max over `consumers[]`, and a consumer with no number is a named violation. The per-model `declared` rung is each model's own GGUF context_length and is never written here (pv refuses it). A `long` rung (and `declared`) is owed only by the models the ladder's `cells.long_rungs_for` names (#3712 amendment 2); a rung larger than a model's measured context_length is not owed by that model (cop ruling, 2026-09-21: 'correct at every rung up to the model's own declared context length', #3710).", + "consumers": [ + { + "consumer": "rah", + "max_prompt_tokens": 148000, + "basis": "measured", + "source": "https://github.com/paiml/aprender/issues/3716#issuecomment-5763300198", + "quote": "max (PMAT-216) | 454,761 | diff alone 120,745 measured; whole ≈ 148k" + }, + { + "consumer": "rmedia", + "max_prompt_tokens": 128000, + "basis": "plan", + "source": "https://github.com/paiml/aprender/issues/3716#issuecomment-5763407650", + "quote": "Ask: 64k usable context minimum, 128k to cover the specialization-scale case." + }, + { + "consumer": "arbiter", + "max_prompt_tokens": 7000, + "basis": "plan", + "source": "https://github.com/paiml/aprender/issues/3716#issuecomment-5763449814", + "quote": "Planned synthesis input = template + top-8 chunks: 9,150 and 21,703 chunk characters for those two questions ... That is ≈3k–7k tokens" + } + ], + "rungs": [ + { + "id": "4k", + "tokens": 4096, + "derived_from": ["https://github.com/paiml/aprender/issues/3710#issuecomment-5763489958 (operator: \"MORE than enough tokens for our needs (overdo it)\"; required rungs 4k · 8k · 20k · 60k · 148k · declared)"] + }, + { + "id": "8k", + "tokens": 8192, + "derived_from": ["https://github.com/paiml/aprender/issues/3710#issuecomment-5763489958 (operator: \"overdo it\"; required rungs 4k · 8k · 20k · 60k · 148k · declared)"] + }, + { + "id": "20k", + "tokens": 20480, + "derived_from": ["https://github.com/paiml/aprender/issues/3716#issuecomment-5763300198 (RAH p50 quorum brief: whole brief ≈ 20k; sized up to 20480 with the binary rungs, #3712 aprender-62)"] + }, + { + "id": "60k", + "tokens": 61440, + "long": true, + "derived_from": ["https://github.com/paiml/aprender/issues/3716#issuecomment-5763300198 (RAH p90 quorum brief: whole ≈ 60k; sized up to 61440, #3712 aprender-62)"] + }, + { + "id": "consumer-max", + "long": true + } + ] +} diff --git a/evidence/release/proof-0.69.0/README.md b/evidence/release/proof-0.69.0/README.md new file mode 100644 index 0000000000..7857cc76df --- /dev/null +++ b/evidence/release/proof-0.69.0/README.md @@ -0,0 +1,48 @@ +# `release-readiness-v1` on v0.69.0's own evidence (aprender#3715 done_when 5) + +The question: had this shape been the gate at 0.69.0's T-1, would it have stopped the train, and would it have +named the leaks that actually shipped to the publish step? **Yes: RED, 795 findings.** Every leak the issue lists +is named. `gate-summary.json` holds the verdict, the counts, every non-cell finding verbatim, and the cell findings +aggregated per host/model. Re-run the command below for all 795 findings. + +## What went in: 0.69.0's measurements, translated, none invented + +| file | from | measured | +|---|---|---| +| `models/{lambda,gx10}.json` | `rel-069-state/ladder/0.69.0/.json` (the v1 ladder receipt, apr 0.69.0 (225b2a9ab)), `q4k-sweep-.jsonl` (the Q4_K sweep at the same binary), and the listing of the ladder contract's inventory dirs on each host (2026-09-21T16:20:36Z, lambda locally, gx10 over the operator-authorized SSH) | yes, by the 0.69.0 train and the sweep | +| `dogfood-receipt.json` | `rel-069-state/preflight-wt/.dogfood/receipt-20260921T121638Z.json`, byte for byte: the receipt 0.69.0's T-1 "inherited" its GO from (#3708) | yes | +| `translate.py` | the translation itself. Its docstring is the rule set | — | + +The rules `translate.py` follows, so that the translation cannot manufacture a pass: +- The **inventory** is what each host holds. A `sha256` is carried **only** where the 0.69.0 ladder hashed that exact file on that host. Everything else has no hash, and pv names it `unmeasuredModel`: the release never identified the file, so it never proved it. +- The ladder row becomes the `(run, think-off, 4k)` row, with `verdict` = its green bit and `fallback` from `backends.cuda`. `prompt_tokens` and `answer_chars` are **absent** because 0.69.0 ran golden prompts and recorded neither. +- A sweep row for a file with no measured hash is carried **unkeyed**: pv names it `unkeyedRow`, with its verdict. +- **No row** exists for chat, serve, code, think-on, or any other rung, because 0.69.0 never ran them. There is **no** kernel-diff or tokenizer-parity receipt, because neither existed. + +## What the shape says + +``` +pv lint contracts --gate shapes --shape release-readiness-v1 \ + --release-version 0.69.0 --release-commit 225b2a9abbd171b3249794722f8da5fc8027e565 \ + --receipts evidence/release/proof-0.69.0/models \ + --kernel-receipts --tokenizer-receipts \ + --dogfood-receipt evidence/release/proof-0.69.0/dogfood-receipt.json +→ exit 1, verdict Fail, 795 findings; cells 768 (16 with a row), 16 models, 8 tokenizer cells +``` + +| leak (#3715 done_when 5) | how the shape names it | +|---|---| +| **qwen3-8b on lambda** | `release-cell/0.69.0/lambda/Qwen3-8B-Q4_K_M.gguf/run/think-off/4k`: `verdict: "fail"`. It is the only ladder model with a failing verdict; the same file on gx10 does not fail | +| **qwen2.5-coder on lambda** (0.5b, 7b) | `release-host/0.69.0/lambda`: `unmeasuredModel` for both files, and `unkeyedRow … verdict=fail` for the sweep rows (GPU gibberish, and a fallback to CPU with rc 14) | +| **qwen3moe on both hosts** | `unmeasuredModel` on lambda (`Qwen3-30B-A3B-Instruct-2507`, `Qwen3-Coder-30B-A3B-Instruct`, the first with `unkeyedRow … verdict=fail`: no gates emitted) and on gx10 (`Qwen3-Coder-30B-A3B-Instruct`, which the gx10 sweep never reached) | +| **chat / serve / code absent** | 752 of 768 cells have no row (`minCount`): for each of the 16 (host, model) pairs, 47 of 48 cells are missing, meaning everything except the one golden `run` | +| **the inherited dogfood** (#3708) | `release-subject/0.69.0`: the receipt says `verdict: "NO-GO"` for commit `5f9bd9344` at version `0.68.2`. It is not GO and not fresh. The autopilot's inherited "GO" was derived from it by excluding its version row | +| no kernel differential, no tokenizer parity | both hosts `kernelReceipt minCount 1`, every model's `release-tokenizer/…` cell missing | + +Every ladder model's one golden row also fails `contextMet` and `answered`, because 0.69.0 never sized a +prompt or measured an answer. That is the "overdo it" bar (#3710) applied to a release that predates it, and it +is honest: nothing measured at 4k tokens was ever recorded. + +**What is NOT shown here:** that 0.69.1 passes. That is the other half of done_when 5 ("on 0.69.1 it passes with +N/N cells named"). It needs the producer (#3712's cells[], inventory arithmetic, kernel and tokenizer receipts) +and the fixes to land. The gate will report it at T-1 through `release.cell_names`. diff --git a/evidence/release/proof-0.69.0/dogfood-receipt.json b/evidence/release/proof-0.69.0/dogfood-receipt.json new file mode 100644 index 0000000000..1491f9a4a1 --- /dev/null +++ b/evidence/release/proof-0.69.0/dogfood-receipt.json @@ -0,0 +1,239 @@ +{ + "crate": "aprender", + "version": "0.68.2", + "timestamp": "20260921T121638Z", + "commit": "5f9bd9344f07f6c6388f03f04fa61e0d93d3b411", + "gates": [ + { + "gate": "git-clean", + "result": "PASS", + "note": "working tree clean" + }, + { + "gate": "declared:check_multiplatform_dogfood", + "result": "DEFER", + "note": "scripts/check_multiplatform_dogfood.sh: 0.68.2 is not on crates.io yet, so no host can have run `cargo install aprender --version 0.68.2`; owed by the post-publish dogfood as evidence/dogfood/0.68.2/{" + }, + { + "gate": "declared:check_model_ladder", + "result": "PASS", + "note": "scripts/check_model_ladder.sh exit=0" + }, + { + "gate": "declared:check_verifier_pinning", + "result": "PASS", + "note": "scripts/check_verifier_pinning.sh exit=0" + }, + { + "gate": "declared:check_dogfood_shim", + "result": "PASS", + "note": "scripts/check_dogfood_shim.sh exit=0" + }, + { + "gate": "declared:check_bench_receipt", + "result": "PASS", + "note": "scripts/check_bench_receipt.sh exit=0" + }, + { + "gate": "declared:check_parity_receipt", + "result": "PASS", + "note": "scripts/check_parity_receipt.sh exit=0" + }, + { + "gate": "declared:check_no_claim_literals", + "result": "PASS", + "note": "scripts/check_no_claim_literals.sh exit=0" + }, + { + "gate": "declared:check_perf041_marker", + "result": "PASS", + "note": "scripts/check_perf041_marker.sh exit=0" + }, + { + "gate": "dogfood-gates", + "result": "PASS", + "note": "8 declared gate(s) discovered and all green" + }, + { + "gate": "version-unpublished", + "result": "FAIL", + "note": "aprender 0.68.2 is ALREADY in the crates.io index \u2014 bump the version" + }, + { + "gate": "changelog", + "result": "PASS", + "note": "CHANGELOG has 0.68.2" + }, + { + "gate": "feature-scope", + "result": "INFO", + "note": "clippy/test run with: --features cli" + }, + { + "gate": "fmt", + "result": "PASS", + "note": "Warning: can't set `wrap_comments = true`, unstable features are only available in nightly channel." + }, + { + "gate": "clippy", + "result": "PASS", + "note": " Compiling thiserror-impl v2.0.18" + }, + { + "gate": "test", + "result": "PASS", + "note": " Compiling thiserror-impl v2.0.18" + }, + { + "gate": "coverage", + "result": "PASS", + "note": "warning: aprender-compute@0.68.2: provable-contracts binding.yaml not found at /mnt/nvme-raid0/agent-wt/rel-069-state/pr" + }, + { + "gate": "security", + "result": "PASS", + "note": "advisories ok" + }, + { + "gate": "security-2nd-source", + "result": "PASS", + "note": "GitHub Advisory DB (independent of RustSec): 0 open Dependabot alerts" + }, + { + "gate": "pmat-pin", + "result": "PASS", + "note": "fleet pmat is the correct verifier for a non-pmat crate (exported to declared gates)" + }, + { + "gate": "release-binary", + "result": "PASS", + "note": "built: /mnt/nvme-raid0/agent-wt/rel-069/target/release/apr" + }, + { + "gate": "deterministic-tools", + "result": "PASS", + "note": "pv 0.68.2, bashrs 7.4.1, pmat 3.41.1, probador 1.0.3" + }, + { + "gate": "pv-contracts", + "result": "PASS", + "note": "1878 contract(s) valid (positive control fired)" + }, + { + "gate": "pv-lint", + "result": "PASS", + "note": "Summary: 0 errors, 11531 warnings, 0 suppressed, 10054 new" + }, + { + "gate": "pv-bindings", + "result": "REPORT", + "note": "aprender: 58/60 binding functions verified in source; 2 ghost(s) \u2014 NOT gating: pv 0.49.0 misses pub(super) fn and is module-blind (paiml/provable-contracts, re-arm when fixed). Triage each ghost by ha" + }, + { + "gate": "bashrs", + "result": "PASS", + "note": "361 file(s) linted (receipt confirmed), 0 SEC/DET/IDEM errors (1 SC10xx suppressed \u2014 bashrs#226; 11 other)" + }, + { + "gate": "renacer", + "result": "SKIP", + "note": "no renacer.toml \u2014 golden tracing not configured for this crate" + }, + { + "gate": "pmat-verify", + "result": "PASS", + "note": "}" + }, + { + "gate": "pmat-comply", + "result": "PASS", + "note": "CB-200 at or under its recorded baseline \u2014 debt held flat, NOT a clean tree; 16 other fail(s), 91 skip(s) of which 3 are Error-severity (#1008). Run `pmat comply check` (without --failures-only) for t" + }, + { + "gate": "reachability", + "result": "WARN", + "note": "unreachable files/tests: 984 13213 (a file the build never compiles cannot be tested)" + }, + { + "gate": "contracts-exit-integrity", + "result": "PASS", + "note": "`contracts:` recipe propagates failure (no bare for-loop, no `|| true`)" + }, + { + "gate": "contracts", + "result": "PASS", + "note": "lint-provenance: self-test ok (markdown + yaml, marked passes, unmarked fails, exempt keys pass, count non-zero)" + }, + { + "gate": "publish-dry-run", + "result": "DEFER", + "note": "a workspace root cannot dry-run before its members are on the registry; discharged by the cascade's own per-tier publish and the post-publish dogfood" + }, + { + "gate": "release-assets", + "result": "SKIP", + "note": "assets exist only after the tag; measured by the post-publish dogfood (DOGFOOD_PHASE=post-publish), not by this phase" + }, + { + "gate": "cli-surface", + "result": "PASS", + "note": "111 advertised subcommand(s) answer --help" + }, + { + "gate": "transport-decl", + "result": "PASS", + "note": "declared: cli http mcp " + }, + { + "gate": "transport-absence", + "result": "PASS", + "note": "no undeclared mcp/http surface advertised by the binary" + }, + { + "gate": "interface-parity", + "result": "PASS", + "note": "3 transport(s), 4 e2e test(s) against the spawned release binary" + }, + { + "gate": "transport-invariance", + "result": "SKIP", + "note": "no [package.metadata.unified_surface] \u2014 declare list/cli/http/probe to enable the simultaneous cross-transport check" + }, + { + "gate": "surface-derivation", + "result": "REPORT", + "note": "no manifest declared in [package.metadata.transports] \u2014 the interface may be hand-rolled per transport; nothing here can tell single-sourcing from three lists that agree (declare manifest = { path, re" + }, + { + "gate": "probador-suite", + "result": "PASS", + "note": "PASSED 4 tests in 73.88s (4 passed, 0 failed, 0 skipped)" + }, + { + "gate": "dogfood-use", + "result": "PASS", + "note": " 7 pass / 0 fail / 0 skip" + }, + { + "gate": "model-parity-falsifier", + "result": "PASS", + "note": "the must-RED twin turns C14 RED (tests/fixtures/parity/defective/one-position-at-0.5.json)" + }, + { + "gate": "model-parity", + "result": "REPORT", + "note": "C14 UNMEASURED on noah-Lambda-Vector: C14: measured=5 rc=1 \u2014 proven on lambda and gx10 by make fleet-verify ROW=release, never here" + }, + { + "gate": "clean-room", + "result": "MANUAL", + "note": "run `make -C ../infra/machines/clean-room clean-room-aprender` (MANDATORY release gate)" + } + ], + "phase": "pre-publish", + "deferred": [ + "declared:check_multiplatform_dogfood", + "publish-dry-run" + ], + "verdict": "NO-GO" +} diff --git a/evidence/release/proof-0.69.0/gate-summary.json b/evidence/release/proof-0.69.0/gate-summary.json new file mode 100644 index 0000000000..6d597c6ae2 --- /dev/null +++ b/evidence/release/proof-0.69.0/gate-summary.json @@ -0,0 +1,180 @@ +{ + "_doc": "Summary of `pv lint contracts --gate shapes --shape release-readiness-v1 \u2026` over this directory (the command is in README.md; re-run it for the full 795-finding report). Per-cell findings are aggregated per host/model; every other finding is verbatim.", + "exit": 1, + "verdict": "Fail", + "findings": 795, + "release": { + "required_hosts": 2, + "model_receipts": 2, + "kernel_receipts": 0, + "models": 16, + "cells": 768, + "cells_with_row": 16, + "kernel_cells": 0, + "tokenizer_cells": 8, + "refusal_cells": 0, + "rung_coverage": 48, + "context_rungs": 5, + "orphan_rows": 10 + }, + "cells_by_host_model": { + "gx10/Qwen3-1.7B-Q4_K_M.gguf": { + "missing": 47, + "failing": 1, + "failing_on": [ + "answered=false", + "contextMet=false" + ] + }, + "gx10/Qwen3-8B-Q4_K_M.gguf": { + "missing": 47, + "failing": 1, + "failing_on": [ + "answered=false", + "contextMet=false" + ] + }, + "gx10/Qwen3.5-0.8B-Q4_K_M.gguf": { + "missing": 47, + "failing": 1, + "failing_on": [ + "answered=false", + "contextMet=false" + ] + }, + "gx10/Qwen3.5-27B-Q4_K_M.gguf": { + "missing": 47, + "failing": 1, + "failing_on": [ + "answered=false", + "contextMet=false" + ] + }, + "gx10/Qwen3.5-2B-Q4_K_M.gguf": { + "missing": 47, + "failing": 1, + "failing_on": [ + "answered=false", + "contextMet=false" + ] + }, + "gx10/Qwen3.5-4B-Q4_K_M.gguf": { + "missing": 47, + "failing": 1, + "failing_on": [ + "answered=false", + "contextMet=false" + ] + }, + "gx10/Qwen3.5-9B-Q4_K_M.gguf": { + "missing": 47, + "failing": 1, + "failing_on": [ + "answered=false", + "contextMet=false" + ] + }, + "gx10/qwen2.5-1.5b-instruct-q4_k_m.gguf": { + "missing": 47, + "failing": 1, + "failing_on": [ + "answered=false", + "contextMet=false" + ] + }, + "lambda/Qwen3-1.7B-Q4_K_M.gguf": { + "missing": 47, + "failing": 1, + "failing_on": [ + "answered=false", + "contextMet=false" + ] + }, + "lambda/Qwen3-8B-Q4_K_M.gguf": { + "missing": 47, + "failing": 1, + "failing_on": [ + "answered=false", + "contextMet=false", + "verdict=fail" + ] + }, + "lambda/Qwen3.5-0.8B-Q4_K_M.gguf": { + "missing": 47, + "failing": 1, + "failing_on": [ + "answered=false", + "contextMet=false" + ] + }, + "lambda/Qwen3.5-27B-Q4_K_M.gguf": { + "missing": 47, + "failing": 1, + "failing_on": [ + "answered=false", + "contextMet=false" + ] + }, + "lambda/Qwen3.5-2B-Q4_K_M.gguf": { + "missing": 47, + "failing": 1, + "failing_on": [ + "answered=false", + "contextMet=false" + ] + }, + "lambda/Qwen3.5-4B-Q4_K_M.gguf": { + "missing": 47, + "failing": 1, + "failing_on": [ + "answered=false", + "contextMet=false" + ] + }, + "lambda/Qwen3.5-9B-Q4_K_M.gguf": { + "missing": 47, + "failing": 1, + "failing_on": [ + "answered=false", + "contextMet=false" + ] + }, + "lambda/qwen2.5-1.5b-instruct-q4_k_m.gguf": { + "missing": 47, + "failing": 1, + "failing_on": [ + "answered=false", + "contextMet=false" + ] + } + }, + "other_findings": [ + "ont:release-subject/0.69.0 violates shape `release-readiness-v1.release` (node): value https://ont.paiml.dev/v1alpha1/release-dogfood/0.69.0 fails the nested shape: ont:release/verdict: \"NO-GO\" is not one of [\"GO\"]; ont:release/fresh: \"false\"^^xsd:boolean is not one of [\"true\"^^xsd:boolean]", + "ont:release-host/0.69.0/gx10 violates shape `release-readiness-v1.host` (minCount): has 0 value(s) of ont:release/dispatchKernel, minCount is 1", + "ont:release-host/0.69.0/gx10 violates shape `release-readiness-v1.host` (minCount): has 0 value(s) of ont:release/kernelReceipt, minCount is 1", + "ont:release-host/0.69.0/gx10 violates shape `release-readiness-v1.host` (maxCount): has 8 value(s) of ont:release/modelWithoutDispatch, maxCount is 0: Qwen3-1.7B-Q4_K_M.gguf, Qwen3-8B-Q4_K_M.gguf, Qwen3.5-0.8B-Q4_K_M.gguf, Qwen3.5-27B-Q4_K_M.gguf, Qwen3.5-2B-Q4_K_M.gguf", + "ont:release-host/0.69.0/gx10 violates shape `release-readiness-v1.host` (maxCount): has 5 value(s) of ont:release/unkeyedRow, maxCount is 0: evidence/release/proof-0.69.0/models/gx10.json: qwen2.5-coder-1.5b-instruct-q4_k_m.apr run think-off 4k verdict=pass, evidence/release/proof-0.69.0/models/gx10.json: qwen2.5-coder-1.5b-instruct-q4_k_m.gguf run think-off 4k verdict=pass, evidence/release/proof-0.69.0/models/gx10.json: qwen2.5-coder-1.5b-instruct-q4k.apr run think-off 4k verdict=pass, evidence/release/proof-0.69.0/models/gx10.json: qwen2.5-coder-32b-instruct-q4_k_m.gguf run think-off 4k verdict=pass, evidence/release/proof-0.69.0/models/gx10.json: qwen2.5-coder-7b-instruct-q4_k_m.gguf run think-off 4k verdict=pass", + "ont:release-host/0.69.0/gx10 violates shape `release-readiness-v1.host` (maxCount): has 6 value(s) of ont:release/unmeasuredModel, maxCount is 0: evidence/release/proof-0.69.0/models/gx10.json: Qwen3-Coder-30B-A3B-Instruct-Q4_K_M.gguf, evidence/release/proof-0.69.0/models/gx10.json: qwen2.5-coder-1.5b-instruct-q4_k_m.apr, evidence/release/proof-0.69.0/models/gx10.json: qwen2.5-coder-1.5b-instruct-q4_k_m.gguf, evidence/release/proof-0.69.0/models/gx10.json: qwen2.5-coder-1.5b-instruct-q4k.apr, evidence/release/proof-0.69.0/models/gx10.json: qwen2.5-coder-32b-instruct-q4_k_m.gguf", + "ont:release-host/0.69.0/lambda violates shape `release-readiness-v1.host` (minCount): has 0 value(s) of ont:release/dispatchKernel, minCount is 1", + "ont:release-host/0.69.0/lambda violates shape `release-readiness-v1.host` (minCount): has 0 value(s) of ont:release/kernelReceipt, minCount is 1", + "ont:release-host/0.69.0/lambda violates shape `release-readiness-v1.host` (maxCount): has 8 value(s) of ont:release/modelWithoutDispatch, maxCount is 0: Qwen3-1.7B-Q4_K_M.gguf, Qwen3-8B-Q4_K_M.gguf, Qwen3.5-0.8B-Q4_K_M.gguf, Qwen3.5-27B-Q4_K_M.gguf, Qwen3.5-2B-Q4_K_M.gguf", + "ont:release-host/0.69.0/lambda violates shape `release-readiness-v1.host` (maxCount): has 5 value(s) of ont:release/unkeyedRow, maxCount is 0: evidence/release/proof-0.69.0/models/lambda.json: Qwen3-30B-A3B-Instruct-2507-Q4_K_M.gguf run think-off 4k verdict=fail, evidence/release/proof-0.69.0/models/lambda.json: qwen2.5-coder-0.5b-instruct-q4_k_m.gguf run think-off 4k verdict=fail, evidence/release/proof-0.69.0/models/lambda.json: qwen2.5-coder-1.5b-instruct-q4k-v2.apr run think-off 4k verdict=pass, evidence/release/proof-0.69.0/models/lambda.json: qwen2.5-coder-1.5b-instruct-q4k.apr run think-off 4k verdict=pass, evidence/release/proof-0.69.0/models/lambda.json: qwen2.5-coder-7b-instruct-q4_k_m.gguf run think-off 4k verdict=fail", + "ont:release-host/0.69.0/lambda violates shape `release-readiness-v1.host` (maxCount): has 10 value(s) of ont:release/unmeasuredModel, maxCount is 0: evidence/release/proof-0.69.0/models/lambda.json: Qwen3-30B-A3B-Instruct-2507-Q4_K_M.gguf, evidence/release/proof-0.69.0/models/lambda.json: Qwen3-Coder-30B-A3B-Instruct-Q4_K_M.gguf, evidence/release/proof-0.69.0/models/lambda.json: Qwen3.5-4B-UD-Q4_K_XL.gguf, evidence/release/proof-0.69.0/models/lambda.json: qwen2.5-coder-0.5b-instruct-q4_k_m.gguf, evidence/release/proof-0.69.0/models/lambda.json: qwen2.5-coder-1.5b-instruct-q4_k_m.gguf", + "ont:model/00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4 (rung qwen35-4b-q4km) violates shape `release-readiness-v1.model` (minCount): has 0 value(s) of ont:release/contextLength, minCount is 1", + "ont:model/03b74727a860a56338e042c4420bb3f04b2fec5734175f4cb9fa853daf52b7e8 (rung qwen35-9b-q4km) violates shape `release-readiness-v1.model` (minCount): has 0 value(s) of ont:release/contextLength, minCount is 1", + "ont:model/6a1a2eb6d15622bf3c96857206351ba97e1af16c30d7a74ee38970e434e9407e (rung qwen2-1.5b-q4km) violates shape `release-readiness-v1.model` (minCount): has 0 value(s) of ont:release/contextLength, minCount is 1", + "ont:model/84b5f7f112156d63836a01a69dc3f11a6ba63b10a23b8ca7a7efaf52d5a2d806 (rung qwen35-27b-q4km) violates shape `release-readiness-v1.model` (minCount): has 0 value(s) of ont:release/contextLength, minCount is 1", + "ont:model/aaf42c8b7c3cab2bf3d69c355048d4a0ee9973d48f16c731c0520ee914699223 (rung qwen35-2b-q4km) violates shape `release-readiness-v1.model` (minCount): has 0 value(s) of ont:release/contextLength, minCount is 1", + "ont:model/b139949c5bd74937ad8ed8c8cf3d9ffb1e99c866c823204dc42c0d91fa181897 (rung qwen3-1.7b-q4km) violates shape `release-readiness-v1.model` (minCount): has 0 value(s) of ont:release/contextLength, minCount is 1", + "ont:model/bd258782e35f7f458f8aced1adc053e6e92e89bc735ba3be89d38a06121dc517 (rung qwen35-0.8b-q4km) violates shape `release-readiness-v1.model` (minCount): has 0 value(s) of ont:release/contextLength, minCount is 1", + "ont:model/d98cdcbd03e17ce47681435b5150e34c1417f50b5c0019dd560e4882c5745785 (rung qwen3-8b-q4km) violates shape `release-readiness-v1.model` (minCount): has 0 value(s) of ont:release/contextLength, minCount is 1", + "ont:release-tokenizer/0.69.0/Qwen3-1.7B-Q4_K_M.gguf violates shape `release-readiness-v1.tokenizer` (minCount): has 0 value(s) of ont:release/row, minCount is 1", + "ont:release-tokenizer/0.69.0/Qwen3-8B-Q4_K_M.gguf violates shape `release-readiness-v1.tokenizer` (minCount): has 0 value(s) of ont:release/row, minCount is 1", + "ont:release-tokenizer/0.69.0/Qwen3.5-0.8B-Q4_K_M.gguf violates shape `release-readiness-v1.tokenizer` (minCount): has 0 value(s) of ont:release/row, minCount is 1", + "ont:release-tokenizer/0.69.0/Qwen3.5-27B-Q4_K_M.gguf violates shape `release-readiness-v1.tokenizer` (minCount): has 0 value(s) of ont:release/row, minCount is 1", + "ont:release-tokenizer/0.69.0/Qwen3.5-2B-Q4_K_M.gguf violates shape `release-readiness-v1.tokenizer` (minCount): has 0 value(s) of ont:release/row, minCount is 1", + "ont:release-tokenizer/0.69.0/Qwen3.5-4B-Q4_K_M.gguf violates shape `release-readiness-v1.tokenizer` (minCount): has 0 value(s) of ont:release/row, minCount is 1", + "ont:release-tokenizer/0.69.0/Qwen3.5-9B-Q4_K_M.gguf violates shape `release-readiness-v1.tokenizer` (minCount): has 0 value(s) of ont:release/row, minCount is 1", + "ont:release-tokenizer/0.69.0/qwen2.5-1.5b-instruct-q4_k_m.gguf violates shape `release-readiness-v1.tokenizer` (minCount): has 0 value(s) of ont:release/row, minCount is 1" + ] +} \ No newline at end of file diff --git a/evidence/release/proof-0.69.0/models/gx10.json b/evidence/release/proof-0.69.0/models/gx10.json new file mode 100644 index 0000000000..f35409e9a5 --- /dev/null +++ b/evidence/release/proof-0.69.0/models/gx10.json @@ -0,0 +1,243 @@ +{ + "schema": "apr-model-ladder-receipt/v2", + "host": "gx10", + "version": "0.69.0", + "sha": "225b2a9ab", + "apr_sha": "225b2a9abbd171b3249794722f8da5fc8027e565", + "apr_version": "apr 0.69.0 (225b2a9ab)", + "cc": "12.1", + "gpu": "NVIDIA GB10", + "translated_from": [ + "rel-069-state/ladder/0.69.0/gx10.json", + "q4k-sweep-gx10.jsonl", + "inventory listing of ~/models, ~/.apr/models, ~/.cache/apr/models (2026-09-21T16:20:36Z)" + ], + "inventory": [ + { + "file": "Qwen3-1.7B-Q4_K_M.gguf", + "bytes": 1107409472, + "sha256": "b139949c5bd74937ad8ed8c8cf3d9ffb1e99c866c823204dc42c0d91fa181897", + "arch": "qwen3" + }, + { + "file": "Qwen3-8B-Q4_K_M.gguf", + "bytes": 5027783488, + "sha256": "d98cdcbd03e17ce47681435b5150e34c1417f50b5c0019dd560e4882c5745785", + "arch": "qwen3" + }, + { + "file": "Qwen3-Coder-30B-A3B-Instruct-Q4_K_M.gguf", + "bytes": 18556689568 + }, + { + "file": "Qwen3.5-0.8B-Q4_K_M.gguf", + "bytes": 532517120, + "sha256": "bd258782e35f7f458f8aced1adc053e6e92e89bc735ba3be89d38a06121dc517", + "arch": "qwen35" + }, + { + "file": "Qwen3.5-27B-Q4_K_M.gguf", + "bytes": 16740812704, + "sha256": "84b5f7f112156d63836a01a69dc3f11a6ba63b10a23b8ca7a7efaf52d5a2d806", + "arch": "qwen35" + }, + { + "file": "Qwen3.5-2B-Q4_K_M.gguf", + "bytes": 1280835840, + "sha256": "aaf42c8b7c3cab2bf3d69c355048d4a0ee9973d48f16c731c0520ee914699223", + "arch": "qwen35" + }, + { + "file": "Qwen3.5-4B-Q4_K_M.gguf", + "bytes": 2740937888, + "sha256": "00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4", + "arch": "qwen35" + }, + { + "file": "Qwen3.5-9B-Q4_K_M.gguf", + "bytes": 5680522464, + "sha256": "03b74727a860a56338e042c4420bb3f04b2fec5734175f4cb9fa853daf52b7e8", + "arch": "qwen35" + }, + { + "file": "qwen2.5-1.5b-instruct-q4_k_m.gguf", + "bytes": 1117320736, + "sha256": "6a1a2eb6d15622bf3c96857206351ba97e1af16c30d7a74ee38970e434e9407e", + "arch": "qwen2" + }, + { + "file": "qwen2.5-coder-1.5b-instruct-q4_k_m.apr", + "bytes": 1115528900 + }, + { + "file": "qwen2.5-coder-1.5b-instruct-q4_k_m.gguf", + "bytes": 1117320768 + }, + { + "file": "qwen2.5-coder-1.5b-instruct-q4k.apr", + "bytes": 1115198720 + }, + { + "file": "qwen2.5-coder-32b-instruct-q4_k_m.gguf", + "bytes": 19851335872 + }, + { + "file": "qwen2.5-coder-7b-instruct-q4_k_m.gguf", + "bytes": 4683073536 + } + ], + "cells": [ + { + "sha256": "b139949c5bd74937ad8ed8c8cf3d9ffb1e99c866c823204dc42c0d91fa181897", + "file": "Qwen3-1.7B-Q4_K_M.gguf", + "verb": "run", + "thinking": "off", + "context": "4k", + "verdict": "pass", + "backend": "cuda", + "fallback": false, + "rc": 0, + "reason": "0.69.0 ladder row (golden prompts; prompt_tokens and answer_chars not recorded)" + }, + { + "sha256": "d98cdcbd03e17ce47681435b5150e34c1417f50b5c0019dd560e4882c5745785", + "file": "Qwen3-8B-Q4_K_M.gguf", + "verb": "run", + "thinking": "off", + "context": "4k", + "verdict": "pass", + "backend": "cuda", + "fallback": false, + "rc": 0, + "reason": "0.69.0 ladder row (golden prompts; prompt_tokens and answer_chars not recorded)" + }, + { + "sha256": "bd258782e35f7f458f8aced1adc053e6e92e89bc735ba3be89d38a06121dc517", + "file": "Qwen3.5-0.8B-Q4_K_M.gguf", + "verb": "run", + "thinking": "off", + "context": "4k", + "verdict": "pass", + "backend": "cuda", + "fallback": false, + "rc": 0, + "reason": "0.69.0 ladder row (golden prompts; prompt_tokens and answer_chars not recorded)" + }, + { + "sha256": "84b5f7f112156d63836a01a69dc3f11a6ba63b10a23b8ca7a7efaf52d5a2d806", + "file": "Qwen3.5-27B-Q4_K_M.gguf", + "verb": "run", + "thinking": "off", + "context": "4k", + "verdict": "pass", + "backend": "cuda", + "fallback": false, + "rc": 0, + "reason": "0.69.0 ladder row (golden prompts; prompt_tokens and answer_chars not recorded)" + }, + { + "sha256": "aaf42c8b7c3cab2bf3d69c355048d4a0ee9973d48f16c731c0520ee914699223", + "file": "Qwen3.5-2B-Q4_K_M.gguf", + "verb": "run", + "thinking": "off", + "context": "4k", + "verdict": "pass", + "backend": "cuda", + "fallback": false, + "rc": 0, + "reason": "0.69.0 ladder row (golden prompts; prompt_tokens and answer_chars not recorded)" + }, + { + "sha256": "00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4", + "file": "Qwen3.5-4B-Q4_K_M.gguf", + "verb": "run", + "thinking": "off", + "context": "4k", + "verdict": "pass", + "backend": "cuda", + "fallback": false, + "rc": 0, + "reason": "0.69.0 ladder row (golden prompts; prompt_tokens and answer_chars not recorded)" + }, + { + "sha256": "03b74727a860a56338e042c4420bb3f04b2fec5734175f4cb9fa853daf52b7e8", + "file": "Qwen3.5-9B-Q4_K_M.gguf", + "verb": "run", + "thinking": "off", + "context": "4k", + "verdict": "pass", + "backend": "cuda", + "fallback": false, + "rc": 0, + "reason": "0.69.0 ladder row (golden prompts; prompt_tokens and answer_chars not recorded)" + }, + { + "sha256": "6a1a2eb6d15622bf3c96857206351ba97e1af16c30d7a74ee38970e434e9407e", + "file": "qwen2.5-1.5b-instruct-q4_k_m.gguf", + "verb": "run", + "thinking": "off", + "context": "4k", + "verdict": "pass", + "backend": "cuda", + "fallback": false, + "rc": 0, + "reason": "0.69.0 ladder row (golden prompts; prompt_tokens and answer_chars not recorded)" + }, + { + "file": "qwen2.5-coder-1.5b-instruct-q4k.apr", + "verb": "run", + "thinking": "off", + "context": "4k", + "verdict": "pass", + "backend": "cuda", + "fallback": false, + "rc": 0, + "reason": "0.69.0 Q4_K sweep row (no sha256 recorded): 3 golden test cases passed" + }, + { + "file": "qwen2.5-coder-1.5b-instruct-q4_k_m.apr", + "verb": "run", + "thinking": "off", + "context": "4k", + "verdict": "pass", + "backend": "cuda", + "fallback": false, + "rc": 0, + "reason": "0.69.0 Q4_K sweep row (no sha256 recorded): 3 golden test cases passed" + }, + { + "file": "qwen2.5-coder-1.5b-instruct-q4_k_m.gguf", + "verb": "run", + "thinking": "off", + "context": "4k", + "verdict": "pass", + "backend": "cuda", + "fallback": false, + "rc": 0, + "reason": "0.69.0 Q4_K sweep row (no sha256 recorded): 3 golden test cases passed" + }, + { + "file": "qwen2.5-coder-32b-instruct-q4_k_m.gguf", + "verb": "run", + "thinking": "off", + "context": "4k", + "verdict": "pass", + "backend": "cuda", + "fallback": false, + "rc": 0, + "reason": "0.69.0 Q4_K sweep row (no sha256 recorded): 3 golden test cases passed" + }, + { + "file": "qwen2.5-coder-7b-instruct-q4_k_m.gguf", + "verb": "run", + "thinking": "off", + "context": "4k", + "verdict": "pass", + "backend": "cuda", + "fallback": false, + "rc": 0, + "reason": "0.69.0 Q4_K sweep row (no sha256 recorded): 3 golden test cases passed" + } + ], + "rungs": [] +} \ No newline at end of file diff --git a/evidence/release/proof-0.69.0/models/lambda.json b/evidence/release/proof-0.69.0/models/lambda.json new file mode 100644 index 0000000000..e2680549b0 --- /dev/null +++ b/evidence/release/proof-0.69.0/models/lambda.json @@ -0,0 +1,259 @@ +{ + "schema": "apr-model-ladder-receipt/v2", + "host": "lambda", + "version": "0.69.0", + "sha": "225b2a9ab", + "apr_sha": "225b2a9abbd171b3249794722f8da5fc8027e565", + "apr_version": "apr 0.69.0 (225b2a9ab)", + "cc": "8.9", + "gpu": "NVIDIA GeForce RTX 4090", + "translated_from": [ + "rel-069-state/ladder/0.69.0/lambda.json", + "q4k-sweep-lambda.jsonl", + "inventory listing of ~/models, ~/.apr/models, ~/.cache/apr/models (2026-09-21T16:20:36Z)" + ], + "inventory": [ + { + "file": "Qwen3-1.7B-Q4_K_M.gguf", + "bytes": 1107409472, + "sha256": "b139949c5bd74937ad8ed8c8cf3d9ffb1e99c866c823204dc42c0d91fa181897", + "arch": "qwen3" + }, + { + "file": "Qwen3-30B-A3B-Instruct-2507-Q4_K_M.gguf", + "bytes": 18556686752 + }, + { + "file": "Qwen3-8B-Q4_K_M.gguf", + "bytes": 5027783488, + "sha256": "d98cdcbd03e17ce47681435b5150e34c1417f50b5c0019dd560e4882c5745785", + "arch": "qwen3" + }, + { + "file": "Qwen3-Coder-30B-A3B-Instruct-Q4_K_M.gguf", + "bytes": 18556689568 + }, + { + "file": "Qwen3.5-0.8B-Q4_K_M.gguf", + "bytes": 532517120, + "sha256": "bd258782e35f7f458f8aced1adc053e6e92e89bc735ba3be89d38a06121dc517", + "arch": "qwen35" + }, + { + "file": "Qwen3.5-27B-Q4_K_M.gguf", + "bytes": 16740812704, + "sha256": "84b5f7f112156d63836a01a69dc3f11a6ba63b10a23b8ca7a7efaf52d5a2d806", + "arch": "qwen35" + }, + { + "file": "Qwen3.5-2B-Q4_K_M.gguf", + "bytes": 1280835840, + "sha256": "aaf42c8b7c3cab2bf3d69c355048d4a0ee9973d48f16c731c0520ee914699223", + "arch": "qwen35" + }, + { + "file": "Qwen3.5-4B-Q4_K_M.gguf", + "bytes": 2740937888, + "sha256": "00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4", + "arch": "qwen35" + }, + { + "file": "Qwen3.5-4B-UD-Q4_K_XL.gguf", + "bytes": 2912109728 + }, + { + "file": "Qwen3.5-9B-Q4_K_M.gguf", + "bytes": 5680522464, + "sha256": "03b74727a860a56338e042c4420bb3f04b2fec5734175f4cb9fa853daf52b7e8", + "arch": "qwen35" + }, + { + "file": "qwen2.5-1.5b-instruct-q4_k_m.gguf", + "bytes": 1117320736, + "sha256": "6a1a2eb6d15622bf3c96857206351ba97e1af16c30d7a74ee38970e434e9407e", + "arch": "qwen2" + }, + { + "file": "qwen2.5-coder-0.5b-instruct-q4_k_m.gguf", + "bytes": 491400064 + }, + { + "file": "qwen2.5-coder-1.5b-instruct-q4_k_m.gguf", + "bytes": 1117320768 + }, + { + "file": "qwen2.5-coder-1.5b-instruct-q4k-v2.apr", + "bytes": 1115198720 + }, + { + "file": "qwen2.5-coder-1.5b-instruct-q4k.apr", + "bytes": 1115198720 + }, + { + "file": "qwen2.5-coder-7b-instruct-q4_k_m.gguf", + "bytes": 4683073536 + }, + { + "file": "qwen2.5-coder-1.5b-q4k.apr", + "bytes": 1115528900 + }, + { + "file": "tinyllama-1.1b-chat-v1.0.Q4_K_M.gguf", + "bytes": 668788096 + } + ], + "cells": [ + { + "sha256": "b139949c5bd74937ad8ed8c8cf3d9ffb1e99c866c823204dc42c0d91fa181897", + "file": "Qwen3-1.7B-Q4_K_M.gguf", + "verb": "run", + "thinking": "off", + "context": "4k", + "verdict": "pass", + "backend": "cuda", + "fallback": false, + "rc": 0, + "reason": "0.69.0 ladder row (golden prompts; prompt_tokens and answer_chars not recorded)" + }, + { + "sha256": "d98cdcbd03e17ce47681435b5150e34c1417f50b5c0019dd560e4882c5745785", + "file": "Qwen3-8B-Q4_K_M.gguf", + "verb": "run", + "thinking": "off", + "context": "4k", + "verdict": "fail", + "backend": "cuda", + "fallback": false, + "rc": 0, + "reason": "0.69.0 ladder row (golden prompts; prompt_tokens and answer_chars not recorded): golden_output: Empty output" + }, + { + "sha256": "bd258782e35f7f458f8aced1adc053e6e92e89bc735ba3be89d38a06121dc517", + "file": "Qwen3.5-0.8B-Q4_K_M.gguf", + "verb": "run", + "thinking": "off", + "context": "4k", + "verdict": "pass", + "backend": "cuda", + "fallback": false, + "rc": 0, + "reason": "0.69.0 ladder row (golden prompts; prompt_tokens and answer_chars not recorded)" + }, + { + "sha256": "84b5f7f112156d63836a01a69dc3f11a6ba63b10a23b8ca7a7efaf52d5a2d806", + "file": "Qwen3.5-27B-Q4_K_M.gguf", + "verb": "run", + "thinking": "off", + "context": "4k", + "verdict": "pass", + "backend": "cuda", + "fallback": false, + "rc": 0, + "reason": "0.69.0 ladder row (golden prompts; prompt_tokens and answer_chars not recorded)" + }, + { + "sha256": "aaf42c8b7c3cab2bf3d69c355048d4a0ee9973d48f16c731c0520ee914699223", + "file": "Qwen3.5-2B-Q4_K_M.gguf", + "verb": "run", + "thinking": "off", + "context": "4k", + "verdict": "pass", + "backend": "cuda", + "fallback": false, + "rc": 0, + "reason": "0.69.0 ladder row (golden prompts; prompt_tokens and answer_chars not recorded)" + }, + { + "sha256": "00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4", + "file": "Qwen3.5-4B-Q4_K_M.gguf", + "verb": "run", + "thinking": "off", + "context": "4k", + "verdict": "pass", + "backend": "cuda", + "fallback": false, + "rc": 0, + "reason": "0.69.0 ladder row (golden prompts; prompt_tokens and answer_chars not recorded)" + }, + { + "sha256": "03b74727a860a56338e042c4420bb3f04b2fec5734175f4cb9fa853daf52b7e8", + "file": "Qwen3.5-9B-Q4_K_M.gguf", + "verb": "run", + "thinking": "off", + "context": "4k", + "verdict": "pass", + "backend": "cuda", + "fallback": false, + "rc": 0, + "reason": "0.69.0 ladder row (golden prompts; prompt_tokens and answer_chars not recorded)" + }, + { + "sha256": "6a1a2eb6d15622bf3c96857206351ba97e1af16c30d7a74ee38970e434e9407e", + "file": "qwen2.5-1.5b-instruct-q4_k_m.gguf", + "verb": "run", + "thinking": "off", + "context": "4k", + "verdict": "pass", + "backend": "cuda", + "fallback": false, + "rc": 0, + "reason": "0.69.0 ladder row (golden prompts; prompt_tokens and answer_chars not recorded)" + }, + { + "file": "qwen2.5-coder-0.5b-instruct-q4_k_m.gguf", + "verb": "run", + "thinking": "off", + "context": "4k", + "verdict": "fail", + "backend": "cuda", + "fallback": true, + "rc": 14, + "reason": "0.69.0 Q4_K sweep row (no sha256 recorded): GPU output failed (CPU passed): golden_output_gpu: gibberish (fragment \"```\\n\" repeats 3+ times)" + }, + { + "file": "qwen2.5-coder-1.5b-instruct-q4k.apr", + "verb": "run", + "thinking": "off", + "context": "4k", + "verdict": "pass", + "backend": "cuda", + "fallback": false, + "rc": 0, + "reason": "0.69.0 Q4_K sweep row (no sha256 recorded): 3 golden test cases passed" + }, + { + "file": "qwen2.5-coder-1.5b-instruct-q4k-v2.apr", + "verb": "run", + "thinking": "off", + "context": "4k", + "verdict": "pass", + "backend": "cuda", + "fallback": false, + "rc": 0, + "reason": "0.69.0 Q4_K sweep row (no sha256 recorded): 3 golden test cases passed" + }, + { + "file": "qwen2.5-coder-7b-instruct-q4_k_m.gguf", + "verb": "run", + "thinking": "off", + "context": "4k", + "verdict": "fail", + "backend": "cuda", + "fallback": true, + "rc": 14, + "reason": "0.69.0 Q4_K sweep row (no sha256 recorded): 3 golden test cases passed" + }, + { + "file": "Qwen3-30B-A3B-Instruct-2507-Q4_K_M.gguf", + "verb": "run", + "thinking": "off", + "context": "4k", + "verdict": "fail", + "backend": "cuda", + "fallback": false, + "rc": 14, + "reason": "0.69.0 Q4_K sweep row (no sha256 recorded): gate absent" + } + ], + "rungs": [] +} \ No newline at end of file diff --git a/evidence/release/proof-0.69.0/translate.py b/evidence/release/proof-0.69.0/translate.py new file mode 100644 index 0000000000..2cfc21ab36 --- /dev/null +++ b/evidence/release/proof-0.69.0/translate.py @@ -0,0 +1,62 @@ +"""0.69.0's MEASURED evidence, carried into apr-model-ladder-receipt/v2 without inventing a measurement. + +Inputs (all measured at the 0.69.0 release, apr 0.69.0 (225b2a9ab)): + ladder rel-069-state/ladder/0.69.0/.json (v1; sha256 per rung, measured on the host) + sweep q4k-sweep-.jsonl (every Q4_K file the sweep reached; NO sha256) + holds inventory listing of the ladder contract's inventory dirs (file + bytes; NO sha256) +Rules: + * inventory = what the host holds. sha256 ONLY where the 0.69.0 ladder measured that exact file on that host; + otherwise absent (pv names it `unmeasuredModel`: the release never identified it, so never proved it). + * arch only from the ladder rung; context_length, thinking, memory arithmetic: absent (0.69.0 measured none). + * cells: the ladder row becomes the (run, think-off, 4k) row -- verdict pass iff the row was green, fallback + from backends.cuda -- with prompt_tokens and answer_chars ABSENT, because 0.69.0 ran golden prompts and + recorded neither. A sweep row for a file with no measured sha is carried UNKEYED (pv: `unkeyedRow`, verdict). + * chat / serve / code / think-on / every other rung: no row, because 0.69.0 never ran them. +""" +import json, sys +AP, SW, SP, OUT = sys.argv[1:5] +REL = "225b2a9abbd171b3249794722f8da5fc8027e565" +for host in ["lambda", "gx10"]: + lad = json.load(open(f"{AP}/ladder/0.69.0/{host}.json")) + ladder_contract = {"qwen2-1.5b-q4km": ("qwen2.5-1.5b-instruct-q4_k_m.gguf", "qwen2"), + "qwen3-1.7b-q4km": ("Qwen3-1.7B-Q4_K_M.gguf", "qwen3"), + "qwen3-8b-q4km": ("Qwen3-8B-Q4_K_M.gguf", "qwen3"), + "qwen35-0.8b-q4km": ("Qwen3.5-0.8B-Q4_K_M.gguf", "qwen35"), + "qwen35-2b-q4km": ("Qwen3.5-2B-Q4_K_M.gguf", "qwen35"), + "qwen35-4b-q4km": ("Qwen3.5-4B-Q4_K_M.gguf", "qwen35"), + "qwen35-9b-q4km": ("Qwen3.5-9B-Q4_K_M.gguf", "qwen35"), + "qwen35-27b-q4km": ("Qwen3.5-27B-Q4_K_M.gguf", "qwen35")} + by_file = {} + for r in lad["rungs"]: + f, arch = ladder_contract[r["id"]] + by_file[f] = (r, arch) + holds = json.load(open(f"{SP}/inv-{host}.json")) + inventory, cells = [], [] + for h in holds: + item = {"file": h["file"], "bytes": h["bytes"]} + if h["file"] in by_file and by_file[h["file"]][0].get("sha256"): + r, arch = by_file[h["file"]] + item["sha256"] = r["sha256"]; item["arch"] = arch + cuda = (r.get("backends") or {}).get("cuda") or {} + cells.append({"sha256": r["sha256"], "file": h["file"], "verb": "run", "thinking": "off", "context": "4k", + "verdict": "pass" if r.get("green") else "fail", "backend": "cuda", + "fallback": bool(cuda.get("fallback")), "rc": cuda.get("rc"), + "reason": "0.69.0 ladder row (golden prompts; prompt_tokens and answer_chars not recorded)" + + ("" if r.get("green") else ": " + str((r.get("golden_output") or {}).get("message", ""))[:120])}) + inventory.append(item) + for l in open(f"{SW}/q4k-sweep-{host}.jsonl"): + s = json.loads(l) + if s["model"] in by_file: + continue # the ladder row, with its measured sha, already carries it + cells.append({"file": s["model"], "verb": "run", "thinking": "off", "context": "4k", + "verdict": "pass" if s["green"] else "fail", "backend": "cuda", + "fallback": bool(s.get("gpu_fallback")), "rc": s.get("gpu_run_rc"), + "reason": "0.69.0 Q4_K sweep row (no sha256 recorded): " + + str((s.get("golden_output") or {}).get("message", ""))[:120]}) + out = {"schema": "apr-model-ladder-receipt/v2", "host": host, "version": "0.69.0", "sha": lad["sha"], + "apr_sha": REL, "apr_version": lad.get("apr_version"), "cc": lad["cc"], "gpu": lad.get("gpu"), + "translated_from": [f"rel-069-state/ladder/0.69.0/{host}.json", f"q4k-sweep-{host}.jsonl", + "inventory listing of ~/models, ~/.apr/models, ~/.cache/apr/models (2026-09-21T16:20:36Z)"], + "inventory": inventory, "cells": cells, "rungs": []} + json.dump(out, open(f"{OUT}/models/{host}.json", "w"), indent=1) + print(host, len(inventory), "held,", sum(1 for i in inventory if "sha256" in i), "hashed,", len(cells), "rows") diff --git a/evidence/tokenizer-parity/corpus/brief-review.txt b/evidence/tokenizer-parity/corpus/brief-review.txt new file mode 100644 index 0000000000..8404df6022 --- /dev/null +++ b/evidence/tokenizer-parity/corpus/brief-review.txt @@ -0,0 +1,133 @@ +You are one of 3 independent reviewers. Judge whether this diff does what its ticket says, and nothing the ticket forbids. Try to REFUTE it: default to FAIL when a test asserts the opposite of the ticket, when a gate is weakened, when a receipt claim is not backed by the diff, or when the change does something the ticket does not ask for. Every finding needs file, line, claim and grounding (cited = you quote the diff; measured = you ran a command; asserted = neither). Return PASS only if you found nothing that refutes it. + +## Ticket(s) ONT-2a + +commit f6fb83dd44838fd9217eda8a5ecb4810d1714a28 +Author: Noah Gift +Date: Mon Sep 14 01:01:33 2026 +0000 + + ONT-2a: andon — L4/L5 print self-declared until grounded (#3224) + + * ONT-2a RED: four tests for an L4 claim nothing grounds (Refs PMAT-3222) + + ONT-001 v3.1 row ONT-2a, the andon. `is_lean_proved` treats a contract's own `verification_summary` as + authoritative whenever it claims `l4_lean_proved > 0`, so a YAML file asserting its own proof IS the + evidence for that proof. These four tests say what the row requires instead, and they do not compile + against the current API — which is the point: the words the report has to print do not exist yet. + + $ cargo test -p aprender-contracts --lib proof_status + error[E0425]: cannot find function `is_l4_self_declared` in this scope + error[E0609]: no field `l4_self_declared_excluded` on type `proof_status::ProofStatusReport` + error[E0609]: no field `l4_self_declared` on type `ProofStatusTotals` + error[E0609]: no field `lean_grounded` on type `ProofStatusTotals` + error[E0609]: no field `l4_self_declared` on type `ContractProofStatus` + error[E0425]: cannot find function `is_l4_self_declared` in this scope + + The four: a claim covering every obligation with no in-tree theorem is not L4, and bindings cannot promote + it to L5; the report flags it, counts zero grounded and says the total excludes it; the text output prints + both `self-declared` and `grounded`; and a contract that claims nothing is NOT flagged, because the flag + marks an unbacked claim rather than the absence of a proof. + + Pmat-Ticket: PMAT-3222 + Co-Authored-By: Claude Opus 5 (1M context) + + * refactor(proof-status): the theorem scan was 132 cognitive, so the file could not be committed at all (Refs PMAT-3222) + + `lean_theorem_names` was one 95-line closure eight levels deep: base → domain → file → content → line → + theorem name → CamelCase → first word. `pmat analyze complexity` put it at cyclomatic 19 / cognitive 132, + against the pre-commit gate's ceiling of 30 / 25, and the gate runs over every STAGED source file — so any + change to proof_status.rs, including a comment, was refused. A gate nothing can pass is the one that + teaches people to reach for --no-verify; this makes the function honest instead. + + Six named pieces, behaviour for behaviour: insert_name_forms (the namespaced / bare / lowercased trio), + camel_case, first_camel_word, insert_theorem_names_from_content, insert_domain_theorems (which is where + `sorry` still disqualifies a file), scan_theorem_base. The base loop keeps its "first base that yields + anything wins" rule. + + before: 13 functions over the file, max cyclomatic 19, max cognitive 132 — 1 error at the gate + after: 22 functions, max cyclomatic 7, max cognitive 13 — 0 over threshold + cargo test -p aprender-contracts --lib: 1505 passed, 0 failed + + No behaviour changed here: this is the commit that makes the next one possible. + + Pmat-Ticket: PMAT-3222 + Co-Authored-By: Claude Opus 5 (1M context) + + * ONT-2a: L4 is what the tree grounds, not what the contract claims (Refs PMAT-3222) + + `is_lean_proved` took a contract's own `verification_summary` as authoritative whenever it claimed + `l4_lean_proved > 0`, and consulted the in-tree Lean scan only when it claimed nothing. A YAML file + asserting its own proof was therefore the evidence for that proof. ONT-001 R-4 (andon before mechanism) + says to withdraw that credit now, in the open, and let ONT-3b earn it back with real grounding. + + WHAT GROUNDS A CLAIM is a sorry-free in-tree Lean theorem an equation names and `lean_theorem_names()` + resolves. That is now the only thing granting L4/L5. A claim with nothing under it is SELF-DECLARED: + flagged per contract, counted in the totals, printed in its own `L4 evidence` column as `self-declared` + beside `grounded`, and excluded from L4. `not_applicable` credit still comes from the summary — it is a + claim about APPLICABILITY, not about a proof, and ONT-8's evidence block is where it gets its provenance. + + MEASURED over the corpus (1828 contracts), before → after: + + L1 1074 → 1075 L2 305 → 306 L3 417 → 442 L4 32 → 5 entries + unique L4 stems 31 → 4 self-declared and excluded: 27 + lean obligations: 181 claimed, 70 grounded + + Nothing downstream encodes those numbers: no README, no `lint-baseline.json` key, no Makefile target and + no workflow regenerates or diffs proof-status output (rg over each, 2026-09-13). + + GROUNDING IS A PARAMETER, not a hidden read. The scan resolves paths relative to the process CWD, so from + a unit test it finds nothing and every fixture is ungrounded by construction — a suite in which the andon + could withdraw ALL credit for ever and no test would notice. `*_with_grounding` takes the count and the + public wrappers do the scan, so the eight existing level tests now say which side they test; two assert + both sides on one contract. + + MUTATION (the row's own): let the claim stand in for the grounding again — `effective = claimed | grounded` + — and 4 of 47 proof_status tests fail; restored, 47 pass. Whole crate: 1505 passed, 0 failed. + + Pmat-Ticket: PMAT-3222 + Co-Authored-By: Claude Opus 5 (1M context) + + * ONT-2a: quorum agreed, 3/3 PASS (Refs PMAT-3222) + + Three independent agy lanes over the diff at 34877899d4: gemini-3.1-pro-high, gemini-3.7-flash-high and + gemini-3.6-flash-high, all PASS, all model_measured, author family claude. + + It took three rounds, and the two that failed said nothing about this diff. In both, lane 2 + (gemini-3.8-flash-high, entry 2 of the default lane_models list) returned NO-VERDICT while lanes 1 and 3 + returned substantive PASS: round 1 spent 926s narrating "waiting for the background task to complete" at + one turn, round 2 ended status=ERROR at 379s. Filed as paiml-implement#190; round 3 put + gemini-3.6-flash-high in that slot and finished normally. + + Pmat-Ticket: PMAT-3222 + Co-Authored-By: Claude Opus 5 (1M context) + + * fix(clippy): the split left an orphaned doc block and a negated is_some_and (Refs PMAT-3222) + + ci / lint failed on two errors in the ONT-2a change, both artefacts of splitting is_lean_proved into is_lean_proved_with_grounding: + + - empty line after doc comment (proof_status.rs:234) — the original function's doc block was left standing above a blank line with no item under it, and the block below it opened with an intra-doc link to [is_lean_proved], a name this file no longer defines. The blank line becomes /// so the two blocks are one doc, and the dangling link is replaced by the sentence it was trying to make. + - nonminimal_bool (proof_status.rs:430) — !path.extension().is_some_and(|e| e == "lean") is is_none_or(|e| e != "lean"). + + cargo clippy -p aprender-contracts --lib --all-features -- -D warnings: clean. cargo test -p aprender-contracts --lib: 1505 passed, 0 failed. + + Pmat-Ticket: PMAT-3222 + Co-Authored-By: Claude Opus 5 (1M context) + + * receipt(PMAT-3222): the ONT-2a andon, and why the split and the N/A credit are what they are (Refs PMAT-3222) + + A review lane refused the head on three grounds and the branch carried no receipt to answer any of them. Measured, not argued: level_strict_summary_cannot_understate_total PASSES (the lane predicted it would fail, and read its fixture's l4_lean_proved as 2 where it is 3); the L4 rule is a strengthening because the binding constraint moved from the summary's self-report to a grounding count read out of the Lean tree; and the scan was split because it stood at cognitive complexity 132 against a gate that refuses it, with the file's worst function now 13. + + Pmat-Ticket: PMAT-3222 + Co-Authored-By: Claude Opus 5 (1M context) + + * quorum(PMAT-3222): 3/3 PASS on the head that carries the receipt (Refs PMAT-3222) + +[... diff truncated for the corpus ...] + + +--- +Answer with ONE JSON object and nothing else. The object has exactly these four keys: +- "verdict": one of "PASS", "FAIL", "do-not-implement-as-written" +- "summary": a string +- "findings": an array of objects, each with "file", "line", "claim", "grounding" and "fix" +- "tree_witness": this token, copied verbatim: 6d559b3160867e925d622cf976ee6f65 diff --git a/evidence/tokenizer-parity/corpus/chat-template.txt b/evidence/tokenizer-parity/corpus/chat-template.txt new file mode 100644 index 0000000000..860c21f72a --- /dev/null +++ b/evidence/tokenizer-parity/corpus/chat-template.txt @@ -0,0 +1,13 @@ +<|im_start|>system +You are a careful code reviewer.<|im_end|> +<|im_start|>user +Why does `elapsed.as_millis()` return u128 here?<|im_end|> +<|im_start|>assistant + + + + +Because `Duration::as_millis` returns `u128`.<|im_end|> +<|im_start|>user +{"name": "read", "arguments": {"path": "src/lib.rs"}}<|im_end|> +<|im_start|>assistant diff --git a/evidence/tokenizer-parity/corpus/code-rust-token.txt b/evidence/tokenizer-parity/corpus/code-rust-token.txt new file mode 100644 index 0000000000..e8cd67543f --- /dev/null +++ b/evidence/tokenizer-parity/corpus/code-rust-token.txt @@ -0,0 +1,364 @@ + +impl GGUFModel { + + /// Get normalization epsilon from metadata + /// Different models use different values (LLaMA: 1e-5, Qwen2: 1e-6) + /// GH-278: Also checks `layer_norm_epsilon` for GPT-2/phi-2 style models + pub fn rms_epsilon(&self) -> Option { + let arch = self.architecture()?; + let rms_key = crate::gguf::keys::arch_key(arch, crate::gguf::keys::ATTENTION_LAYER_NORM_RMS_EPSILON); + if let Some(GGUFValue::Float32(eps)) = self.metadata.get(&rms_key) { + return Some(*eps); + } + let ln_key = crate::gguf::keys::arch_key(arch, crate::gguf::keys::ATTENTION_LAYER_NORM_EPSILON); + if let Some(GGUFValue::Float32(eps)) = self.metadata.get(&ln_key) { + return Some(*eps); + } + None + } + + /// Get RoPE type from metadata or infer from architecture + /// Returns: 0 = NORM (adjacent pairs), 2 = NEOX (split halves) + /// Per llama.cpp: LLAMA_ROPE_TYPE_NORM = 0, LLAMA_ROPE_TYPE_NEOX = 2 + /// + /// GH-329: Delegates to shared `infer_rope_type()` for architecture inference. + pub fn rope_type(&self) -> Option { + let arch = self.architecture()?; + let key = crate::gguf::keys::arch_key(arch, crate::gguf::keys::ROPE_SCALING_TYPE); + // Try rope type from scaling type first + if let Some(GGUFValue::String(s)) = self.metadata.get(&key) { + match s.as_str() { + "none" | "linear" => return Some(0), // NORM style + "yarn" | "neox" => return Some(2), // NEOX style + _ => {}, + } + } + // GH-329: Use shared inference function (single source of truth) + Some(crate::gguf::infer_rope_type(arch)) + } + + /// Get BOS (beginning of sentence) token ID + #[must_use] + pub fn bos_token_id(&self) -> Option { + if let Some(GGUFValue::UInt32(id)) = self.metadata.get(crate::gguf::keys::TOKENIZER_BOS_ID) { + Some(*id) + } else { + None + } + } + + /// Get EOS (end of sentence) token ID + #[must_use] + pub fn eos_token_id(&self) -> Option { + if let Some(GGUFValue::UInt32(id)) = self.metadata.get(crate::gguf::keys::TOKENIZER_EOS_ID) { + Some(*id) + } else { + None + } + } + + /// Get vocabulary tokens from metadata + /// + /// Returns the token strings indexed by token ID. + /// Uses "tokenizer.ggml.tokens" key from GGUF metadata. + #[must_use] + pub fn vocabulary(&self) -> Option> { + if let Some(GGUFValue::Array(arr)) = self.metadata.get(crate::gguf::keys::TOKENIZER_TOKENS) { + let tokens: Vec = arr + .iter() + .filter_map(|v| { + if let GGUFValue::String(s) = v { + Some(s.clone()) + } else { + None + } + }) + .collect(); + if tokens.is_empty() { + None + } else { + Some(tokens) + } + } else { + None + } + } + + /// PMAT-341: Get BPE merge rules from metadata. + /// + /// Returns merge pairs as (first, second) tuples. + /// Uses "tokenizer.ggml.merges" key from GGUF metadata. + #[must_use] + pub fn merge_rules(&self) -> Option> { + contract_pre_merge_rule!(); + if let Some(GGUFValue::Array(arr)) = self.metadata.get("tokenizer.ggml.merges") { + let merges: Vec<(String, String)> = arr + .iter() + .filter_map(|v| { + if let GGUFValue::String(s) = v { + let parts: Vec<&str> = s.splitn(2, ' ').collect(); + if parts.len() == 2 { + Some((parts[0].to_string(), parts[1].to_string())) + } else { + None + } + } else { + None + } + }) + .collect(); + if merges.is_empty() { None } else { Some(merges) } + } else { + None + } + } + + /// Decode token IDs to text using vocabulary + /// + /// Returns decoded string. Unknown tokens are replaced with "�". + /// Handles BPE markers: + /// - GPT-2 style: Ġ (U+0120) → space, Ċ (U+010A) → newline + /// - SentencePiece: ▁ (U+2581) → space + /// - Byte tokens: <0xHH> → actual byte value + #[must_use] + pub fn decode(&self, token_ids: &[u32]) -> String { + if let Some(vocab) = self.vocabulary() { + // Detect tokenizer type from metadata + let is_gpt2_style = self + .metadata + .get(crate::gguf::keys::TOKENIZER_MODEL) + .is_some_and(|v| matches!(v, GGUFValue::String(s) if s == "gpt2" || s == "bpe")); + + // Collect raw tokens and convert byte tokens to actual bytes + let mut bytes: Vec = Vec::new(); + + for &id in token_ids { + let token = vocab + .get(id as usize) + .map_or("�", std::string::String::as_str); + + // Check if this is a byte token like <0xE6> + if token.starts_with("<0x") && token.ends_with('>') && token.len() == 6 { + if let Ok(byte_val) = u8::from_str_radix(token.get(3..5).expect("byte token <0xNN> has len 6, indices 3..5 always valid"), 16) { + bytes.push(byte_val); + continue; + } + } + + // For GPT-2 style tokenizers, decode byte-level BPE properly + // Each unicode character in the token represents a raw byte + if is_gpt2_style { + for c in token.chars() { + if let Some(byte) = gpt2_unicode_to_byte(c) { + bytes.push(byte); + } + } + } else { + // SentencePiece style - tokens are regular strings + bytes.extend_from_slice(token.as_bytes()); + } + } + + // Decode bytes as UTF-8 (lossy for invalid sequences) + let raw = String::from_utf8_lossy(&bytes).into_owned(); + + // Post-process BPE markers (only for SentencePiece, GPT-2 already handled) + if !is_gpt2_style { + raw.replace('▁', " ") // SentencePiece word boundary + } else { + raw + } + } else { + // Fallback to ASCII if no vocabulary + token_ids + .iter() + .map(|&t| char::from_u32(t.min(127)).unwrap_or('?')) + .collect() + } + } + + /// Encode text to token IDs using vocabulary + /// + /// A byte-level (`tokenizer.ggml.model = "gpt2"`) vocabulary whose pre-tokenizer is + /// implemented is encoded canonically by [`crate::gguf::byte_level_bpe`]: special tokens, + /// then the model's pre-tokenizer, then its ranked merges, identical to llama.cpp (#3726). + /// Everything else still takes the greedy longest-match below, which is NOT the model's + /// tokenization; that fallback says so once on stderr. + /// Returns None if no vocabulary is available. + /// + /// Supports both tokenizer types: + /// - SentencePiece (llama): Uses `▁` (U+2581) for word boundaries + /// - GPT-2 (qwen2, gpt2): Uses `Ġ` (U+0120) for space prefixes + #[must_use] + pub fn encode(&self, text: &str) -> Option> { + let vocab = self.vocabulary()?; + + // #3726: greedy longest-match gave byte-level vocabularies ids the model never saw in + // training (" quorum" -> `Ġquo|rum`, where the merges give `Ġqu|orum`; every byte + // outside the printable glyphs -> id 0), and the model quoted identifiers back + // corrupted (#3693). + let byte_level = self + .metadata + .get("tokenizer.ggml.model") + .is_some_and(|v| matches!(v, GGUFValue::String(s) if s == "gpt2" || s == "bpe")); + if byte_level { + match crate::gguf::byte_level_bpe::ByteLevelBpe::from_gguf(&self.metadata, &vocab) { + Ok(bpe) => return Some(bpe.encode(text)), + Err(refusal) => warn_greedy_tokenizer_fallback_once(&refusal), + } + } + + // Build reverse lookup: token string -> token ID + let token_to_id: std::collections::HashMap<&str, u32> = vocab + .iter() + .enumerate() + .map(|(id, token)| (token.as_str(), id as u32)) + .collect(); + + // GH-320: Identify special tokens by pattern, not by hardcoded ID threshold. + // Matches <|...|> tokens at any ID position in the vocabulary. + let special_tokens: Vec<(&str, u32)> = vocab + .iter() + .enumerate() + .filter(|(_id, tok)| tok.starts_with("<|") && tok.ends_with("|>")) + .map(|(id, tok)| (tok.as_str(), id as u32)) + .collect(); + + // Detect tokenizer type from metadata + // GPT-2 style uses Ġ (U+0120), SentencePiece uses ▁ (U+2581) + let is_gpt2_style = self + .metadata + .get("tokenizer.ggml.model") + .is_some_and(|v| matches!(v, GGUFValue::String(s) if s == "gpt2" || s == "bpe")); + + let space_char = if is_gpt2_style { '\u{0120}' } else { '▁' }; + + // Split text on special tokens first, preserving them + let mut segments: Vec<(bool, &str)> = Vec::new(); // (is_special, text) + let mut text_remaining = text; + while !text_remaining.is_empty() { + // Find earliest special token match + let mut earliest_match: Option<(usize, &str, u32)> = None; + for &(special_tok, special_id) in &special_tokens { + if let Some(pos) = text_remaining.find(special_tok) { + if earliest_match.is_none() + || pos < earliest_match.as_ref().map_or(usize::MAX, |m| m.0) + { + earliest_match = Some((pos, special_tok, special_id)); + } + } + } + + if let Some((pos, special_tok, _)) = earliest_match { + if pos > 0 { + segments.push((false, &text_remaining[..pos])); + } + segments.push((true, special_tok)); + text_remaining = &text_remaining[pos + special_tok.len()..]; + } else { + segments.push((false, text_remaining)); + break; + } + } + + let mut tokens = Vec::new(); + + for (is_special, segment) in segments { + if is_special { + // Direct lookup for special token + if let Some(&id) = token_to_id.get(segment) { + tokens.push(id); + } + continue; + } + + // Process non-special segment with character replacement + let text_with_prefix = if is_gpt2_style { + segment.to_string() + } else if segment.starts_with(' ') { + segment.to_string() + } else { + format!(" {}", segment) + }; + + let processed = if is_gpt2_style { + text_with_prefix + .replace(' ', &space_char.to_string()) + .replace('\n', "\u{010A}") // Ċ = GPT-2 newline + } else { + text_with_prefix.replace(' ', &space_char.to_string()) + }; + + let mut remaining = processed.as_str(); + + while !remaining.is_empty() { + // Greedy longest match using character boundaries (not byte indices) + let mut best_byte_len = 0; + let mut best_id = None; + + // Collect character byte offsets for proper slicing + let char_indices: Vec = remaining + .char_indices() + .map(|(i, _)| i) + .chain(std::iter::once(remaining.len())) + .collect(); + + // Try all prefixes up to 32 chars (reasonable max token length) + for char_count in 1..=char_indices.len().saturating_sub(1).min(32) { + let byte_end = char_indices[char_count]; + let prefix = &remaining[..byte_end]; + if let Some(&id) = token_to_id.get(prefix) { + best_byte_len = byte_end; + best_id = Some(id); + } + } + + if let Some(id) = best_id { + tokens.push(id); + remaining = &remaining[best_byte_len..]; + } else { + // No match found - try single UTF-8 char as byte tokens + // SAFETY: remaining is non-empty (loop condition guarantees this) + let ch = remaining + .chars() + .next() + .expect("loop invariant: remaining non-empty"); + let ch_len = ch.len_utf8(); + + // Look for byte tokens like <0x48> for 'H' + for byte in remaining[..ch_len].bytes() { + let byte_token = format!("<0x{:02X}>", byte); + if let Some(&id) = token_to_id.get(byte_token.as_str()) { + tokens.push(id); + } else { + // Unknown byte - use a common unknown token ID (usually 0 or 1) + tokens.push(0); + } + } + remaining = &remaining[ch_len..]; + } + } + } + + Some(tokens) + } +} + +use crate::gguf::{ + OwnedQKVWeights, OwnedQuantizedLayer, OwnedQuantizedModel, OwnedQuantizedTensor, + QuantizedGGUFTransformer, +}; + +include!("loader_parse.rs"); +include!("metadata.rs"); + +/// The greedy longest-match fallback is not the model's tokenization; say so, once (#3726). +fn warn_greedy_tokenizer_fallback_once(refusal: &crate::gguf::byte_level_bpe::ByteLevelBpeRefusal) { + static WARNED: std::sync::Once = std::sync::Once::new(); + WARNED.call_once(|| { + eprintln!( + "warning: byte-level BPE tokenizer: {refusal}; falling back to greedy longest-match, \ + which does NOT reproduce the model's tokenization (#3726)" + ); + }); +} diff --git a/evidence/tokenizer-parity/corpus/identifiers.txt b/evidence/tokenizer-parity/corpus/identifiers.txt new file mode 100644 index 0000000000..ce85f4a55d --- /dev/null +++ b/evidence/tokenizer-parity/corpus/identifiers.txt @@ -0,0 +1,11 @@ +machines/lambda-labs/forjar.yaml crates/prah-lanes/src/local/admit.rs +elapsed.as_millis() u64::saturating_sub apr_inference_time_ms Backend::Cuda +quorum quoorum forbids refutes cited paiml-implement -skill +tree_witness: 83a760139b7d7d73f583c45f0b2fc630 sha256 03b74727a860a56338e042c4420bb3f04b2fec5734175f4cb9fa853daf52b7e8 +index bed4ce6b..9f2519ef 100644 @@ -6715,3 +6715,21 @@ +2026-09-21T14:18:59Z 3.14159 1e-6 0x7fffffff -42 +17 1,234,567.89 v0.69.0 #3726 PMAT-3693 +camelCaseIdentifier snake_case_identifier SCREAMING_CASE kebab-case-name __dunder__ +https://github.com/paiml/aprender/issues/3693#issuecomment-5763494060 +it's they're we've I'm you'll he'd DON'T IT'S +!!! ??? ... --- === +++ ### *** /// \\ ~~~ ``` <<< >>> ||| &&& +`code` "quoted" 'single' (parens) [brackets] {braces} diff --git a/evidence/tokenizer-parity/corpus/nonascii.txt b/evidence/tokenizer-parity/corpus/nonascii.txt new file mode 100644 index 0000000000..3d3c698e45 --- /dev/null +++ b/evidence/tokenizer-parity/corpus/nonascii.txt @@ -0,0 +1,12 @@ +CJK: 中文分词测试，日本語のテキスト、한국어 문장입니다。 +Accented Latin: café naïve Ærøskøbing Straße ñandú façade coöperate Dvořák Łódź. +Emoji: 👋🏽 🧑‍💻 ❤️ 🇺🇸 🇩🇪 👨‍👩‍👧‍👦 ✅ 🚀 1️⃣ +Box drawing: ──────── ┌─┬─┐ │ │ │ └─┴─┘ ═══ ╔╗╚╝ ▓▒░█ +Zero-width: a​b c‌d e‍f g word⁠joiner +Combining marks: é ä ñ Tiếng Việt có dấu, ǘ, Z̧͑ä͞lgo +RTL: العربية نص تجريبي — עברית טקסט +Indic and Thai: नमस्ते दुनिया, தமிழ், สวัสดีครับ +Cyrillic and Greek: Привет, мир! Αλφάβητο αβγδε. +Fullwidth and math: ＡＢＣ１２３ ∑∫∂≠≤≥∞ √2 ≈ 1.41421 ± 0.00001 → ⇒ ⟨x⟩ +Spaces: a b  c d　e f
g h i +Smart quotes: “double” ‘single’ «guillemets» „low” — en–dash … ellipsis diff --git a/evidence/tokenizer-parity/corpus/whitespace.txt b/evidence/tokenizer-parity/corpus/whitespace.txt new file mode 100644 index 0000000000..590acda8f6 --- /dev/null +++ b/evidence/tokenizer-parity/corpus/whitespace.txt @@ -0,0 +1,19 @@ +leading spaces + indented four + one tab + two tabs +trailing spaces +crlf line +another + +blank crlf above +mixed tabs and spaces + + + +three newlines above + + + spaces-only lines above +form feed and vertical tab +end with spaces \ No newline at end of file diff --git a/evidence/tokenizer-parity/mutant-id0-lambda.txt b/evidence/tokenizer-parity/mutant-id0-lambda.txt new file mode 100644 index 0000000000..94a724e5fd --- /dev/null +++ b/evidence/tokenizer-parity/mutant-id0-lambda.txt @@ -0,0 +1,57 @@ +# #3726 MUTANT: the old id-0 byte fallback reintroduced in byte_level_bpe.rs (bytes >= 0x80 -> id 0); the gate must be RED +# host: noah-Lambda-Vector measured: 2026-09-21T16:5xZ +# command: bash scripts/tokenizer_parity.sh --model Qwen3.5-9B-Q4_K_M.gguf --model Qwen3-8B-Q4_K_M.gguf (11 corpus files) + +apr: /mnt/nvme-raid0/targets/c3-3726/release/apr (apr 0.69.0 (262dfb2f6)) +llama-tokenize: /mnt/nvme-raid0/work-c3/llama-cpu-build/bin/llama-tokenize (version: 0.4.1-dev (build 10987, commit d1d3c3396)) +models: 2 corpus files: 11 +FAIL Qwen3.5-9B-Q4_K_M.gguf brief-review.txt first id mismatch at 214 (apr 6774 ids, llama.cpp 6774) + apr: 64 25 321 263 [31780] 436 19 7199 20 + llama.cpp: 64 25 321 263 [1892] 436 19 7199 20 +FAIL Qwen3.5-9B-Q4_K_M.gguf code-rust-token.txt first id mismatch at 1170 (apr 3678 ids, llama.cpp 3690) + apr: 513 12215 440 328 [11726] 3158 198 262 1013 + llama.cpp: 513 12215 440 328 [5513] 3158 198 262 1013 +FAIL Qwen3.5-9B-Q4_K_M.gguf code-shell-llama_bin.txt first id mismatch at 4 (apr 7633 ids, llama.cpp 7633) + apr: 2 90448 21144 2318 [31780] 8563 279 90448 6982 + llama.cpp: 2 90448 21144 2318 [1892] 8563 279 90448 6982 +FAIL Qwen3.5-9B-Q4_K_M.gguf nonascii.txt first id mismatch at 2 (apr 308 ids, llama.cpp 358) + apr: 86274 25 [729] 48962 16582 67437 48962 + llama.cpp: 86274 25 [220] 99986 95880 96919 99449 +FAIL Qwen3.5-9B-Q4_K_M.gguf brief18.txt first id mismatch at 128 (apr 6267 ids, llama.cpp 6264) + apr: 12 23 24 16 [31780] 279 3515 369 42912 + llama.cpp: 12 23 24 16 [1892] 279 3515 369 42912 +FAIL Qwen3.5-9B-Q4_K_M.gguf lane.json.prompt.txt first id mismatch at 339 (apr 13498 ids, llama.cpp 13495) + apr: 31930 2136 16208 63 [31780] 1189 411 1558 11 + llama.cpp: 31930 2136 16208 63 [1892] 1189 411 1558 11 +FAIL Qwen3.5-9B-Q4_K_M.gguf lane.json.prompt.txt first id mismatch at 128 (apr 6267 ids, llama.cpp 6264) + apr: 12 23 24 16 [31780] 279 3515 369 42912 + llama.cpp: 12 23 24 16 [1892] 279 3515 369 42912 +FAIL Qwen3-8B-Q4_K_M.gguf brief-review.txt first id mismatch at 212 (apr 6475 ids, llama.cpp 6475) + apr: 64 25 323 263 [32870] 444 19 7434 20 + llama.cpp: 64 25 323 263 [1959] 444 19 7434 20 +FAIL Qwen3-8B-Q4_K_M.gguf code-rust-token.txt first id mismatch at 1084 (apr 3415 ids, llama.cpp 3423) + apr: 525 12575 448 330 [2928] 8958 624 262 1048 + llama.cpp: 525 12575 448 330 [5691] 22956 262 1048 21008 +FAIL Qwen3-8B-Q4_K_M.gguf code-shell-llama_bin.txt first id mismatch at 4 (apr 7402 ids, llama.cpp 7405) + apr: 2 93676 21816 2395 [32870] 8830 279 93676 7208 + llama.cpp: 2 93676 21816 2395 [1959] 8830 279 93676 7208 +FAIL Qwen3-8B-Q4_K_M.gguf nonascii.txt first id mismatch at 2 (apr 312 ids, llama.cpp 364) + apr: 89349 25 [753] 50667 17085 69800 50667 + llama.cpp: 89349 25 [72858] 16744 17177 99689 81705 +FAIL Qwen3-8B-Q4_K_M.gguf brief18.txt first id mismatch at 127 (apr 6114 ids, llama.cpp 6111) + apr: 12 23 24 16 [32870] 279 3638 374 44387 + llama.cpp: 12 23 24 16 [1959] 279 3638 374 44387 +FAIL Qwen3-8B-Q4_K_M.gguf lane.json.prompt.txt first id mismatch at 335 (apr 12484 ids, llama.cpp 12483) + apr: 33025 2205 16698 63 [32870] 1231 419 1614 11 + llama.cpp: 33025 2205 16698 63 [1959] 1231 419 1614 11 +FAIL Qwen3-8B-Q4_K_M.gguf lane.json.prompt.txt first id mismatch at 127 (apr 6114 ids, llama.cpp 6111) + apr: 12 23 24 16 [32870] 279 3638 374 44387 + llama.cpp: 12 23 24 16 [1959] 279 3638 374 44387 +# PASS lines condensed: 8 PASS over 2 model(s), 4 file(s) each. +# full per-pair log: lambda:/mnt/nvme-raid0/work-c3/3726-mutant-id0.log +# model files-passed total-ids +# Qwen3.5-9B-Q4_K_M.gguf 4 6774 +# Qwen3-8B-Q4_K_M.gguf 4 6619 +# files: brief18-ascii.txt, chat-template.txt, identifiers.txt, whitespace.txt +--- tokenizer parity: 8 pass, 14 fail, 0 uncovered --- +rc=1 elapsed=70s diff --git a/evidence/tokenizer-parity/receipt-gx10-9026b70a5.txt b/evidence/tokenizer-parity/receipt-gx10-9026b70a5.txt new file mode 100644 index 0000000000..0f6f76b20c --- /dev/null +++ b/evidence/tokenizer-parity/receipt-gx10-9026b70a5.txt @@ -0,0 +1,25 @@ +# #3726 tokenizer parity on gx10 (GB10, aarch64) with the 4 real briefs; the pinned build prints its commit as d1d3c339 +# host: gx10-a5b5 measured: 2026-09-21T17:22:16Z +# command: bash scripts/tokenizer_parity.sh --apr --corpus <7 committed + 4 briefs> + +apr: /mnt/nvme-raid0/work-c3/apr-fixed-9026b70a5 (apr 0.69.0 (9026b70a5)) +llama-tokenize: /home/noah/src/llama.cpp-d1d3c3396/build/bin/llama-tokenize (version: 0.4.1-dev (build 2423, commit d1d3c339)) +models: 12 corpus files: 11 +# PASS lines condensed: 132 PASS over 12 model(s), 11 file(s) each. +# full per-pair log: gx10:/mnt/nvme-raid0/work-c3/3726-parity-gx10.log +# model files-passed total-ids +# qwen2.5-1.5b-instruct-q4_k_m.gguf 11 48993 +# qwen2.5-coder-1.5b-instruct-q4_k_m.gguf 11 48993 +# qwen2.5-coder-32b-instruct-q4_k_m.gguf 11 48993 +# qwen2.5-coder-7b-instruct-q4_k_m.gguf 11 48993 +# Qwen3-1.7B-Q4_K_M.gguf 11 48991 +# Qwen3.5-0.8B-Q4_K_M.gguf 11 51252 +# Qwen3.5-27B-Q4_K_M.gguf 11 51252 +# Qwen3.5-2B-Q4_K_M.gguf 11 51252 +# Qwen3.5-4B-Q4_K_M.gguf 11 51252 +# Qwen3.5-9B-Q4_K_M.gguf 11 51252 +# Qwen3-8B-Q4_K_M.gguf 11 48991 +# Qwen3-Coder-30B-A3B-Instruct-Q4_K_M.gguf 11 48991 +# files: brief-review.txt, brief18-ascii.txt, brief18.txt, c9-lane.txt, chat-template.txt, code-rust-token.txt, code-shell-llama_bin.txt, identifiers.txt, nonascii.txt, r891-lane.txt, whitespace.txt +--- tokenizer parity: 132 pass, 0 fail, 0 uncovered --- +rc=0 diff --git a/evidence/tokenizer-parity/receipt-lambda-9026b70a5-with-briefs.txt b/evidence/tokenizer-parity/receipt-lambda-9026b70a5-with-briefs.txt new file mode 100644 index 0000000000..213fa9553e --- /dev/null +++ b/evidence/tokenizer-parity/receipt-lambda-9026b70a5-with-briefs.txt @@ -0,0 +1,34 @@ +# #3726 tokenizer parity with 4 real consumer briefs (read in place, not committed: they quote private-repo diffs) +# host: noah-Lambda-Vector measured: 2026-09-21T17:34:37Z +# command: bash scripts/tokenizer_parity.sh --apr --llama-tokenize --corpus <6 committed> --corpus brief18.txt brief18-ascii.txt + +apr: /mnt/nvme-raid0/work-c3/bin/apr-fixed-9026b70a5 (apr 0.69.0 (9026b70a5)) +llama-tokenize: /mnt/nvme-raid0/work-c3/llama-cpu-build/bin/llama-tokenize (version: 0.4.1-dev (build 10987, commit d1d3c3396)) +models: 21 corpus files: 10 +# PASS lines condensed: 210 PASS over 21 model(s), 10 file(s) each (9 distinct basenames: two lane prompts share one). +# full per-pair log: lambda:/mnt/nvme-raid0/work-c3/3726-parity-lambda-briefs-final.log +# model files-passed total-ids +# Qwen2.5-0.5B-Instruct-f16.gguf 10 37761 +# Qwen2.5-0.5B-Instruct-IQ3_M.gguf 10 37761 +# Qwen2.5-0.5B-Instruct-IQ4_XS.gguf 10 37761 +# qwen2.5-1.5b-instruct-q4_k_m.gguf 10 37761 +# qwen2.5-coder-0.5b-instruct-q4_k_m.gguf 10 37761 +# qwen2.5-coder-1.5b-instruct-q4_k_m.gguf 10 37761 +# qwen2.5-coder-7b-instruct-q4_k_m.gguf 10 37761 +# Qwen3-0.6B-BF16.gguf 10 37759 +# Qwen3-1.7B-Q4_K_M.gguf 10 37759 +# Qwen3-30B-A3B-Instruct-2507-Q4_K_M.gguf 10 37759 +# Qwen3.5-0.8B-IQ4_XS.gguf 10 39540 +# Qwen3.5-0.8B-Q4_K_M.gguf 10 39540 +# Qwen3.5-0.8B-UD-IQ2_XXS.gguf 10 39540 +# Qwen3.5-27B-Q4_K_M.gguf 10 39540 +# Qwen3.5-2B-Q4_K_M.gguf 10 39540 +# Qwen3.5-35B-A3B-UD-IQ4_XS.gguf 10 39540 +# Qwen3.5-4B-Q4_K_M.gguf 10 39540 +# Qwen3.5-4B-UD-Q4_K_XL.gguf 10 39540 +# Qwen3.5-9B-Q4_K_M.gguf 10 39540 +# Qwen3-8B-Q4_K_M.gguf 10 37759 +# Qwen3-Coder-30B-A3B-Instruct-Q4_K_M.gguf 10 37759 +# files: brief-review.txt, brief18-ascii.txt, brief18.txt, chat-template.txt, code-rust-token.txt, identifiers.txt, lane.json.prompt.txt, nonascii.txt, whitespace.txt +--- tokenizer parity: 210 pass, 0 fail, 0 uncovered --- +rc=0 elapsed=549s diff --git a/evidence/tokenizer-parity/receipt-lambda-9026b70a5.txt b/evidence/tokenizer-parity/receipt-lambda-9026b70a5.txt new file mode 100644 index 0000000000..e0cd22b55a --- /dev/null +++ b/evidence/tokenizer-parity/receipt-lambda-9026b70a5.txt @@ -0,0 +1,34 @@ +# #3726 tokenizer parity, release mode (defaults: pinned llama-tokenize, ~/models, committed corpus) +# host: noah-Lambda-Vector measured: 2026-09-21T17:22:15Z +# command: bash scripts/tokenizer_parity.sh --apr + +apr: /mnt/nvme-raid0/work-c3/bin/apr-fixed-9026b70a5 (apr 0.69.0 (9026b70a5)) +llama-tokenize: /home/noah/src/llama.cpp-d1d3c3396/build/bin/llama-tokenize (version: 0.4.1-dev (build 10987, commit d1d3c3396)) +models: 21 corpus files: 7 +# PASS lines condensed: 147 PASS over 21 model(s), 7 file(s) each. +# full per-pair log: lambda:/mnt/nvme-raid0/work-c3/3726-parity-lambda-final.log +# model files-passed total-ids +# Qwen2.5-0.5B-Instruct-f16.gguf 7 18200 +# Qwen2.5-0.5B-Instruct-IQ3_M.gguf 7 18200 +# Qwen2.5-0.5B-Instruct-IQ4_XS.gguf 7 18200 +# qwen2.5-1.5b-instruct-q4_k_m.gguf 7 18200 +# qwen2.5-coder-0.5b-instruct-q4_k_m.gguf 7 18200 +# qwen2.5-coder-1.5b-instruct-q4_k_m.gguf 7 18200 +# qwen2.5-coder-7b-instruct-q4_k_m.gguf 7 18200 +# Qwen3-0.6B-BF16.gguf 7 18198 +# Qwen3-1.7B-Q4_K_M.gguf 7 18198 +# Qwen3-30B-A3B-Instruct-2507-Q4_K_M.gguf 7 18198 +# Qwen3.5-0.8B-IQ4_XS.gguf 7 18990 +# Qwen3.5-0.8B-Q4_K_M.gguf 7 18990 +# Qwen3.5-0.8B-UD-IQ2_XXS.gguf 7 18990 +# Qwen3.5-27B-Q4_K_M.gguf 7 18990 +# Qwen3.5-2B-Q4_K_M.gguf 7 18990 +# Qwen3.5-35B-A3B-UD-IQ4_XS.gguf 7 18990 +# Qwen3.5-4B-Q4_K_M.gguf 7 18990 +# Qwen3.5-4B-UD-Q4_K_XL.gguf 7 18990 +# Qwen3.5-9B-Q4_K_M.gguf 7 18990 +# Qwen3-8B-Q4_K_M.gguf 7 18198 +# Qwen3-Coder-30B-A3B-Instruct-Q4_K_M.gguf 7 18198 +# files: brief-review.txt, chat-template.txt, code-rust-token.txt, code-shell-llama_bin.txt, identifiers.txt, nonascii.txt, whitespace.txt +--- tokenizer parity: 147 pass, 0 fail, 0 uncovered --- +rc=0 diff --git a/scripts/check_baseline_ratchets.sh b/scripts/check_baseline_ratchets.sh index 40c27e21f2..d8dfc93bc6 100644 --- a/scripts/check_baseline_ratchets.sh +++ b/scripts/check_baseline_ratchets.sh @@ -94,6 +94,7 @@ classify() { # classify -> "[reason]", rc 1 if unclassifie test_fixture_path_baseline.txt) printf 'count\n' ;; tracked_ignored_baseline.txt) printf 'count\n' ;; unwired_guards_baseline.txt) printf 'set-aperture\tscripts/check_guards_are_wired.sh\n' ;; # NAME entries: a guard file that predates the comparand may be ledgered when the meta-guard itself widens (#3644) + wallclock_assert_baseline.txt) printf 'keyed\n' ;; # #3703: ; keyed on TEXT so a line moving is not growth # NOT a ratchet either, and for the same reason one level along: this # registry is DERIVED from the test sources on every run # (scripts/check_tree_reader_tests.sh) and must equal that derivation diff --git a/scripts/check_ci_gate_mutants_rule.sh b/scripts/check_ci_gate_mutants_rule.sh new file mode 100644 index 0000000000..544a64b362 --- /dev/null +++ b/scripts/check_ci_gate_mutants_rule.sh @@ -0,0 +1,79 @@ +#!/usr/bin/env bash +# check_ci_gate_mutants_rule.sh -- the `gate` job accepts a SKIPPED mutants job only +# when the skip is that job's own filter (#3676). +# +# mutants runs only on pull_request (`if: github.event_name == 'pull_request'`), +# so on merge_group, push and workflow_dispatch it is skipped by design. On a +# pull_request it must RUN: a diff with no .rs still succeeds, because its +# mutation steps skip and the job passes. The old rule read "success/skipped both +# pass", which also passed a pull_request whose mutants job never ran at all. +# +# This guard does not re-implement the rule. It EXTRACTS the block between the +# GATE-MUTANTS-RULE-BEGIN/END markers from .github/workflows/ci.yml and executes +# it for every (event, result) pair, so it judges the code the gate runs. A +# missing or empty block is ENV rc=2, never a pass. +# +# check_ci_gate_mutants_rule.sh the case table over ci.yml's block +# check_ci_gate_mutants_rule.sh --self-test the old rule, planted: it must go RED +set -uo pipefail +ROOT="$(cd "$(dirname "$0")/.." && pwd)" || exit 2 +WF="${GATE_RULE_WORKFLOW:-$ROOT/.github/workflows/ci.yml}" + +# extract -> the rule block, dedented; empty when the markers are absent +extract() { + awk '/GATE-MUTANTS-RULE-BEGIN/{f=1; next} /GATE-MUTANTS-RULE-END/{f=0} f' "$1" | sed -E 's/^ {10}//' +} + +# verdict -> exit status of the block with EVT/MUT set +verdict() { + EVT=$2 MUT=$3 bash -c "$1" >/dev/null 2>&1 +} + +table() { # table -> 0 iff every row holds + local blk bad=0 n=0 evt mut want got + blk=$(extract "$1") + [ -n "$blk" ] || { printf 'ENV no GATE-MUTANTS-RULE block in %s -- cannot judge, not a pass\n' "$1" >&2; return 2; } + while read -r evt mut want; do + [ -n "$evt" ] || continue + n=$((n + 1)); got=0; verdict "$blk" "$evt" "$mut" || got=1 + if [ "$got" = "$want" ]; then printf 'ok row %-2s %-14s %-10s -> %s\n' "$n" "$evt" "$mut" "$([ "$want" = 0 ] && echo pass || echo FAIL)" + else printf 'FAIL row %-2s %-14s %-10s wanted %s, got %s\n' "$n" "$evt" "$mut" "$want" "$got" >&2; bad=1; fi + done <<'ROWS' +pull_request success 0 +pull_request skipped 1 +pull_request cancelled 1 +pull_request failure 1 +merge_group skipped 0 +push skipped 0 +workflow_dispatch skipped 0 +merge_group success 0 +merge_group failure 1 +merge_group cancelled 1 +ROWS + return "$bad" +} + +case "${1:-}" in -h|--help) sed -n '2,18p' "$0" | sed 's/^# \{0,1\}//'; exit 0 ;; esac + +if [ "${1:-}" = "--self-test" ]; then + echo "=== gate mutants rule: the old rule must turn the table RED ===" + d=$(mktemp -d "${TMPDIR:-/tmp}/gate-rule.XXXXXX") || exit 2 + rmtree() { case "${1:-}" in ''|/) return 0 ;; *) [ -d "$1" ] && rm -rf -- "$1" ;; esac; return 0; } + trap 'rmtree "${d:-}"' EXIT + printf ' # --- GATE-MUTANTS-RULE-BEGIN ---\n if [ "$MUT" = "failure" ]; then exit 1; fi\n # --- GATE-MUTANTS-RULE-END ---\n' > "$d/old.yml" + printf 'jobs: {}\n' > "$d/none.yml" + bad=0 + if table "$d/old.yml" > "$d/out" 2>&1; then printf 'FAIL the OLD rule ("success/skipped both pass") passed the table\n'; bad=1 + else printf 'ok the OLD rule is RED: %s\n' "$(grep -c '^FAIL' "$d/out") row(s), e.g. $(grep -m1 '^FAIL' "$d/out" | cut -c7-70)"; fi + rc=0; table "$d/none.yml" > /dev/null 2>&1 || rc=$? + [ "$rc" = 2 ] && printf 'ok a workflow with no rule block is ENV rc=2, never a pass\n' || { printf 'FAIL no rule block gave rc=%s\n' "$rc"; bad=1; } + [ "$bad" = 0 ] && { echo "SELF-TEST PASSED"; exit 0; } + echo "SELF-TEST FAILED" >&2; exit 1 +fi + +echo "=== gate accepts a skipped mutants job only when the skip is its filter (check_ci_gate_mutants_rule.sh) ===" +# the workflow is needed only here: --self-test builds its own fixtures (quorum lane 1 on #3688) +[ -f "$WF" ] || { printf 'ENV %s is missing\n' "$WF" >&2; exit 2; } +table "$WF"; rc=$? +[ "$rc" = 0 ] && echo PASS || echo "FAIL (rc=$rc)" >&2 +exit "$rc" diff --git a/scripts/check_coverage_has_producers.sh b/scripts/check_coverage_has_producers.sh new file mode 100644 index 0000000000..97758facb3 --- /dev/null +++ b/scripts/check_coverage_has_producers.sh @@ -0,0 +1,139 @@ +#!/usr/bin/env bash +# check_coverage_has_producers.sh -- every consumer of aprender's coverage number +# still has a producer (#3676). +# +# Operator 2026-09-21: "YES, coverage on tags release only." `ci / coverage` left +# the PR / merge-group / push path: it had gated nothing and measured 0 tests on +# the facade root. A deletion like that can starve a gate that READ the number +# from CI -- rmedia's release gate did exactly that ("could not look"). So the +# chain is asserted, not remembered: +# +# R1 `make -n coverage-check` reaches llvm-cov -- the PRODUCER the +# pre-publish dogfood uses measures coverage itself, reads nothing from CI. +# R2 scripts/dogfood.sh still runs `make ... coverage-check` -- the release +# CONSUMER exists (without it the chain would be vacuously intact). +# R3 .github/workflows/coverage-nightly.yml triggers on `schedule:` and NOT on +# `v*` tags -- the nightly trend stays, and a tag is measured ONCE, by ci.yml. +# R4 the Makefile defines a numeric COV_FLOOR -- the floor `make coverage` enforces. +# R5 ci.yml hands sovereign-ci `coverage_on: tag` (no `skip_coverage: true`) +# AND its own `on:` has `push: tags: ['v*']` -- the reusable's two required +# edits. sovereign-ci's gate then prints "coverage: NOT MEASURED" on a pull +# request and REQUIRES coverage to succeed on a v* tag. `skip_coverage: true` +# was the first spelling (#3688): that gate counts a skipped coverage job as +# a mandatory failure, so it turned `ci / gate` red on every PR. +# Any link missing is FAIL; a file that cannot be read is ENV rc=2. +# +# check_coverage_has_producers.sh judge the tree +# check_coverage_has_producers.sh --self-test each rule against fixtures that break it +# +# Test seams: COVCHAIN_ROOT (the tree judged). +set -uo pipefail +ROOT="${COVCHAIN_ROOT:-$(cd "$(dirname "$0")/.." && pwd)}" + +# on_block -> the lines of its top-level `on:` block, comments stripped +on_block() { + awk '/^on:/{f=1; next} f && /^[^[:space:]#]/{exit} f' "$1" | sed 's/#.*$//' +} +# live -> the file with comments stripped (a setting in a comment is not a setting) +live() { sed 's/#.*$//' "$1"; } +V_TAGS_RE="^[[:space:]]+tags:[[:space:]]*\[[^]]*['\"]?v\*['\"]?" + +judge() { # judge -> 0 all links hold, 1 a link broke, 2 ENV + local r=$1 bad=0 dry blk ci + local mk="$r/Makefile" df="$r/scripts/dogfood.sh" wf="$r/.github/workflows/coverage-nightly.yml" cy="$r/.github/workflows/ci.yml" + for f in "$mk" "$df" "$wf" "$cy"; do [ -r "$f" ] || { printf 'ENV %s is not readable -- cannot judge, not a pass\n' "$f"; return 2; }; done + + dry=$(make -n -C "$r" coverage-check 2>/dev/null) || dry="" + if [[ $dry == *"llvm-cov"* ]]; then printf 'ok R1 make -n coverage-check reaches llvm-cov (the release producer measures, it reads nothing from CI)\n' + else printf 'FAIL R1 make -n coverage-check does not reach llvm-cov -- the dogfood coverage gate would measure nothing\n'; bad=1; fi + + if grep -qE '^[[:space:]]*gate[[:space:]]+coverage[[:space:]]+make[[:space:]].*coverage-check' "$df"; then + printf 'ok R2 scripts/dogfood.sh runs make ... coverage-check (the release consumer exists)\n' + else printf 'FAIL R2 scripts/dogfood.sh no longer runs coverage-check -- nothing checks coverage at the release\n'; bad=1; fi + + blk=$(on_block "$wf") + if ! grep -qE '^[[:space:]]+schedule:' <<<"$blk"; then + printf 'FAIL R3 coverage-nightly.yml must trigger on schedule (on: block, comments ignored)\n'; bad=1 + elif grep -qE "$V_TAGS_RE" <<<"$blk"; then + printf 'FAIL R3 coverage-nightly.yml also triggers on v* tags -- ci.yml owns tag coverage (R5); a tag would be measured twice\n'; bad=1 + else printf 'ok R3 coverage-nightly.yml triggers on schedule and not on v* tags\n'; fi + + if grep -qE '^COV_FLOOR[[:space:]]*:?=[[:space:]]*[0-9]+' "$mk"; then + printf 'ok R4 the Makefile defines a numeric COV_FLOOR\n' + else printf 'FAIL R4 the Makefile has no numeric COV_FLOOR -- make coverage enforces nothing\n'; bad=1; fi + + ci=$(live "$cy") + if grep -qE '^[[:space:]]+skip_coverage:[[:space:]]*true' <<<"$ci"; then + printf 'FAIL R5 ci.yml sets skip_coverage: true -- sovereign-ci'"'"'s gate reads that skip as a mandatory failure (ci / gate red on every PR); use coverage_on: tag\n'; bad=1 + elif ! grep -qE "^[[:space:]]+coverage_on:[[:space:]]*['\"]?tag['\"]?[[:space:]]*$" <<<"$ci"; then + printf 'FAIL R5 ci.yml does not hand sovereign-ci coverage_on: tag (on a live line)\n'; bad=1 + elif ! grep -qE "$V_TAGS_RE" <<<"$(on_block "$cy")"; then + printf "FAIL R5 ci.yml's on: block has no push tags [v*] -- coverage_on: tag alone never fires on a tag (the reusable's second required edit)\n"; bad=1 + else printf 'ok R5 ci.yml: coverage_on: tag, and on: push tags [v*] (coverage is owed on every release tag)\n'; fi + return "$bad" +} + +# The header is every leading comment line, found rather than counted. +case "${1:-}" in -h|--help) awk 'NR == 1 { next } !/^#/ { exit } { sub(/^# ?/, ""); print }' "$0"; exit 0 ;; esac + +if [ "${1:-}" = "--self-test" ]; then + echo "=== coverage producer chain: each rule must turn RED when its link breaks ===" + d=$(mktemp -d "${TMPDIR:-/tmp}/covchain.XXXXXX") || exit 2 + rmtree() { case "${1:-}" in ''|/) return 0 ;; *) [ -d "$1" ] && rm -rf -- "$1" ;; esac; return 0; } + trap 'rmtree "${d:-}"' EXIT + bad=0; n=0 + fixture() { # fixture -> a tree where every link holds + mkdir -p "$1/scripts" "$1/.github/workflows" + printf 'COV_FLOOR := 88\nLLVMCOV := llvm-cov\ncoverage:\n\t$(LLVMCOV) report --fail-under-lines $(COV_FLOOR)\ncoverage-check: coverage\n' > "$1/Makefile" + printf '#!/usr/bin/env bash\n gate coverage make -C "$d" coverage-check\n' > "$1/scripts/dogfood.sh" + printf "on:\n schedule:\n - cron: '0 22 * * *'\n workflow_dispatch: {}\njobs: {}\n" > "$1/.github/workflows/coverage-nightly.yml" + printf "on:\n push:\n branches: [main, master]\n tags: ['v*']\n pull_request:\n branches: [main, master]\njobs:\n ci:\n uses: paiml/.github/.github/workflows/sovereign-ci.yml@x\n with:\n coverage_on: tag\n" > "$1/.github/workflows/ci.yml" + } + # one mutation per row, as a function (no eval: the mutation is code, not a string) + m_none() { :; } + m_r1() { sed -i 's/\$(LLVMCOV) report.*/echo nothing/' Makefile; } + m_r2_gone() { printf '#!/usr/bin/env bash\n' > scripts/dogfood.sh; } + m_r2_comment() { printf '#!/usr/bin/env bash\n# gate coverage make -C x coverage-check\n' > scripts/dogfood.sh; } + m_r3_nosched() { sed -i '/schedule:/d; /cron:/d' .github/workflows/coverage-nightly.yml; } + m_r3_tagsback() { printf "on:\n schedule:\n - cron: '0 22 * * *'\n push:\n tags: ['v*']\njobs: {}\n" > .github/workflows/coverage-nightly.yml; } + m_r4() { sed -i '/^COV_FLOOR/d' Makefile; } + m_r5_skip() { sed -i 's/coverage_on: tag/skip_coverage: true/' .github/workflows/ci.yml; } + m_r5_both() { printf ' skip_coverage: true\n' >> .github/workflows/ci.yml; } + m_r5_nocov() { sed -i '/coverage_on:/d' .github/workflows/ci.yml; } + m_r5_comment() { sed -i 's/ coverage_on: tag/ # coverage_on: tag/' .github/workflows/ci.yml; } + m_r5_always() { sed -i 's/coverage_on: tag/coverage_on: always/' .github/workflows/ci.yml; } + m_r5_notag() { sed -i "/ tags: \['v\*'\]/d" .github/workflows/ci.yml; } + m_r5_tagcomment() { sed -i "s/^ tags: \['v\*'\]/ # tags: ['v*']/" .github/workflows/ci.yml; } + m_missing() { rm -f .github/workflows/coverage-nightly.yml; } + m_missing_ci() { rm -f .github/workflows/ci.yml; } + row() { # row WANT-RC LABEL MUTATION-FUNCTION + local want=$1 label=$2 rc=0; n=$((n + 1)); rmtree "$d/t"; fixture "$d/t" + ( cd "$d/t" && "$3" ) || { printf 'FAIL row %s fixture mutation failed: %s\n' "$n" "$label"; bad=1; return; } + judge "$d/t" > "$d/out" 2>&1 || rc=$? + if [ "$rc" = "$want" ]; then printf 'ok row %-2s rc=%s %s\n' "$n" "$rc" "$label" + else printf 'FAIL row %-2s rc=%s (wanted %s) %s\n' "$n" "$rc" "$want" "$label"; sed 's/^/ /' "$d/out"; bad=1; fi + } + row 0 "every link holds -> PASS" m_none + row 1 "R1: coverage no longer runs llvm-cov -> RED" m_r1 + row 1 "R2: dogfood stops calling coverage-check -> RED" m_r2_gone + row 1 "R2: the call only in a COMMENT -> RED" m_r2_comment + row 1 "R3: nightly schedule removed -> RED" m_r3_nosched + row 1 "R3: the nightly's v* tag trigger back (a tag measured twice) -> RED" m_r3_tagsback + row 1 "R4: COV_FLOOR removed -> RED" m_r4 + row 1 "R5: skip_coverage: true instead of coverage_on (the #3688 red) -> RED" m_r5_skip + row 1 "R5: skip_coverage: true alongside coverage_on -> RED" m_r5_both + row 1 "R5: coverage_on removed -> RED" m_r5_nocov + row 1 "R5: coverage_on only in a COMMENT -> RED" m_r5_comment + row 1 "R5: coverage_on: always (coverage back on every PR) -> RED" m_r5_always + row 1 "R5: ci.yml's v* tag trigger removed -> RED" m_r5_notag + row 1 "R5: ci.yml's v* tag trigger only in a COMMENT -> RED" m_r5_tagcomment + row 2 "coverage-nightly.yml missing is ENV rc=2, never a pass" m_missing + row 2 "ci.yml missing is ENV rc=2, never a pass" m_missing_ci + [ "$bad" = 0 ] && { printf 'SELF-TEST PASSED: %s rows\n' "$n"; exit 0; } + printf 'SELF-TEST FAILED\n' >&2; exit 1 +fi + +echo "=== every consumer of aprender's coverage number has a producer (check_coverage_has_producers.sh) ===" +judge "$ROOT"; rc=$? +[ "$rc" = 0 ] && echo PASS || echo "FAIL (rc=$rc)" >&2 +exit "$rc" diff --git a/scripts/check_dogfood_version_row.sh b/scripts/check_dogfood_version_row.sh new file mode 100755 index 0000000000..f2cd2892ae --- /dev/null +++ b/scripts/check_dogfood_version_row.sh @@ -0,0 +1,228 @@ +#!/usr/bin/env bash +# check_dogfood_version_row.sh -- the dogfood's version row is PHASE-AWARE, and the banner names the +# phase that ran (#3543). +# +# THE DEFECT, measured on the 0.68.2 train (2026-09-20, right after a 74/74 cascade): +# `scripts/dogfood.sh --phase post-publish` printed +# [FAIL] version-unpublished aprender 0.68.2 is ALREADY on crates.io — bump the version +# ══ dogfood pre-release: aprender v0.68.2 ══ +# The row is a PRE-publish gate; a successful publish turns it RED forever, so the post-publish +# dogfood could never say GO, and the DEFERs it exists to discharge never were. The banner lied +# about which phase ran. +# +# WHAT THIS RUNS. The REAL "2 + 10. version + publish dry-run" section of scripts/dogfood.sh -- +# extracted from its section header to the next one, so the call sites run, not just the helpers +# -- with stubs for `mark` (records rows), the publish dry-run (the `cargo` name on PATH) and `curl` +# (serves a fixture crates.io sparse index: present / absent / 404 / an HTML body / down). Plus +# the phase_banner line. +# +# THE ROWS (#3543's falsifier, both directions, both phases) +# pre-present pre-publish, version in the index -> version-unpublished FAIL +# pre-absent pre-publish, index lacks the version -> version-unpublished PASS +# pre-404 pre-publish, crate not in the index -> version-unpublished PASS +# pre-down pre-publish, index unreachable -> version-unpublished FAIL, UNKNOWN +# post-present post-publish, version in the index AND it installs from crates.io +# (`--version =9.9.9 --locked`, its binary reports 9.9.9) -> version-published PASS, +# and NO version-unpublished row (it is a pre-publish gate) +# post-install-fails in the index, but the install of it fails -> version-published FAIL +# post-install-wrongver it installs, but the binary reports another version -> FAIL +# post-install-nobin it "installs" but no binary lands -> FAIL +# post-absent post-publish, index lacks the version -> version-published FAIL, and no install is +# attempted +# post-404 post-publish, crate not in the index -> version-published FAIL +# post-html post-publish, a 200 that is not the index -> version-published FAIL, UNKNOWN +# post-dry-run post-publish still runs the publish dry-run: row 10 reads its exit code +# full-unchanged full phase: the dry-run path as before (already exists -> FAIL), no index call +# banner post-publish prints "══ dogfood post-publish: aprender v9.9.9 ══" +# index-path crate "Aprender" is looked up at index.crates.io/ap/re/aprender (lowercased) +# THE MUTANTS -- one per row; each must turn its row RED. See the `mutant` lines at the bottom. +# +# Exit 0 = every row green and every mutant killed. 1 = a row RED or a mutant survived. +# 2 = ENV: the subject or a mutation anchor is missing -- the table judged nothing. +set -uo pipefail +ROOT="$(cd "$(dirname "$0")/.." && pwd)" || exit 2 +SUBJECT="$ROOT/scripts/dogfood.sh" +SEC_START='# ── 2 + 10. version + publish dry-run' +SEC_END='# ── 3. changelog mentions the version' + +env_die() { printf 'ENV %s -- the table judged nothing, not a pass\n' "$*" >&2; exit 2; } +[ -r "$SUBJECT" ] || env_die "no $SUBJECT" +for t in python3 sed; do command -v "$t" > /dev/null 2>&1 || env_die "no $t"; done +for a in "$SEC_START" "$SEC_END" 'phase_banner() {' 'strip_ansi() {'; do + grep -qF -- "$a" "$SUBJECT" || env_die "dogfood.sh has no '$a' -- the subject moved" +done + +TMP=$(mktemp -d) || exit 2 +# SEC011: validate before rm -rf. An empty or '/' value must never reach it. +cleanup() { case "${TMP:-}" in ''|/) return 0 ;; *) [ -d "$TMP" ] && rm -rf -- "$TMP" ;; esac; } +trap cleanup EXIT + +mkdir -p "$TMP/bin" +# curl: -o FILE -w '%{http_code}' ... URL. FX_INDEX picks what the fixture index answers. +cat > "$TMP/bin/curl" <<'STUB' +#!/usr/bin/env bash +out=""; url="" +while [ $# -gt 0 ]; do case "$1" in -o) out=$2; shift 2 ;; -w|-A) shift 2 ;; -*) shift ;; *) url=$1; shift ;; esac; done +printf 'curl %s\n' "$url" >> "$FX_LOG" +case "${FX_INDEX:-}" in + present) printf '{"name":"aprender","vers":"9.9.8"}\n{"name":"aprender","vers":"9.9.9"}\n' > "$out"; printf 200 ;; + absent) printf '{"name":"aprender","vers":"9.9.8"}\n' > "$out"; printf 200 ;; + 404) : > "$out"; printf 404 ;; + html) printf 'captive portal\n' > "$out"; printf 200 ;; + down) echo "curl: (6) Could not resolve host: index.crates.io" >&2; exit 6 ;; + *) exit 99 ;; +esac +STUB +# the publish dry-run: FX_DRY=already prints the already-exists warning; exit FX_DRC (default 0). +# install: FX_INSTALL = ok (default) | fail | wrongver | nobin; the binary lands under --root +cat > "$TMP/bin/cargo" <<'STUB' +#!/usr/bin/env bash +if [ "${1:-}" = install ]; then + printf 'install %s\n' "$*" >> "$FX_LOG" + root=""; prev=""; for a in "$@"; do [ "$prev" = --root ] && root=$a; prev=$a; done + case "${FX_INSTALL:-ok}" in + fail) echo "error: failed to compile \`aprender v9.9.9\`, intermediate artifacts can be found at ..."; exit 101 ;; + nobin) exit 0 ;; + esac + v=9.9.9; [ "${FX_INSTALL:-ok}" = wrongver ] && v=9.9.8 + mkdir -p "$root/bin" && printf '#!/usr/bin/env bash\necho "apr %s (fixture)"\n' "$v" > "$root/bin/apr" && chmod +x "$root/bin/apr" + exit 0 +fi +printf 'dry-run %s\n' "$*" >> "$FX_LOG" +[ "${FX_DRY:-}" = already ] && echo "warning: crate aprender@9.9.9 already exists on crates.io index" +exit "${FX_DRC:-0}" +STUB +chmod +x "$TMP/bin/curl" "$TMP/bin/cargo" + +# run_section SRC NAME PHASE [ENV=VAL ...] -> rows in $TMP/NAME/rows, calls in $TMP/NAME/log, DRC in .../drc +run_section() { + local src=$1 d="$TMP/$2" phase=$3; shift 3 + mkdir -p "$d/work" || return 2 + : > "$d/rows"; : > "$d/log" + awk -v s="$SEC_START" -v e="$SEC_END" 'index($0, s) == 1 {f = 1} index($0, e) == 1 {f = 0} f' "$src" > "$d/section.sh" + grep -E '^strip_ansi\(\) \{' "$src" > "$d/prelude.sh" + [ -s "$d/section.sh" ] && [ -s "$d/prelude.sh" ] || return 2 + ( export PATH="$TMP/bin:$PATH" FX_LOG="$d/log" DOGFOOD_PHASE="$phase" VERSION=9.9.9 WORKLOG="$d/work" + for kv in "$@"; do export "${kv?}"; done + # AFTER the per-row exports: a crate name set before them never saw FX_CRATE (the index-path + # row passed for the wrong reason until its mutant survived) + export CRATE="${FX_CRATE:-aprender}" + mark() { printf '%s %s %s\n' "$1" "$2" "$3" >> "$FX_ROWS"; } + export FX_ROWS="$d/rows" + . "$d/prelude.sh"; . "$d/section.sh" + printf '%s\n' "${DRC:-unset}" > "$d/drc" ) > "$d/out" 2>&1 +} +# expect NAME ROW STATUS NEEDLE -> 0 when the section marked ROW STATUS with a note holding NEEDLE +expect() { + grep -qF -- "$2 $3 " "$TMP/$1/rows" || { printf 'no "%s %s" row: %s\n' "$2" "$3" "$(tr '\n' '|' < "$TMP/$1/rows")"; return 1; } + [ -z "$4" ] || grep -qF -- "$4" <<< "$(grep -F -- "$2 $3 " "$TMP/$1/rows")" || { printf '"%s %s" never said "%s": %s\n' "$2" "$3" "$4" "$(grep -F "$2 " "$TMP/$1/rows")"; return 1; } + return 0 +} + +# ---- row bodies: TAG SRC -> 0 green, 1 red (prints why), 2 fixture ------------------------------ +pre_present() { run_section "$2" "prep-$1" pre-publish FX_INDEX=present || return 2; expect "prep-$1" version-unpublished FAIL "ALREADY in the crates.io index"; } +pre_absent() { run_section "$2" "prea-$1" pre-publish FX_INDEX=absent || return 2; expect "prea-$1" version-unpublished PASS "absent from the crates.io index"; } +pre_404() { run_section "$2" "pre4-$1" pre-publish FX_INDEX=404 || return 2; expect "pre4-$1" version-unpublished PASS "HTTP 404"; } +pre_down() { run_section "$2" "pred-$1" pre-publish FX_INDEX=down || return 2; expect "pred-$1" version-unpublished FAIL "UNKNOWN"; } +post_present() { + run_section "$2" "posp-$1" post-publish FX_INDEX=present || return 2 + expect "posp-$1" version-published PASS "of it works: 1 binary(ies) installed from crates.io" || return 1 + grep -q -- '^install install aprender --version =9.9.9 --locked ' "$TMP/posp-$1/log" \ + || { printf 'not installed as a user would (--version =9.9.9 --locked): %s\n' "$(grep '^install ' "$TMP/posp-$1/log")"; return 1; } + ! grep -q '^version-unpublished ' "$TMP/posp-$1/rows" || { printf 'post-publish still emitted the pre-publish row: %s\n' "$(grep '^version-unpublished ' "$TMP/posp-$1/rows")"; return 1; } + return 0 +} +post_absent() { + run_section "$2" "posa-$1" post-publish FX_INDEX=absent || return 2 + expect "posa-$1" version-published FAIL "is NOT in the crates.io index" || return 1 + ! grep -q '^install ' "$TMP/posa-$1/log" || { printf 'an install was attempted for a version the index does not carry\n'; return 1; } + return 0 +} +post_install_fails() { run_section "$2" "posif-$1" post-publish FX_INDEX=present FX_INSTALL=fail || return 2; expect "posif-$1" version-published FAIL "aprender --version =9.9.9 --locked\` exited 101"; } +post_install_wrongver() { run_section "$2" "posiw-$1" post-publish FX_INDEX=present FX_INSTALL=wrongver || return 2; expect "posiw-$1" version-published FAIL "do not report 9.9.9"; } +post_install_nobin() { run_section "$2" "posin-$1" post-publish FX_INDEX=present FX_INSTALL=nobin || return 2; expect "posin-$1" version-published FAIL "installed no binary"; } +post_404() { run_section "$2" "pos4-$1" post-publish FX_INDEX=404 || return 2; expect "pos4-$1" version-published FAIL "HTTP 404"; } +post_html() { run_section "$2" "posh-$1" post-publish FX_INDEX=html || return 2; expect "posh-$1" version-published FAIL "UNKNOWN"; } +post_dry_run() { + run_section "$2" "posd-$1" post-publish FX_INDEX=present FX_DRY=already FX_DRC=0 || return 2 + grep -q '^dry-run publish --dry-run' "$TMP/posd-$1/log" || { printf 'post-publish no longer runs the publish dry-run (row 10 reads its exit code)\n'; return 1; } + [ "$(cat "$TMP/posd-$1/drc")" = 0 ] || { printf 'DRC=%s after the post-publish section, want 0\n' "$(cat "$TMP/posd-$1/drc")"; return 1; } + return 0 +} +full_unchanged() { + run_section "$2" "full-$1" full FX_INDEX=present FX_DRY=already || return 2 + expect "full-$1" version-unpublished FAIL "ALREADY on crates.io" || return 1 + ! grep -q '^curl ' "$TMP/full-$1/log" || { printf 'the full phase consulted the index\n'; return 1; } + return 0 +} +banner() { + local got + grep -E '^phase_banner\(\) \{' "$2" > "$TMP/banner-$1.sh" || return 2 + got=$(. "$TMP/banner-$1.sh"; phase_banner post-publish aprender 9.9.9) + [ "$got" = "══ dogfood post-publish: aprender v9.9.9 ══" ] || { printf 'banner: %s\n' "$got"; return 1; } + grep -qE '^phase_banner "\$DOGFOOD_PHASE" "\$CRATE" "\$VERSION"$' "$2" || { printf 'the banner is not printed from DOGFOOD_PHASE\n'; return 1; } + return 0 +} +index_path() { + run_section "$2" "path-$1" post-publish FX_INDEX=present FX_CRATE=Aprender || return 2 + grep -qx 'curl https://index.crates.io/ap/re/aprender' "$TMP/path-$1/log" || { printf 'looked up: %s\n' "$(grep '^curl ' "$TMP/path-$1/log")"; return 1; } + return 0 +} + +fails=0; rows=0 +row() { # row NAME RC MESSAGE + rows=$((rows + 1)) + if [ "$2" = 0 ]; then printf 'ok %s\n' "$1" + elif [ "$2" = 2 ]; then env_die "row $1 could not build its fixture" + else printf 'FAIL %s: %s\n' "$1" "$3" >&2; fails=$((fails + 1)); fi +} + +for spec in "pre-present pre_present" "pre-absent pre_absent" "pre-404 pre_404" "pre-down pre_down" \ + "post-present post_present" "post-absent post_absent" "post-404 post_404" "post-html post_html" \ + "post-install-fails post_install_fails" "post-install-wrongver post_install_wrongver" "post-install-nobin post_install_nobin" \ + "post-dry-run post_dry_run" "full-unchanged full_unchanged" "banner banner" "index-path index_path"; do + set -- $spec + msg=$($2 real "$SUBJECT"); row "$1" "$?" "$msg" +done + +# ---- the mutants ------------------------------------------------------------------------------ +# mutant NAME ANCHOR REPLACEMENT ROWFN -- ANCHOR must occur exactly once in dogfood.sh +mutant() { + local name=$1 m="$TMP/m-$1.sh" msg mrc + python3 - "$SUBJECT" "$m" "$2" "$3" <<'PY' || env_die "$name mutant: anchor not found exactly once" +import sys +src, dst, anchor, repl = sys.argv[1:5] +s = open(src).read() +assert s.count(anchor) == 1, anchor +open(dst, "w").write(s.replace(anchor, repl, 1)) +PY + msg=$($4 "m-$name" "$m"); mrc=$? + [ "$mrc" = 2 ] && env_die "$name mutant could not build its fixture" + [ "$mrc" != 0 ]; row "mutant $name is killed by $4 (${msg:-survived})" "$?" "the mutant PASSED -- the row does not discriminate" +} +mutant pre-present-pass "pre-publish:present) printf 'version-unpublished FAIL" "pre-publish:present) printf 'version-unpublished PASS" pre_present +mutant pre-absent-fail "pre-publish:absent) printf 'version-unpublished PASS" "pre-publish:absent) printf 'version-unpublished FAIL" pre_absent +mutant pre-404-fail "pre-publish:crate-absent) printf 'version-unpublished PASS" "pre-publish:crate-absent) printf 'version-unpublished FAIL" pre_404 +mutant down-is-absent "printf 'unknown index.crates.io not consulted (curl exit=%s): %s\\n'" "printf 'absent\\n'; : " pre_down +mutant post-phase-blind "post-publish:present) printf 'version-published PASS" "post-publish:present) printf 'version-unpublished FAIL" post_present +mutant post-absent-pass "post-publish:absent) printf 'version-published FAIL" "post-publish:absent) printf 'version-published PASS" post_absent +mutant post-404-pass "post-publish:crate-absent) printf 'version-published FAIL" "post-publish:crate-absent) printf 'version-published PASS" post_404 +mutant html-is-absent " *) printf 'unknown index.crates.io answered HTTP 200 but the body is not the index" " *) printf 'absent\\n'; : printf 'unknown" post_html +mutant no-post-call " mark_version_row post-publish +" "" post_present +mutant post-skips-dry-run 'publish --dry-run --allow-dirty 2>&1); DRC=$?' 'publish --help > /dev/null 2>&1); DRY=""; DRC=0' post_dry_run +mutant full-uses-index 'if [ "$DOGFOOD_PHASE" = post-publish ]; then + # #3543' 'if [ "$DOGFOOD_PHASE" != pre-publish ]; then + # #3543' full_unchanged +mutant banner-fixed "phase_banner() { printf '══ dogfood %s: %s v%s ══\\n' \"\$1\"" "phase_banner() { printf '══ dogfood pre-release: %s v%s ══\\n' \"\$2\"" banner +mutant index-case "name=\$(printf '%s' \"\$1\" | tr '[:upper:]' '[:lower:]')" "name=\$1" index_path +mutant install-ignored ' *) st=FAIL; note="$CRATE $VERSION is in the crates.io index but is NOT installable' ' *) note="$CRATE $VERSION is in the crates.io index but is NOT installable' post_install_fails +mutant version-unchecked ' case "$v" in *"$2"*) ;; *) bad="$bad $(basename "$b")='"'"'$v'"'"'" ;; esac' ' :' post_install_wrongver +mutant nobin-ok ' [ "$n" -gt 0 ] || { printf' ' true || { printf' post_install_nobin +mutant install-on-absent ' if [ "$1" = post-publish ] && [ "$state" = present ]; then' ' if [ "$1" = post-publish ]; then' post_absent +mutant unlocked 'install "$1" --version "=$2" --locked --root' 'install "$1" --version "=$2" --root' post_present + +# VACUITY FLOOR: a table that ran fewer rows than it declares is not a pass. +[ "$rows" -ge 33 ] || { printf 'VACUOUS %s row(s) ran, fewer than the 33 declared\n' "$rows" >&2; exit 1; } +[ "$fails" -eq 0 ] || { printf 'RED %s of %s row(s) failed\n' "$fails" "$rows" >&2; exit 1; } +printf 'PASS %s row(s): the dogfood version row is phase-aware and the banner names the phase (#3543)\n' "$rows" diff --git a/scripts/check_llama_pin.sh b/scripts/check_llama_pin.sh index a222e272b4..26e6877797 100755 --- a/scripts/check_llama_pin.sh +++ b/scripts/check_llama_pin.sh @@ -88,6 +88,23 @@ mk_tree "$td/nocache" "version: 4567 (abcdef1)" NOCACHE # The declared cmake line every row is checked against unless it says otherwise. DEFAULT_FLAGS="cmake -B build -DGGML_CUDA=ON" +# THE CANONICAL LOCATION (#3563): with nothing named, the resolver looks for the pinned build +# at ${LLAMA_PIN_SRC_ROOT:-$HOME/src}/llama.cpp-. Every row runs against a +# FIXTURE root (empty unless the row says otherwise), never the operator's $HOME -- a row +# whose verdict depends on what this box has built is the anecdote class. SUBJECT is the +# resolver under test, so a mutant can be run through the same rows. +mkdir -p "$td/src-empty" +PIN_SRC_ROOT="$td/src-empty" +SUBJECT="" +mk_tree "$td/srcgood/llama.cpp-abcdef1/build" "version: 4567 (abcdef1)" +mk_tree "$td/srcwrong/llama.cpp-abcdef1/build" "version: 9999 (999999f)" +mk_tree "$td/srccuda/llama.cpp-abcdef1/build" "version: 4567 (abcdef1)" 'GGML_CUDA:BOOL=OFF' +# THE ABBREVIATION LENGTH (#3563): git prints the shortest abbreviation unique IN THE CLONE, so the +# same commit reads d1d3c3396 on lambda and d1d3c339 on gx10. New --version shape, three lengths. +mk_tree "$td/abbr7" "version: 0.4.1-dev (build 2423, commit abcdef1)" +mk_tree "$td/abbr10" "version: 0.4.1-dev (build 2423, commit abcdef1234)" +mk_tree "$td/abbr6" "version: 0.4.1-dev (build 2423, commit abcdef)" + run_case() { # run_case [expiry] [expected-reason] [build-flags] local name="$1" pin="$2" cand="$3" want="$4" @@ -108,8 +125,9 @@ run_case() { export LLAMA_BENCH_PATH="$cand" export LLAMA_PIN_HOST=lambda export LLAMA_PIN_TODAY="$TABLE_TODAY" + export LLAMA_PIN_SRC_ROOT="$PIN_SRC_ROOT" # shellcheck disable=SC1090 - . "$OLDPWD/scripts/llama_bin.sh" 2>/dev/null || true + . "${SUBJECT:-$OLDPWD/scripts/llama_bin.sh}" 2>/dev/null || true llama_bin_resolve >/dev/null 2>&1 printf '%s %s' "$?" "${LLAMA_PIN_REASON:-}" ) @@ -142,6 +160,56 @@ run_case "unpinned, binary present" "UNPINNED" "$td/good/bin/llama-bench run_case "unpinned, no binary named" "UNPINNED" "" 2 "$FRESH_EXPIRY" unpinned run_case "pinned but no binary named" "abcdef1" "" 1 "$FRESH_EXPIRY" no_binary_named +# ── #3563, the CANONICAL LOCATION ────────────────────────────────────────── +# `scripts/llama_bin.sh` reported `no_binary_named` on lambda while the pinned build sat at +# ~/src/llama.cpp-d1d3c3396, because nothing ever named it. With nothing named, the build of the +# pinned commit at /llama.cpp- is the candidate -- and it passes the same +# oracle, commit and cmake checks as a named one, or it does not pass. +printf ' +canonical location (#3563) +' +PIN_SRC_ROOT="$td/srcgood" run_case "nothing named: the canonical build resolves" "abcdef1" "" 0 "$FRESH_EXPIRY" ok +PIN_SRC_ROOT="$td/srcwrong" run_case "the canonical build is the WRONG commit" "abcdef1" "" 1 "$FRESH_EXPIRY" wrong_build +PIN_SRC_ROOT="$td/srccuda" run_case "the canonical build's cmake disagrees" "abcdef1" "" 1 "$FRESH_EXPIRY" cmake_mismatch +PIN_SRC_ROOT="$td/srcgood" run_case "a NAMED path wins over the canonical build" "abcdef1" "$td/wrong/bin/llama-bench" 1 "$FRESH_EXPIRY" wrong_build +PIN_SRC_ROOT="$td/srcgood" run_case "unpinned never falls back to a canonical build" "UNPINNED" "" 2 "$FRESH_EXPIRY" unpinned +# The ABBREVIATION rows: a pin of 8 hex digits against an oracle printing 7, 10 and 6. +run_case "oracle abbreviates SHORTER than the pin" "abcdef12" "$td/abbr7/bin/llama-bench" 0 "$FRESH_EXPIRY" ok +run_case "oracle abbreviates LONGER than the pin" "abcdef12" "$td/abbr10/bin/llama-bench" 0 "$FRESH_EXPIRY" ok +run_case "6 hex digits is below git's minimum" "abcdef12" "$td/abbr6/bin/llama-bench" 1 "$FRESH_EXPIRY" wrong_build +run_case "a different commit, same length" "abcdef12" "$td/wrong/bin/llama-bench" 1 "$FRESH_EXPIRY" wrong_build +# THE ABBREVIATION MUTANT: the pre-#3563 substring test (the pin must appear inside the version +# line). gx10's correct build was refused by exactly this; the SHORTER row must go RED under it. +python3 - "$OLDPWD/scripts/llama_bin.sh" "$td/llama_bin.abbr-mutant.sh" <<'PY' || { printf 'FAIL abbreviation mutant could not be built\n'; rc=1; } +import sys +src, dst = sys.argv[1:3] +s = open(src).read() +a = ' if ! llama_commit_match "$llama_bin_want" "$llama_bin_got"; then' +assert s.count(a) == 1 +open(dst, "w").write(s.replace(a, ' if case "$LLAMA_BUILD" in *"$llama_bin_want"*) false ;; *) true ;; esac; then', 1)) +PY +mutant_out=$(rc=0; SUBJECT="$td/llama_bin.abbr-mutant.sh" \ + run_case "oracle abbreviates SHORTER than the pin" "abcdef12" "$td/abbr7/bin/llama-bench" 0 "$FRESH_EXPIRY" ok; echo "RC=$rc") +case "$mutant_out" in + *"RC=1"*) printf 'ok %-38s killed: %s\n' "substring-match mutant" "$(head -n 1 <<< "$mutant_out" | cut -c7-90)" ;; + *) printf 'FAIL %-38s SURVIVED -- the abbreviation row does not discriminate\n' "substring-match mutant"; rc=1 ;; +esac +# THE MUTANT: the canonical lookup pointed nowhere, as before #3563. The first row above must go +# RED under it, or that row does not discriminate. +sed 's|LLAMA_PIN_CANON="${LLAMA_PIN_SRC_ROOT:-$HOME/src}/llama.cpp-$llama_bin_want/build/bin/llama-bench"|LLAMA_PIN_CANON="/nonexistent/llama-bench"|' \ + "$OLDPWD/scripts/llama_bin.sh" > "$td/llama_bin.mutant.sh" +if cmp -s "$OLDPWD/scripts/llama_bin.sh" "$td/llama_bin.mutant.sh"; then + printf 'FAIL %-38s the canonical-lookup anchor is gone -- the mutant is identical\n' "canonical-lookup mutant" + rc=1 +else + mutant_out=$(rc=0; SUBJECT="$td/llama_bin.mutant.sh" PIN_SRC_ROOT="$td/srcgood" \ + run_case "nothing named: the canonical build resolves" "abcdef1" "" 0 "$FRESH_EXPIRY" ok; echo "RC=$rc") + case "$mutant_out" in + *"RC=1"*) printf 'ok %-38s killed: %s\n' "canonical-lookup mutant" "$(head -n 1 <<< "$mutant_out" | cut -c7-90)" ;; + *) printf 'FAIL %-38s SURVIVED -- the canonical row does not discriminate\n' "canonical-lookup mutant"; rc=1 ;; + esac +fi + # ── PP-20, the EXPIRY half ───────────────────────────────────────────────── # Nothing could ever emit COMPARATOR_STALE before these rows existed: the pin # carried no expiry field at all, so the status in §7.4 had no producer. diff --git a/scripts/check_model_ladder.sh b/scripts/check_model_ladder.sh index 40be43269b..b42e950d81 100755 --- a/scripts/check_model_ladder.sh +++ b/scripts/check_model_ladder.sh @@ -3,7 +3,17 @@ # # Reads one receipt per REQUIRED host (written by scripts/model_ladder.sh) for # the version being cut and is green only when every required rung is present -# and green on every one of them. It is declared in Cargo.toml +# and green on every one of them, AND every model the host HOLDS is too. +# +# #3712 (operator 2026-09-21: "you must ensure all models Q4_K CUDA work; the end", +# and publishing with "most working" is a "p0 tire fire"). Three rules, no exemptions +# and no thresholds: +# * No Q4_K rung is optional. `required: false` on a Q4_K rung is REFUSED, and so is +# a Q4_K rung that does not claim cuda. +# * The universe is the host's measured inventory. A receipt must be schema v2 and +# carry a non-empty `inventory`. Every inventory model must appear in the run and be +# green on CUDA, or it is a FAIL naming it. +# * A skipped capability_match or golden_output, a fallback line, or a run rc != 0 is RED. It is declared in Cargo.toml # [package.metadata.dogfood] so scripts/dogfood.sh runs it in every phase; a # missing receipt is FAIL, not DEFER — a dev build can measure this, no # published crate is needed. @@ -31,18 +41,24 @@ while [ $# -gt 0 ]; do --ladder) [ $# -ge 2 ] || { echo "--ladder needs a value" >&2; exit 2; }; LADDER="$2"; shift 2 ;; --ladder-main) [ $# -ge 2 ] || { echo "--ladder-main needs a value" >&2; exit 2; }; LADDER_MAIN_OVERRIDE="$2"; shift 2 ;; --version) [ $# -ge 2 ] || { echo "--version needs a value" >&2; exit 2; }; VERSION_OVERRIDE="$2"; shift 2 ;; - -h|--help) sed -n '2,22p' "$0"; exit 0 ;; + -h|--help) awk 'NR == 1 { next } !/^#/ { exit } { sub(/^# ?/, ""); print }' "$0"; exit 0 ;; *) echo "check_model_ladder: unknown argument '$1'" >&2; exit 2 ;; esac done -cd "$(git rev-parse --show-toplevel 2>/dev/null || pwd)" || exit 2 +SELF="$(cd "$(dirname "$0")" && pwd)/$(basename "$0")" # before the cd: the mutants copy this file +# The root is derived from this file's path, never from `git rev-parse` (it dies in the CI container, +# aprender#3581) or from the caller's cwd. MODEL_LADDER_ROOT is how a mutant copy, which lives in a +# temp dir, is told the tree it judges. +cd "${MODEL_LADDER_ROOT:-$(dirname "$SELF")/..}" || exit 2 # ---------------------------------------------------------------- the judge -# judge → exit 0/1/2 +# judge [context-rungs.json] [its origin/main copy] → exit 0/1/2 judge() { - python3 - "$1" "$2" "$3" "$4" <<'PY' + python3 - "$1" "$2" "$3" "$4" "${5:-}" "${6:-}" <<'PY' import json, os, sys, yaml -ladder_p, main_p, rdir, version = sys.argv[1:5] +ladder_p, main_p, rdir, version, rungs_p, rungs_main_p = sys.argv[1:7] +sys.path.insert(0, os.environ.get("MODEL_LADDER_CELLS_LIB") or "scripts/lib") # a mutant copy of the module, in --self-test +import model_ladder_cells try: L = yaml.safe_load(open(ladder_p))["ladder"] except Exception as e: @@ -52,6 +68,32 @@ rungs = L.get("rungs", []) if not hosts: print("decline: ladder names no required host"); sys.exit(2) if not rungs: print("decline: ladder has no rungs"); sys.exit(2) rc = 0 +import re +def is_q4k(r): # a Q4_K model, by its file or its id (#3712) + return bool(re.search(r"q4_?k", f"{r.get('gguf', '')} {r.get('id', '')}", re.I)) +def why_of(x, backends): # every reason a measured row is not green on the claimed backends + why = [] + cm, go = x.get("capability_match") or {}, x.get("golden_output") or {} + claims_gpu = bool({"cuda", "gpu"} & set(backends)) + cap_ok = (cm.get("passed") and not cm.get("skipped")) or (cm.get("skipped") and not claims_gpu) + if not cap_ok: why.append("capability_match " + ("SKIPPED" if cm.get("skipped") else "FAIL") + ": " + str(cm.get("message", ""))[:60]) + if not (go.get("passed") and not go.get("skipped")): why.append("golden_output " + ("SKIPPED" if go.get("skipped") else "FAIL") + ": " + str(go.get("message", ""))[:60]) + be = x.get("backends") or {} + for b in backends: + v = be.get(b) + if v is None: why.append(f"{b}: not measured") + elif v.get("fallback"): why.append(f"{b}: FELL BACK — the claimed backend did not run") + elif not v.get("ran"): why.append(f"{b}: did not run (rc={v.get('rc')})") + return why +# #3712: no Q4_K rung is optional, and every one claims cuda. The key is refused, not tolerated. +for r in rungs: + if is_q4k(r) and r.get("required") is not True: + print(f"FAIL rung {r['id']} is a Q4_K rung with required: {r.get('required')!r} — no Q4_K model is optional; every one must be green on CUDA (#3712)"); rc = 1 + if is_q4k(r) and "cuda" not in (r.get("backends") or []): + print(f"FAIL rung {r['id']} is a Q4_K rung that does not claim cuda — every Q4_K model must be green on CUDA (#3712)"); rc = 1 +inv_backends = list((L.get("inventory") or {}).get("backends") or []) +if not (L.get("inventory") or {}).get("patterns") or "cuda" not in inv_backends: + print("FAIL the ladder declares no inventory (patterns + backends incl. cuda) — the universe cannot be the host's measured Q4_K models (#3712)"); rc = 1 # anti-shrink vs origin/main if main_p and os.path.exists(main_p): try: @@ -69,6 +111,15 @@ if main_p and os.path.exists(main_p): for r in rungs: if r.get("required") and r["id"] in mrh and not mrh[r["id"]] <= hostset(r, allh): print(f"FAIL hosts DROPPED on {r['id']} vs origin/main: {sorted(mrh[r['id']] - hostset(r, allh))} — a rung may gain hosts, never lose one"); rc = 1 + mc, hc = M.get("cells") or {}, L.get("cells") or {} + if mc and not hc: + print("FAIL the cells block DROPPED vs origin/main -- verbs x thinking x context would owe nothing"); rc = 1 + elif mc: + for what, a, b in (("verbs", mc.get("verbs"), hc.get("verbs")), + ("long-rung families", (mc.get("long_rungs_for") or {}).get("families"), (hc.get("long_rungs_for") or {}).get("families")), + ("long-rung representatives", list((mc.get("long_rungs_for") or {}).get("representatives") or {}), list((hc.get("long_rungs_for") or {}).get("representatives") or {}))): + gone = set(a or []) - set(b or []) + if gone: print(f"FAIL cells {what} DROPPED vs origin/main: {sorted(gone)}"); rc = 1 mh = {h["id"] for h in M.get("hosts", []) if h.get("required")} for hid in mh - {h["id"] for h in hosts}: print(f"FAIL required host DROPPED vs origin/main: {hid}"); rc = 1 @@ -77,6 +128,7 @@ if main_p and os.path.exists(main_p): print(f"FAIL ladder at origin/main unreadable: {e}"); rc = 1 else: print("! BOOTSTRAP: no ladder at origin/main yet; the anti-shrink floor arms when this lands") +good = {} # host id -> a receipt that passed the host-level checks; the cells judge reads only these for h in hosts: f = os.path.join(rdir, f"{h['id']}.json") if not os.path.exists(f): @@ -89,9 +141,29 @@ for h in hosts: print(f"FAIL {h['id']:7} receipt is for {R.get('version')!r}, this cut is {version!r} — STALE"); rc = 1; continue if int(R.get("executed", 0)) < 1: print(f"FAIL {h['id']:7} receipt executed=0 — a receipt that measured nothing is not evidence"); rc = 1; continue + inv = R.get("inventory") + if R.get("schema") != "apr-model-ladder-receipt/v2" or not isinstance(inv, list): + print(f"FAIL {h['id']:7} receipt carries no measured inventory (schema {R.get('schema')!r}) — the universe is what the host HOLDS, not a list (#3712)"); rc = 1; continue + if not inv: + print(f"FAIL {h['id']:7} measured inventory is EMPTY — a host holding no Q4_K model proved nothing (#3712)"); rc = 1; continue + good[h["id"]] = R by = {r.get("id"): r for r in R.get("rungs", [])} + by_file = {x.get("file"): x for x in R.get("rungs", []) if x.get("file")} + ladder_files = {r.get("gguf") for r in rungs} + inv_green = 0 + for item in inv: + f = item.get("file") + x = by_file.get(f) + if x is None or not x.get("present"): # held by the host, absent from the run + print(f"FAIL {h['id']:7} inventory model {f} is MISSING from the run — the host holds it, so the release must prove it (#3712)"); rc = 1; continue + if f in ladder_files: + continue # a ladder rung: judged, required, in the rung loop below + why = why_of(x, inv_backends) + if why: print(f"FAIL {h['id']:7} inv:{f:22} " + "; ".join(why)); rc = 1 + else: inv_green += 1; print(f"ok {h['id']:7} inv:{f:22} green on {','.join(inv_backends)}") + print(f"ok {h['id']:7} inventory: {len(inv)} Q4_K model(s) held, every one in the run") for r in rungs: - rid = r["id"]; req = bool(r.get("required")) + rid = r["id"]; req = bool(r.get("required")) or is_q4k(r) x = by.get(rid) tag = "required" if req else "optional" listed = r.get("hosts") @@ -103,28 +175,71 @@ for h in hosts: continue if x.get("sha_ok") is False: print(f"FAIL {h['id']:7} {rid:22} sha256 mismatch — a different file is a different measurement"); rc = 1; continue - why = [] - cm, go = x.get("capability_match") or {}, x.get("golden_output") or {} - claims_gpu = bool({"cuda", "gpu"} & set(r.get("backends", []))) - cap_ok = (cm.get("passed") and not cm.get("skipped")) or (cm.get("skipped") and not claims_gpu) - if not cap_ok: why.append("capability_match " + ("SKIPPED" if cm.get("skipped") else "FAIL") + ": " + str(cm.get("message",""))[:60]) - if not (go.get("passed") and not go.get("skipped")): why.append("golden_output " + ("SKIPPED" if go.get("skipped") else "FAIL") + ": " + str(go.get("message",""))[:60]) - be = x.get("backends") or {} - for b in r.get("backends", []): - v = be.get(b) - if v is None: why.append(f"{b}: not measured") - elif v.get("fallback"): why.append(f"{b}: FELL BACK — the claimed backend did not run") - elif not v.get("ran"): why.append(f"{b}: did not run (rc={v.get('rc')})") + why = why_of(x, r.get("backends", [])) if why: if req: print(f"FAIL {h['id']:7} {rid:22} " + "; ".join(why)); rc = 1 else: print(f"warn {h['id']:7} {rid:22} ({tag}) " + "; ".join(why)) else: print(f"ok {h['id']:7} {rid:22} green on {','.join(r.get('backends', []))} ({R.get('gpu') or 'no-gpu'}, sha {R.get('sha')})") +# ---- cells: verb x thinking x context rung, per model per host (#3712 row B) +rungs_doc = None +if rungs_p and os.path.exists(rungs_p): + try: + rungs_doc = json.load(open(rungs_p)) + except Exception as e: + print(f"FAIL context rungs {rungs_p} unreadable: {e}") +rungs_main = None +if rungs_main_p and os.path.exists(rungs_main_p) and os.path.getsize(rungs_main_p) > 0: + try: + rungs_main = json.load(open(rungs_main_p)) + except Exception as e: + print(f"FAIL context rungs at origin/main unreadable: {e}"); rc = 1 +if model_ladder_cells.judge(L, good, rungs_doc, print, rungs_main): + rc = 1 sys.exit(rc) PY } # ---------------------------------------------------------------- self-test +# ---------------------------------------------------------------- the GPU lock +# Cop ruling 2026-09-21 (#3712): every apr call scripts/model_ladder.sh makes runs under the fleet GPU +# lock (flock, bounded wait) and choom -n 1000, through ONE function, apr_locked. Two halves: +# lock_audit (static, also in the REAL run): no apr subcommand is invoked on "$APR" directly. +# lock_probe (behavioural, self-test): a fake apr, called through --lock-probe, must see the lock +# held and its own oom_score_adj at 1000; with the lock held elsewhere the call must +# decline (exit 2) within the bounded wait, naming the holder's pid. +# lock_audit -> prints FAIL lines, exit 1 on any raw call +lock_audit() { + python3 - "$1" <<'LOCKPY' +import re, sys +bad = 0 +for n, line in enumerate(open(sys.argv[1]), 1): + code = re.sub(r"(^|\s)#.*$", "", line) # a call in a comment is not a call + for m in re.finditer(r'(? -> prints ok/FAIL lines, exit 1 on any failure +lock_probe() { + local prod=$1 w=$2 out rc hp bad=0 + mkdir -p "$w"; : > "$w/lock" + printf '#!/usr/bin/env bash\nif flock -n "$FAKE_LOCK" true; then l=UNLOCKED; else l=LOCKED; fi\necho "fake-apr $1 lock=$l oom=$(cat /proc/self/oom_score_adj)"\n' > "$w/apr" + chmod +x "$w/apr" + out=$(FAKE_LOCK="$w/lock" MODEL_LADDER_ROOT="$PWD" MODEL_LADDER_GPU_LOCK="$w/lock" DOGFOOD_ALLOW_UNPINNED=1 APR="$w/apr" timeout 60 bash "$prod" --lock-probe qa probe 2>&1); rc=$? + if [ "$rc" = 0 ] && grep -q 'lock=LOCKED oom=1000' <<< "$out"; then echo "ok lock: an apr call runs holding the lock, at oom_score_adj 1000" + else echo "FAIL lock: the probe call did not run holding the lock at oom 1000 (rc=$rc): $out"; bad=1; fi + python3 -c 'import fcntl, sys, time; f = open(sys.argv[1], "a"); fcntl.flock(f, fcntl.LOCK_EX); time.sleep(60)' "$w/lock" & + hp=$! + sleep 0.5 + out=$(FAKE_LOCK="$w/lock" MODEL_LADDER_ROOT="$PWD" MODEL_LADDER_GPU_LOCK="$w/lock" MODEL_LADDER_LOCK_WAIT=1 DOGFOOD_ALLOW_UNPINNED=1 APR="$w/apr" timeout 8 bash "$prod" --lock-probe run probe 2>&1); rc=$? + kill "$hp" 2> /dev/null; wait "$hp" 2> /dev/null + if [ "$rc" = 2 ] && grep -q 'was not free after 1s' <<< "$out" && grep -q "holder: pid $hp" <<< "$out"; then echo "ok lock: a held lock declines (exit 2) in the bounded wait, naming the holder's pid" + else echo "FAIL lock: a held lock did not decline in the bounded wait naming its holder (rc=$rc): $out"; bad=1; fi + return "$bad" +} + if [ "$SELF_TEST" = 1 ]; then n=0; bad=0 for c in "$CASES_DIR"/*/; do @@ -134,7 +249,7 @@ if [ "$SELF_TEST" = 1 ]; then want=$(cat "$c/expected_rc") lad="$c/ladder.yaml"; [ -f "$lad" ] || lad="$LADDER" main=""; [ -f "$c/ladder_main.yaml" ] && main="$c/ladder_main.yaml" - out=$(judge "$lad" "$main" "$c/receipts" "$(cat "$c/version" 2>/dev/null || echo 0.0.0-case)"); got=$? + out=$(judge "$lad" "$main" "$c/receipts" "$(cat "$c/version" 2>/dev/null || echo 0.0.0-case)" "$c/context-rungs.json" "$c/context-rungs_main.json"); got=$? n=$((n+1)) if [ "$got" = "$want" ] && { [ ! -f "$c/must_match" ] || grep -qE "$(cat "$c/must_match")" <<< "$out"; }; then printf 'ok case %-28s rc=%s\n' "$name" "$got" @@ -144,6 +259,59 @@ if [ "$SELF_TEST" = 1 ]; then fi done if [ "$n" -lt 6 ] && [ -z "$ONLY_CASE" ]; then echo "FAIL only $n case(s) ran; the table needs >= 6 to discriminate"; bad=$((bad+1)); fi + if [ -n "$ONLY_CASE" ] && [ "$n" -eq 0 ]; then echo "FAIL no case named $ONLY_CASE under $CASES_DIR -- a case that did not run is not a pass"; bad=$((bad+1)); fi + # Mutants (#3712): each refusal is deleted in a copy of this script, and the case that + # names it must go RED under the copy. A rule no case can tell from its absence is theater. + if [ -z "$ONLY_CASE" ]; then + mdir=$(mktemp -d "${TMPDIR:-/tmp}/ladder-mut.XXXXXX") || exit 2 + mutant() { # mutant