diff --git a/ci/explicit-test-commands.d/450-aprender-contracts-cli-ont-release-readiness.cmd b/ci/explicit-test-commands.d/450-aprender-contracts-cli-ont-release-readiness.cmd new file mode 100644 index 0000000000..102d61ebf3 --- /dev/null +++ b/ci/explicit-test-commands.d/450-aprender-contracts-cli-ont-release-readiness.cmd @@ -0,0 +1 @@ +cargo test -p aprender-contracts-cli --test ont_release_readiness diff --git a/contracts/contracts.nt b/contracts/contracts.nt index 7646da81bd..a958ebaa1e 100644 --- a/contracts/contracts.nt +++ b/contracts/contracts.nt @@ -8911,6 +8911,18 @@ . "contracts/pacha/registry-integrity-v1.yaml"^^ . "registry-integrity-v1"^^ . + . + . + . + . + . + "release-evidence"^^ . + "contracts/release-readiness-v1.yaml"^^ . + "release-readiness-v1"^^ . + "pattern"^^ . + "release-readiness"^^ . + "active"^^ . + "1.0.0"^^ . . . "contracts/release-schedule-06x-v1.yaml"^^ . diff --git a/contracts/lint-baseline.json b/contracts/lint-baseline.json index 5230262959..3dc3abed58 100644 --- a/contracts/lint-baseline.json +++ b/contracts/lint-baseline.json @@ -4,14 +4,14 @@ "armed_shapes": ["ont-shapes-v1", "ladder-measured", "ladder-green"], "ont": { "consumer_present": true, - "contracts_total": 1881, - "entity_types_registered": 10, - "extractors_implemented": 7, - "contracts_anchored": 6, - "contracts_shaped": 5, + "contracts_total": 1882, + "entity_types_registered": 11, + "extractors_implemented": 8, + "contracts_anchored": 7, + "contracts_shaped": 6, "unanchored_but_bindable": 297, "formal_prose": 1464, "legacy_unresolved_depends_on": 8, - "shapes_unarmed": 6 + "shapes_unarmed": 15 } } diff --git a/contracts/ontology.yaml b/contracts/ontology.yaml index e026c513a5..ae7c0aa0c6 100644 --- a/contracts/ontology.yaml +++ b/contracts/ontology.yaml @@ -144,6 +144,9 @@ entity_types: # it moves under a shared `Receipt` class once quorum and dispatch receipts join, so the three families # inherit common shapes. NOT the throughput parity receipts — a different family with a different validator. - {name: parity-receipt, extractor: parity_receipt, implemented: true} + # aprender#3715: a release's receipts (per-host inventory + cells, kernel diffs, context rungs, the dogfood + # receipt R5 judged), extracted ONLY when a release subject is given — an ordinary PR has no release. + - {name: release-evidence, extractor: release_evidence, implemented: true} extractors: - {name: pv_contract, reader: ontology/extract/pv_contract.rs, implemented: true} @@ -156,6 +159,7 @@ extractors: - {name: gguf, reader: ontology/extract/gguf.rs, implemented: true} - {name: csv, reader: ontology/extract/csv.rs, implemented: false} - {name: parity_receipt, reader: ontology/extract/parity_receipt.rs, implemented: true} + - {name: release_evidence, reader: ontology/extract/release_evidence.rs, implemented: true} # What this ontology does NOT express, said out loud so nobody encodes it by accident. Each names the reader that # would have to change first. diff --git a/contracts/release-readiness-v1.yaml b/contracts/release-readiness-v1.yaml new file mode 100644 index 0000000000..bf8bc8d8db --- /dev/null +++ b/contracts/release-readiness-v1.yaml @@ -0,0 +1,262 @@ +# ───────────────────────────────────────────────────────────────────────────── +# release-readiness-v1 — the tag is REFUSED unless every cell of the release matrix has a Pass receipt +# (aprender#3715, EPIC #3710, measured side #3712, T-1/T-4 wiring #3708/#3717) +# +# Operator, 2026-09-21, verbatim: "you need to improve our dogfood process and pv SHACL so leaks don't exist +# pre-release. this is clown town." Every 0.69.0 leak was ABSENCE READ AS CONFORMANCE — a rung marked optional, a +# GPU golden error read as a skip (#3711), an inherited dogfood (#3708), models on a host that were not on the +# ladder (#3712), qwen3moe with no CUDA path and no ticket (#3714). So every obligation is a focus node, and a +# missing receipt is a missing edge that `minCount 1` rejects BY CONSTRUCTION. +# +# WHAT IT GOVERNS +# provable_contracts::ontology::extract::release_evidence — the release's receipts as a graph (Σ extractor +# `release_evidence`, entity type `release-evidence`), built ONLY when a release subject is given +# (`pv lint --gate shapes --shape release-readiness-v1 --release-version V --release-commit MC …`). +# provable_contracts::ontology::extract::release_inputs — the subject, the consumer-derived context rungs +# (evidence/release/context-rungs.json), the dogfood receipt R5 judged, the kernel-diff receipts. +# provable_contracts::ontology::receipts — apr-model-ladder-receipt v1 AND v2 (#3712's). +# +# THE UNIVERSE (computed, never listed). Hosts = every `ladder.hosts[]` entry with `required: true`. Per host, +# models = its measured `inventory[]` ∪ the ladder's cuda rungs listed for it. Per model: verbs {run, chat, serve, +# code} × thinking {on, off} (only off when the host MEASURED no thinking mode, from the template's markers) × +# context rungs: the SHORT ones {4k, 8k, 20k} for every model, the LONG ones {60k, consumer-max (computed = max +# over the consumer records), declared (the model's own GGUF context_length)} for the models +# `ladder.cells.long_rungs_for` names (every model when the ladder names none) — minus any rung larger than the +# model's measured length. Kernels = every (kernel, quant) on ANY required host's dispatch path, owed on EVERY +# required host (each GPU arch). +# +# THE FAMILY. `--shape release-readiness-v1` grades these nine shapes, armed whatever `armed_shapes` says, and +# reports them in this declaration order: release → host → context → model → coverage → tokenizer → kernel → +# refusal → cell, so a tokenizer that disagrees with llama.cpp (#3726) or a kernel that diverges across arches is +# named before any model cell it breaks (#3715 addenda). +# +# THE MEMORY RULE (#3710, operator decision "a"): a (model, host, rung) cell is owed on a host only if it FITS — +# weights + KV-per-token × rung + workspace ≤ the host's measured GPU memory, all carried in the host receipt. +# Where it does not fit, the owed cell is a `release:RefusalCell`: an honest pre-load refusal naming the +# arithmetic. Every rung a model declares must be owed on ≥ 1 required host. An unmeasured term leaves the cell +# owed in full (the stricter reading). +# +# NO WAIVERS. #3710 allows "a FAIL for the release unless the operator waives that cell by name", but every agent +# session posts through the operator's gh token, so a comment-based waiver is forgeable by any of them. Cop +# ruling (c): no waiver input exists; a red cell is released over only by the operator overriding the gate +# himself. The operator-only surface is #3722. +# +# KIND: pattern. Vocabulary, extractor and shapes over a graph; the proof is the gate's own case table. +# ───────────────────────────────────────────────────────────────────────────── +name: release-readiness +version: "1.0.0" +scope: > + Whether a release's evidence proves every (model, host, verb, thinking, context) cell, every (kernel, GPU + arch) cell and every model's tokenizer-parity cell, from receipts measured at the release commit. Out of scope: producing the receipts (#3712, + scripts/model_ladder.sh), calling the gate (#3708/#3717, autopilot T-1 and check_publish_preflight T-4), + waivers (#3722), the intel-CPU column (#3710, aprender-a8). +status: active + +metadata: + version: "1.0.0" + kind: pattern + created: '2026-09-21' + last_modified: '2026-09-21' + author: PAIML Engineering + description: > + The release matrix as focus nodes: every obligation of a release is a node derived from measured + inventory, consumer records and dispatch paths, and each needs exactly one fresh Pass receipt; absence, + a skip, a fallback, a stale sha, an unclosed think block, an empty answer or an unmet context size is a + violation naming the cell. + references: + - 'aprender#3715 (this row), #3710 (the 0.69.1 bar, issuecomment-5763489958), #3712 (the receipt format, issuecomment-5763332078 / -5763412455 / -5763540875), #3716 (consumer records), #3722 (waivers)' + - 'contracts/ont-model-receipts-v1.yaml (the ladder join this generalises), contracts/model-capability-ladder-v1.yaml (the required hosts)' + - 'crates/aprender-contracts/src/ontology/extract/{release_evidence.rs, release_inputs.rs}, ontology/receipts.rs, lint/shapes_gate.rs' + +entity: + type: release-evidence + +relations: + depends_on: [ont-shapes-v1, ont-model-receipts-v1, model-capability-ladder-v1] + +shapes: + - id: release-readiness-v1.release + targetClass: release:Release + properties: + - {path: release:requiredHost, minCount: 1, nodeKind: IRI, class: release:Host} + - {path: release:contextRung, minCount: 1, nodeKind: IRI, class: release:ContextRung} + # the dogfood receipt R5 judged: GO, at THIS commit and version — an inherited GO is stale (#3708) + - {path: release:dogfoodReceipt, minCount: 1, maxCount: 1, nodeKind: IRI, + node: {properties: [{path: release:verdict, minCount: 1, maxCount: 1, in: [GO]}, + {path: release:fresh, minCount: 1, maxCount: 1, in: [true]}]}} + - id: release-readiness-v1.host + targetClass: release:Host + properties: + - {path: release:hostReceipt, minCount: 1, maxCount: 1, nodeKind: IRI, + node: {properties: [{path: release:schema, minCount: 1, in: ["apr-model-ladder-receipt/v2"]}, + {path: release:fresh, minCount: 1, maxCount: 1, in: [true]}]}} + # a host that holds nothing proved nothing + - {path: release:holds, minCount: 1} + - {path: release:unmeasuredModel, maxCount: 0} + - {path: release:unkeyedRow, maxCount: 0} + - {path: release:kernelReceipt, minCount: 1, nodeKind: IRI, + node: {properties: [{path: release:fresh, minCount: 1, maxCount: 1, in: [true]}]}} + - {path: release:dispatchKernel, minCount: 1} + - {path: release:modelWithoutDispatch, maxCount: 0} + - id: release-readiness-v1.context + targetClass: release:ContextRung + properties: + - {path: release:tokens, minCount: 1, maxCount: 1} + - {path: release:derivedFrom, minCount: 1} + - {path: release:unmeasuredConsumer, maxCount: 0} + - id: release-readiness-v1.model + targetClass: release:Model + properties: + # the declared rung's size is read from the GGUF on the host, never picked + - {path: release:contextLength, minCount: 1, maxCount: 1} + # `long_rungs_for` keys on general.architecture (#3712 amendment 2) + - {path: release:arch, minCount: 1, maxCount: 1} + # thinking must be what its template evidence says, and the producer's long-rung echo what pv derives + - {path: release:thinkingContradiction, maxCount: 0} + - {path: release:longRungsMismatch, maxCount: 0} + # the memory rule (#3710, operator "a"): every rung a model declares is owed — and so must Pass — on ≥ 1 + # required host; a rung that fits NOWHERE by the declared arithmetic is named here + - id: release-readiness-v1.coverage + targetClass: release:RungCoverage + properties: + - {path: release:owedOn, minCount: 1, nodeKind: IRI, class: release:Host} + # tokenizer parity (#3726): apr's ids against the pinned llama.cpp, one cell per model — the blind spot of every + # apr-vs-apr gate. Zero tokens compared is no measurement, never parity; the round trip must hold too. + - id: release-readiness-v1.tokenizer + targetClass: release:TokenizerCell + properties: + - {path: release:row, minCount: 1, nodeKind: IRI, + node: {properties: [{path: release:verdict, minCount: 1, maxCount: 1, in: [pass]}, + {path: release:parityExact, minCount: 1, maxCount: 1, in: [true]}, + {path: release:roundTrip, minCount: 1, maxCount: 1, in: [true]}, + {path: release:corpusSha, minCount: 1}, + {path: release:comparatorSha, minCount: 1}, + {path: release:fresh, minCount: 1, maxCount: 1, in: [true]}]}} + - id: release-readiness-v1.kernel + targetClass: release:KernelCell + properties: + - {path: release:row, minCount: 1, nodeKind: IRI, + node: {properties: [{path: release:verdict, minCount: 1, maxCount: 1, in: [pass]}, + {path: release:withinBound, minCount: 1, maxCount: 1, in: [true]}, + {path: release:boundSource, minCount: 1}, + {path: release:fresh, minCount: 1, maxCount: 1, in: [true]}]}} + # a cell that does not FIT the host is owed as an HONEST PRE-LOAD REFUSAL naming its arithmetic — never an OOM + # mid-run, never a silent truncation, never a pass (#3710 rule) + - id: release-readiness-v1.refusal + targetClass: release:RefusalCell + properties: + - {path: release:row, minCount: 1, maxCount: 1, nodeKind: IRI, + node: {properties: [{path: release:verdict, minCount: 1, maxCount: 1, in: [refused]}, + {path: release:arithmeticNamed, minCount: 1, maxCount: 1, in: [true]}, + {path: release:fresh, minCount: 1, maxCount: 1, in: [true]}]}} + - id: release-readiness-v1 + targetClass: release:Cell + properties: + - {path: release:row, minCount: 1, maxCount: 1, nodeKind: IRI, + node: {properties: [{path: release:verdict, minCount: 1, maxCount: 1, in: [pass]}, + {path: release:backend, minCount: 1, maxCount: 1, in: [cuda]}, + {path: release:fallback, minCount: 1, maxCount: 1, in: [false]}, + {path: release:fresh, minCount: 1, maxCount: 1, in: [true]}, + {path: release:contextMet, minCount: 1, maxCount: 1, in: [true]}, + {path: release:thinkOk, minCount: 1, maxCount: 1, in: [true]}, + {path: release:answered, minCount: 1, maxCount: 1, in: [true]}]}} + +equations: + cell_universe: + formula: "Cells = ⋃_{h ∈ H_req} ⋃_{m ∈ inv(h) ∪ rungs_cuda(h)} VERBS × modes(m) × ({r ∈ R : (¬long(r) ∨ owes_long(m)) ∧ (tokens(r) ≤ len(m) ∨ len(m) = ⊥)} ∪ {declared | owes_long(m)})" + domain: "H_req = ladder.hosts with required: true; inv(h) = the host receipt's inventory[] with a sha256; R = the declared context rungs" + codomain: "one release:Cell node per element, each with 0..n release:row edges" + invariants: + - "an inventory row without sha256 is a named unmeasuredModel, never dropped" + - "modes(m) = {off} iff the host measured thinking = false; otherwise {on, off}" + - "owes_long(m) ⇔ long_rungs_for absent ∨ arch(m) ∈ families ∨ representatives[arch(m)] = file(m)" + preconditions: + - "a release subject was given: version and a full 40-hex commit" + postconditions: + - "two extractions of the same inputs are byte-identical (R-15)" + lean_theorem: none — L4 not declared + cell_pass: + formula: "Pass(c) ⇔ |rows(c)| = 1 ∧ verdict = pass ∧ backend = cuda ∧ fallback = false ∧ fresh ∧ contextMet ∧ thinkOk ∧ answered" + domain: "every release:Cell" + codomain: "a PV-ONT-011 violation naming the cell, or none" + invariants: + - "fresh ⇔ receipt.version = V ∧ receipt.apr_sha = (receipts_commit ∨ release_commit)" + - "thinkOk ⇔ thinking = off ∨ think_closed = true" + - "answered ⇔ answer_chars > 0" + - "contextMet ⇔ prompt_tokens ≥ tokens(rung), or for declared prompt_tokens + max_tokens ≥ context_length" + preconditions: + - "an absent field is never the passing value" + postconditions: + - "the gate is Pass only if every cell of every required host passes" + lean_theorem: none — L4 not declared + kernel_pass: + formula: "Pass(k, h) ⇔ |rows(k, h)| ≥ 1 ∧ ∀ row: verdict = pass ∧ max_err ≤ bound ∧ bound_source ≠ ⊥ ∧ fresh" + domain: "K × H_req, K = ⋃_h dispatch kernels(h)" + codomain: "a violation naming (kernel, host) — the arch that diverges" + invariants: + - "a kernel measured on one arch only is a missing row on the other" + preconditions: + - "every required host has a kernel-diff receipt listing its dispatch path" + postconditions: + - "kernel violations are reported before model-cell violations" + lean_theorem: none — L4 not declared + +invariants: + - id: RR-INV-001 + property: a missing cell receipt fails the gate naming the cell + formal: '|rows(c)| = 0 ⇒ Fail(release-readiness-v1, c)' + prose: false + - id: RR-INV-002 + property: a skipped, errored, fallback or stale receipt fails the gate naming the cell + formal: 'verdict(row) ≠ pass ∨ fallback(row) ∨ ¬fresh(row) ⇒ Fail(release-readiness-v1, c)' + prose: false + - id: RR-INV-003 + property: a thinking-ON row with an unclosed block or an empty answer fails + formal: 'thinking(c) = on ∧ (¬think_closed(row) ∨ answer_chars(row) = 0) ⇒ Fail(release-readiness-v1, c)' + prose: false + - id: RR-INV-004 + property: a kernel that passes on one arch and fails on another fails on the failing arch + formal: 'Pass(k, gx10) ∧ ¬Pass(k, lambda) ⇒ Fail(release-readiness-v1.kernel, (k, lambda))' + prose: false + - id: RR-INV-005 + property: a required host with no receipt, or an empty inventory, fails naming the host + formal: '|hostReceipt(h)| = 0 ∨ |holds(h)| = 0 ⇒ Fail(release-readiness-v1.host, h)' + prose: false + - id: RR-INV-007 + property: a model whose tokenizer disagrees with the pinned llama.cpp on one id fails + formal: 'mismatches(model) ≥ 1 ∨ tokens_compared(model) = 0 ⇒ Fail(release-readiness-v1.tokenizer, model)' + prose: false + - id: RR-INV-006 + property: no release subject, no verdict + formal: 'subject = ⊥ ⇒ verdict(--shape release-readiness-v1) = Unknown{NoFocus}' + prose: false + +falsification_tests: + - id: FALSIFY-RR-001 + rule: the universe and the join + prediction: inventory ∪ cuda rungs per host, both thinking modes unless measured otherwise, rungs up to the declared length, rows keyed by (sha256, verb, thinking, rung); a foreign schema is refused by name + test: cargo test -p aprender-contracts --lib ontology::extract::release_evidence + if_fails: a model, mode or rung silently drops out of the universe, or a row keys onto the wrong cell + - id: FALSIFY-RR-002 + rule: v1 and v2 receipts + prediction: a v2 receipt joins its rungs like v1 and carries inventory, apr_sha and cells; a v3 is refused by name + test: cargo test -p aprender-contracts --lib ontology::receipts + if_fails: the first committed v2 receipt turns every PR's shapes gate into exit 3 + - id: FALSIFY-RR-003 + rule: every single-cell failure turns the gate RED + prediction: "the all-green fixture passes with every cell named; a missing, skipped, fallback, stale-sha, unclosed-think, empty-answer, short-context cell each fail naming that cell; a kernel green on sm_121 and red on sm_89 fails naming lambda; a missing host receipt, an inherited dogfood and an absent declared length each fail; no subject declines" + test: cargo test -p aprender-contracts-cli --test ont_release_readiness + if_fails: a leak class from 0.69.0 reaches the tag again + +qa_gate: + id: F-RR-001 + name: release-readiness + description: 'Every release cell is a focus node with exactly one fresh Pass receipt (aprender#3715)' + checks: + - universe_derived_not_listed + - absence_is_a_violation + - single_cell_failures_turn_red + - kernel_arch_divergence_named_first + - no_subject_declines + pass_criteria: All 3 falsification tests pass; `pv lint contracts --gate shapes --shape release-readiness-v1 --release-version V --release-commit MC --receipts DIR --dogfood-receipt FILE` exits 0 only when every cell passes + falsification: 'flip one cell row of tests/fixtures/ont/release-green to verdict skip → the gate exits 1 naming that cell' diff --git a/contracts/shapes.ttl b/contracts/shapes.ttl index ebfcdfafc0..667b5d4245 100644 --- a/contracts/shapes.ttl +++ b/contracts/shapes.ttl @@ -248,3 +248,318 @@ ] ; . + a sh:NodeShape ; + sh:targetClass ; + sh:property [ + sh:path ; + sh:minCount 1 ; + sh:class ; + sh:nodeKind sh:IRI ; + ] ; + sh:property [ + sh:path ; + sh:minCount 1 ; + sh:class ; + sh:nodeKind sh:IRI ; + ] ; + sh:property [ + sh:path ; + sh:minCount 1 ; + sh:maxCount 1 ; + sh:nodeKind sh:IRI ; + sh:node ; + ] ; +. + + a sh:NodeShape ; + sh:property [ + sh:path ; + sh:minCount 1 ; + sh:maxCount 1 ; + sh:in ( "GO" ) ; + ] ; + sh:property [ + sh:path ; + sh:minCount 1 ; + sh:maxCount 1 ; + sh:in ( "true"^^ ) ; + ] ; +. + + a sh:NodeShape ; + sh:targetClass ; + sh:property [ + sh:path ; + sh:minCount 1 ; + sh:maxCount 1 ; + sh:nodeKind sh:IRI ; + sh:node ; + ] ; + sh:property [ + sh:path ; + sh:minCount 1 ; + ] ; + sh:property [ + sh:path ; + sh:maxCount 0 ; + ] ; + sh:property [ + sh:path ; + sh:maxCount 0 ; + ] ; + sh:property [ + sh:path ; + sh:minCount 1 ; + sh:nodeKind sh:IRI ; + sh:node ; + ] ; + sh:property [ + sh:path ; + sh:minCount 1 ; + ] ; + sh:property [ + sh:path ; + sh:maxCount 0 ; + ] ; +. + + a sh:NodeShape ; + sh:property [ + sh:path ; + sh:minCount 1 ; + sh:in ( "apr-model-ladder-receipt/v2" ) ; + ] ; + sh:property [ + sh:path ; + sh:minCount 1 ; + sh:maxCount 1 ; + sh:in ( "true"^^ ) ; + ] ; +. + + a sh:NodeShape ; + sh:property [ + sh:path ; + sh:minCount 1 ; + sh:maxCount 1 ; + sh:in ( "true"^^ ) ; + ] ; +. + + a sh:NodeShape ; + sh:targetClass ; + sh:property [ + sh:path ; + sh:minCount 1 ; + sh:maxCount 1 ; + ] ; + sh:property [ + sh:path ; + sh:minCount 1 ; + ] ; + sh:property [ + sh:path ; + sh:maxCount 0 ; + ] ; +. + + a sh:NodeShape ; + sh:targetClass ; + sh:property [ + sh:path ; + sh:minCount 1 ; + sh:maxCount 1 ; + ] ; + sh:property [ + sh:path ; + sh:minCount 1 ; + sh:maxCount 1 ; + ] ; + sh:property [ + sh:path ; + sh:maxCount 0 ; + ] ; + sh:property [ + sh:path ; + sh:maxCount 0 ; + ] ; +. + + a sh:NodeShape ; + sh:targetClass ; + sh:property [ + sh:path ; + sh:minCount 1 ; + sh:class ; + sh:nodeKind sh:IRI ; + ] ; +. + + a sh:NodeShape ; + sh:targetClass ; + sh:property [ + sh:path ; + sh:minCount 1 ; + sh:nodeKind sh:IRI ; + sh:node ; + ] ; +. + + a sh:NodeShape ; + sh:property [ + sh:path ; + sh:minCount 1 ; + sh:maxCount 1 ; + sh:in ( "pass" ) ; + ] ; + sh:property [ + sh:path ; + sh:minCount 1 ; + sh:maxCount 1 ; + sh:in ( "true"^^ ) ; + ] ; + sh:property [ + sh:path ; + sh:minCount 1 ; + sh:maxCount 1 ; + sh:in ( "true"^^ ) ; + ] ; + sh:property [ + sh:path ; + sh:minCount 1 ; + ] ; + sh:property [ + sh:path ; + sh:minCount 1 ; + ] ; + sh:property [ + sh:path ; + sh:minCount 1 ; + sh:maxCount 1 ; + sh:in ( "true"^^ ) ; + ] ; +. + + a sh:NodeShape ; + sh:targetClass ; + sh:property [ + sh:path ; + sh:minCount 1 ; + sh:nodeKind sh:IRI ; + sh:node ; + ] ; +. + + a sh:NodeShape ; + sh:property [ + sh:path ; + sh:minCount 1 ; + sh:maxCount 1 ; + sh:in ( "pass" ) ; + ] ; + sh:property [ + sh:path ; + sh:minCount 1 ; + sh:maxCount 1 ; + sh:in ( "true"^^ ) ; + ] ; + sh:property [ + sh:path ; + sh:minCount 1 ; + ] ; + sh:property [ + sh:path ; + sh:minCount 1 ; + sh:maxCount 1 ; + sh:in ( "true"^^ ) ; + ] ; +. + + a sh:NodeShape ; + sh:targetClass ; + sh:property [ + sh:path ; + sh:minCount 1 ; + sh:maxCount 1 ; + sh:nodeKind sh:IRI ; + sh:node ; + ] ; +. + + a sh:NodeShape ; + sh:property [ + sh:path ; + sh:minCount 1 ; + sh:maxCount 1 ; + sh:in ( "refused" ) ; + ] ; + sh:property [ + sh:path ; + sh:minCount 1 ; + sh:maxCount 1 ; + sh:in ( "true"^^ ) ; + ] ; + sh:property [ + sh:path ; + sh:minCount 1 ; + sh:maxCount 1 ; + sh:in ( "true"^^ ) ; + ] ; +. + + a sh:NodeShape ; + sh:targetClass ; + sh:property [ + sh:path ; + sh:minCount 1 ; + sh:maxCount 1 ; + sh:nodeKind sh:IRI ; + sh:node ; + ] ; +. + + a sh:NodeShape ; + sh:property [ + sh:path ; + sh:minCount 1 ; + sh:maxCount 1 ; + sh:in ( "pass" ) ; + ] ; + sh:property [ + sh:path ; + sh:minCount 1 ; + sh:maxCount 1 ; + sh:in ( "cuda" ) ; + ] ; + sh:property [ + sh:path ; + sh:minCount 1 ; + sh:maxCount 1 ; + sh:in ( "false"^^ ) ; + ] ; + sh:property [ + sh:path ; + sh:minCount 1 ; + sh:maxCount 1 ; + sh:in ( "true"^^ ) ; + ] ; + sh:property [ + sh:path ; + sh:minCount 1 ; + sh:maxCount 1 ; + sh:in ( "true"^^ ) ; + ] ; + sh:property [ + sh:path ; + sh:minCount 1 ; + sh:maxCount 1 ; + sh:in ( "true"^^ ) ; + ] ; + sh:property [ + sh:path ; + sh:minCount 1 ; + sh:maxCount 1 ; + sh:in ( "true"^^ ) ; + ] ; +. + diff --git a/crates/aprender-contracts-cli/src/cli.rs b/crates/aprender-contracts-cli/src/cli.rs index 538dc5e94f..c790ff16f8 100644 --- a/crates/aprender-contracts-cli/src/cli.rs +++ b/crates/aprender-contracts-cli/src/cli.rs @@ -105,6 +105,12 @@ pub enum Commands { /// Write nothing; exit 1 if the tracked files differ from a fresh extraction (what CI runs) #[arg(long)] check: bool, + /// With `--release-*`: write the corpus graph PLUS the release evidence to this N-Triples file, and leave + /// the tracked contracts.nt / shapes.ttl untouched (aprender#3715) + #[arg(long)] + out: Option, + #[command(flatten)] + release: Box, }, /// Show cross-contract obligation coverage report Coverage { @@ -284,6 +290,12 @@ pub enum Commands { /// Run ONE named gate and report only it (ONT-001 section 5 ONT-2b): `--gate sigma`. #[arg(long)] gate: Option, + /// With `--gate shapes`: grade only this shape family (the shape and every `.*` shape), armed + /// whatever `armed_shapes` says (aprender#3715: `--shape release-readiness-v1`). + #[arg(long)] + shape: Option, + #[command(flatten)] + release: Box, }, /// Score contracts or a codebase directory Score { @@ -469,3 +481,73 @@ pub enum CensusFormat { /// The bytes `contracts/census.json` carries. Json, } + +/// aprender#3715 — the release subject `extract:release-evidence` reads, for `pv lint --gate shapes` and +/// `pv extract`. All absent → no release graph. `--release-version` and `--release-commit` come together; the +/// rest need them. +#[derive(clap::Args, Debug, Default, Clone)] +pub struct ReleaseArgs { + /// The release version whose evidence `release-readiness-v1` grades + #[arg(long)] + pub release_version: Option, + /// The release commit (MC), full 40-hex: the dogfood receipt's commit, and the receipts' unless --receipts-commit + #[arg(long)] + pub release_commit: Option, + /// T-4 only: the sha the committed receipts were measured at, after R7 proved the tree equal modulo evidence/ + #[arg(long)] + pub receipts_commit: Option, + /// The per-host model receipts (default: evidence/dogfood/models//) + #[arg(long)] + pub receipts: Option, + /// The per-host kernel-diff receipts (default: evidence/dogfood/kernels//) + #[arg(long)] + pub kernel_receipts: Option, + /// The dogfood receipt R5 judged (without it the release has no dogfood receipt, which is a violation) + #[arg(long)] + pub dogfood_receipt: Option, + /// The tokenizer-parity receipts, apr vs the pinned llama.cpp (default: evidence/dogfood/tokenizer//) + #[arg(long)] + pub tokenizer_receipts: Option, +} + +impl ReleaseArgs { + /// Did the caller pass any release flag at all? + #[must_use] + pub fn any(&self) -> bool { + self.release_version.is_some() + || self.release_commit.is_some() + || self.receipts_commit.is_some() + || self.receipts.is_some() + || self.kernel_receipts.is_some() + || self.dogfood_receipt.is_some() + || self.tokenizer_receipts.is_some() + } + + /// The subject, or `None` when no flag was passed. A partial set is refused, never completed by a default. + pub fn subject( + &self, + ) -> Result, String> + { + use provable_contracts::ontology::extract::release_inputs::Subject; + if !self.any() { + return Ok(None); + } + let (Some(v), Some(c)) = (&self.release_version, &self.release_commit) else { + return Err( + "--release-version and --release-commit are both required with any --release-*, \ + --receipts*, --kernel-receipts or --dogfood-receipt flag" + .into(), + ); + }; + let mut s = Subject::new(v, c).map_err(|e| e.to_string())?; + if let Some(rc) = &self.receipts_commit { + s = s.with_receipts_commit(rc).map_err(|e| e.to_string())?; + } + s.receipts_dir.clone_from(&self.receipts); + s.kernel_receipts_dir.clone_from(&self.kernel_receipts); + s.dogfood_receipt.clone_from(&self.dogfood_receipt); + s.tokenizer_receipts_dir + .clone_from(&self.tokenizer_receipts); + Ok(Some(s)) + } +} diff --git a/crates/aprender-contracts-cli/src/commands/extract_rdf.rs b/crates/aprender-contracts-cli/src/commands/extract_rdf.rs index 2b5d5f2998..a962e180df 100644 --- a/crates/aprender-contracts-cli/src/commands/extract_rdf.rs +++ b/crates/aprender-contracts-cli/src/commands/extract_rdf.rs @@ -10,6 +10,9 @@ use std::path::Path; use provable_contracts::lint::shapes_gate::collect_shapes; use provable_contracts::ontology::extract; +use provable_contracts::ontology::extract::release_inputs::Subject; + +use crate::contract_walk::ReleaseArgsRefused; use provable_contracts::ontology::shapes::to_turtle; use sha2::{Digest, Sha256}; @@ -22,8 +25,24 @@ struct ExtractReport<'a> { check: Option>, } -/// `check == true` → compare and report drift, write nothing. -pub fn run(contract_dir: &Path, check: bool) -> Result<(), Box> { +/// `check == true` → compare and report drift, write nothing. With a release `subject` (aprender#3715) the +/// release evidence joins the graph, and the result goes ONLY to `out`: the tracked `contracts.nt` is the corpus, +/// and a release's receipts written into it would be a release baked into every later PR's baseline. +pub fn run( + contract_dir: &Path, + check: bool, + subject: Option<&Subject>, + out: Option<&Path>, +) -> Result<(), Box> { + if let Some(subject) = subject { + return run_release(contract_dir, check, subject, out); + } + if out.is_some() { + return Err(ReleaseArgsRefused( + "--out writes the release evidence graph; it needs --release-version and --release-commit".into(), + ) + .into()); + } let extraction = match extract::all(contract_dir) { Ok(x) => x, Err(e) => { @@ -90,3 +109,54 @@ pub fn run(contract_dir: &Path, check: bool) -> Result<(), Box { + triples: usize, + sha256: String, + out: &'a str, + release: provable_contracts::ontology::extract::release_evidence::ReleaseStats, +} + +/// `pv extract --release-version V --release-commit MC … --out FILE`: the corpus graph plus the release evidence, +/// written to FILE (never to the tracked files), with its content address and what was derived. +fn run_release( + contract_dir: &Path, + check: bool, + subject: &Subject, + out: Option<&Path>, +) -> Result<(), Box> { + if check { + return Err(ReleaseArgsRefused( + "--check compares the TRACKED corpus files; a release graph is never tracked, so --check with \ + --release-* has nothing to compare" + .into(), + ) + .into()); + } + let Some(out) = out else { + return Err(ReleaseArgsRefused( + "--release-* needs --out FILE: the release graph is never written into the tracked contracts.nt".into(), + ) + .into()); + }; + let extraction = match extract::all_with(contract_dir, Some(subject)) { + Ok(x) => x, + Err(e) => { + eprintln!("error: {e}"); + std::process::exit(3); + } + }; + let nt = extraction.graph.to_ntriples(); + let mut hasher = Sha256::new(); + hasher.update(nt.as_bytes()); + std::fs::write(out, &nt)?; + let report = ReleaseExtractReport { + triples: extraction.graph.len(), + sha256: format!("{:x}", hasher.finalize()), + out: out.to_str().unwrap_or("?"), + release: extraction.release.unwrap_or_default(), + }; + println!("{}", serde_json::to_string_pretty(&report)?); + Ok(()) +} diff --git a/crates/aprender-contracts-cli/src/commands/lint.rs b/crates/aprender-contracts-cli/src/commands/lint.rs index c63ee905fa..8c9c62febc 100644 --- a/crates/aprender-contracts-cli/src/commands/lint.rs +++ b/crates/aprender-contracts-cli/src/commands/lint.rs @@ -2,6 +2,7 @@ use std::path::Path; use provable_contracts::lint::config::{find_config, load_config}; use provable_contracts::lint::rules::RuleSeverity; +use provable_contracts::lint::shapes_gate::ShapesOptions; use provable_contracts::lint::trend; use provable_contracts::lint::{run_lint, GateDetail, LintConfig, LintReport}; use provable_contracts::ontology::verdict::Verdict; @@ -48,6 +49,7 @@ pub fn run( strict_test_binding: bool, armed_baseline_ref: Option<&str>, gate: Option<&str>, + shapes_opts: ShapesOptions, ) -> Result<(), Box> { refuse_missing_corpus(contract_dir)?; // Both preconditions before any dispatch: a single file under @@ -56,7 +58,7 @@ pub fn run( // missing on exactly the input the refusal exists for. refuse_single_file_strict_binding(contract_dir, strict_test_binding)?; if let Some(name) = gate { - return run_single_gate(contract_dir, name); + return run_single_gate(contract_dir, name, &shapes_opts); } if watch { return run_watch( @@ -160,6 +162,28 @@ fn report_coverage( } } +/// aprender#3715: `--shape` and `--release-*` as the shapes gate's options. They mean something ONLY under +/// `--gate shapes`; anywhere else they are refused (exit 3), because a release flag that was silently ignored is +/// a release gate that silently did not run. +pub fn shapes_options( + gate: Option<&str>, + shape: Option, + release: &crate::cli::ReleaseArgs, +) -> Result { + use crate::contract_walk::ReleaseArgsRefused; + if (shape.is_some() || release.any()) && gate != Some("shapes") { + return Err(ReleaseArgsRefused( + "--shape and --release-* / --receipts* / --kernel-receipts / --dogfood-receipt apply only to \ + `--gate shapes`" + .into(), + )); + } + Ok(ShapesOptions { + only: shape, + release: release.subject().map_err(ReleaseArgsRefused)?, + }) +} + /// One gate's report. NOT a `LintReport`: `--gate` answers about one gate, and a reader must be able to tell the /// two apart without counting keys. #[derive(serde::Serialize)] @@ -187,8 +211,12 @@ struct SingleGateFinding<'a> { /// ONT-2b: `--gate ` runs ONE gate and reports only it, mapping its verdict through ONT-6's lattice — /// Pass 0 · Fail 1 `reject:` · no Σ 2 `decline:` (R-2, zero is a decline) · malformed Σ 3 `error:`. -fn run_single_gate(contract_dir: &Path, name: &str) -> Result<(), Box> { - let (result, findings) = decide_named_gate(contract_dir, name)?; +fn run_single_gate( + contract_dir: &Path, + name: &str, + shapes_opts: &ShapesOptions, +) -> Result<(), Box> { + let (result, findings) = decide_named_gate(contract_dir, name, shapes_opts)?; let report = SingleGateReport { gate: &result.name, @@ -235,12 +263,13 @@ type NamedGateAnswer = ( fn decide_named_gate( contract_dir: &Path, name: &str, + shapes_opts: &ShapesOptions, ) -> Result> { use provable_contracts::lint::{ relations_gate::RelationsOutcome, sigma_gate::SigmaOutcome, NamedGateOutcome, NAMED_GATES, }; - match provable_contracts::lint::run_named_gate(contract_dir, name) { + match provable_contracts::lint::run_named_gate_with(contract_dir, name, shapes_opts) { NamedGateOutcome::UnknownGate => Err(crate::contract_walk::UnknownGate { asked: name.to_string(), known: NAMED_GATES.iter().map(|g| (*g).to_string()).collect(), diff --git a/crates/aprender-contracts-cli/src/contract_walk.rs b/crates/aprender-contracts-cli/src/contract_walk.rs index d04c0299e3..45dff87517 100644 --- a/crates/aprender-contracts-cli/src/contract_walk.rs +++ b/crates/aprender-contracts-cli/src/contract_walk.rs @@ -226,6 +226,19 @@ impl fmt::Display for UnknownGate { impl std::error::Error for UnknownGate {} +/// aprender#3715: a `--shape` / `--release-*` flag set that cannot name a release (a partial subject, a short +/// sha, a flag outside `--gate shapes`). The CALLER's error, exit 3 — never a verdict about the evidence. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ReleaseArgsRefused(pub String); + +impl fmt::Display for ReleaseArgsRefused { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{}", self.0) + } +} + +impl std::error::Error for ReleaseArgsRefused {} + /// Exit status of an `armed_gates` list that dropped a gate its comparand armed (ONT-001 §3.9). pub const ARMED_GATES_SHRANK_EXIT: i32 = 3; @@ -239,6 +252,7 @@ pub fn exit_code_for(err: &(dyn std::error::Error + 'static)) -> i32 { } else if err.downcast_ref::().is_some() || err.downcast_ref::().is_some() || err.downcast_ref::().is_some() + || err.downcast_ref::().is_some() { ARMED_GATES_SHRANK_EXIT } else { diff --git a/crates/aprender-contracts-cli/src/lib.rs b/crates/aprender-contracts-cli/src/lib.rs index 9dc9adee4f..a1050b0411 100644 --- a/crates/aprender-contracts-cli/src/lib.rs +++ b/crates/aprender-contracts-cli/src/lib.rs @@ -125,7 +125,14 @@ pub fn dispatch(command: Commands) -> Result<(), Box> { Commands::Extract { contract_dir, check, - } => commands::extract_rdf::run(&contract_dir, check), + out, + release, + } => { + let subject = release + .subject() + .map_err(crate::contract_walk::ReleaseArgsRefused)?; + commands::extract_rdf::run(&contract_dir, check, subject.as_ref(), out.as_deref()) + } Commands::Coverage { contract_dir, binding, @@ -205,6 +212,8 @@ pub fn dispatch(command: Commands) -> Result<(), Box> { strict_test_binding, armed_baseline_ref, gate, + shape, + release, .. } => { if let Some(ref rule_id) = explain { @@ -236,6 +245,7 @@ pub fn dispatch(command: Commands) -> Result<(), Box> { strict_test_binding, armed_baseline_ref.as_deref(), gate.as_deref(), + commands::lint::shapes_options(gate.as_deref(), shape, &release)?, ) } Commands::Score { diff --git a/crates/aprender-contracts-cli/tests/ont4b_shapes_gate.rs b/crates/aprender-contracts-cli/tests/ont4b_shapes_gate.rs index 4b2f19820f..1057520206 100644 --- a/crates/aprender-contracts-cli/tests/ont4b_shapes_gate.rs +++ b/crates/aprender-contracts-cli/tests/ont4b_shapes_gate.rs @@ -228,11 +228,13 @@ fn the_tracked_repo_graph_is_fresh() { // is shared across branches, so a sibling PR that also adds a shape // (#3600's parity-receipt-v2) will need the number raised again at merge — // that is the ratchet working, not a conflict to route around. It did: the - // 0.69 batch folded #3600 in and the count went 6 -> 9 with its three shapes. + // 0.69 batch folded #3600 in and the count went 6 -> 9 with its three shapes. #3715 added the nine-shape + // `release-readiness-v1` family (shapes_n=18, triples=15863, measured on its branch); it contributes no focus + // node to a PR's graph — the release evidence is extracted only under `--release-*`. assert_eq!( v["shapes_n"], - 9, - "ont-shapes-v1 + ladder-measured + ladder-green (ONT-4c1) + bound-symbols-resolve + lean-statements-grounded (ONT-4b2) + refusal-receipt-v1 (#3605) + parity-receipt-complete + parity-comparator-self + parity-comparator-oracle (parity-receipt-v2, #3600)\n{}", + 18, + "ont-shapes-v1 + ladder-measured + ladder-green (ONT-4c1) + bound-symbols-resolve + lean-statements-grounded (ONT-4b2) + refusal-receipt-v1 (#3605) + parity-receipt-complete + parity-comparator-self + parity-comparator-oracle (parity-receipt-v2, #3600) + release-readiness-v1{{,.release,.host,.context,.model,.coverage,.tokenizer,.kernel,.refusal}} (#3715)\n{}", show(&r) ); } diff --git a/crates/aprender-contracts-cli/tests/ont_release_readiness.rs b/crates/aprender-contracts-cli/tests/ont_release_readiness.rs new file mode 100644 index 0000000000..7f60586a5b --- /dev/null +++ b/crates/aprender-contracts-cli/tests/ont_release_readiness.rs @@ -0,0 +1,812 @@ +//! aprender#3715 — `release-readiness-v1` on the CLI: the red-turning case table. +//! +//! Every case copies `tests/fixtures/ont/release-green/` AND the real `contracts/release-readiness-v1.yaml` (so the +//! fixture cannot drift from the shapes it proves) into a scratch repo, changes exactly ONE thing, and runs +//! `pv lint contracts --gate shapes --shape release-readiness-v1 --release-version 0.69.1 --release-commit MC …`. +//! +//! DISCRIMINATION: the green base must PASS with every cell named (48/48, 4 kernel cells) — so a build that +//! rejects everything fails this file — and each single-cell change must FAIL naming that cell — so a build that +//! accepts everything fails it too. The ONT-001 rule: a shape that has never gone red is not armed. + +use std::path::{Path, PathBuf}; +use std::process::Command; + +use serde_json::Value; + +const MC: &str = "1111111111111111111111111111111111111111"; +const PARENT: &str = "3333333333333333333333333333333333333333"; +const BUMP: &str = "2222222222222222222222222222222222222222"; +const MODEL: &str = "tiny-qwen35.gguf"; + +struct Run { + code: i32, + stdout: String, + stderr: String, +} + +fn show(r: &Run) -> String { + format!( + "exit {}\n--- stdout\n{}\n--- stderr\n{}", + r.code, r.stdout, r.stderr + ) +} + +fn repo_root() -> PathBuf { + Path::new(env!("CARGO_MANIFEST_DIR")).join("../..") +} + +fn copy_dir(from: &Path, to: &Path) { + std::fs::create_dir_all(to).expect("mkdir"); + for e in std::fs::read_dir(from).expect("read fixture dir").flatten() { + let p = e.path(); + let dst = to.join(e.file_name()); + if p.is_dir() { + copy_dir(&p, &dst); + } else { + std::fs::copy(&p, &dst).expect("copy fixture file"); + } + } +} + +/// The green base: the fixture plus the REAL shape contract. +fn green() -> tempfile::TempDir { + let t = tempfile::tempdir().expect("tempdir"); + copy_dir( + &repo_root().join("tests/fixtures/ont/release-green"), + t.path(), + ); + std::fs::copy( + repo_root().join("contracts/release-readiness-v1.yaml"), + t.path().join("contracts/release-readiness-v1.yaml"), + ) + .expect("copy the real contract"); + t +} + +fn pv(cwd: &Path, args: &[&str]) -> Run { + let out = Command::new(env!("CARGO_BIN_EXE_pv")) + .current_dir(cwd) + .args(args) + .output() + .expect("spawn pv"); + Run { + code: out.status.code().unwrap_or(-1), + stdout: String::from_utf8_lossy(&out.stdout).into_owned(), + stderr: String::from_utf8_lossy(&out.stderr).into_owned(), + } +} + +fn s(p: &Path) -> String { + p.to_str().expect("utf-8 path").to_string() +} + +/// The release gate as T-1 calls it, plus `extra` flags. +fn gate(t: &Path, extra: &[&str]) -> Run { + let contracts = s(&t.join("contracts")); + let dogfood = s(&t.join("dogfood-receipt.json")); + let mut args = vec![ + "lint", + &contracts, + "--gate", + "shapes", + "--shape", + "release-readiness-v1", + "--release-version", + "0.69.1", + "--release-commit", + MC, + "--dogfood-receipt", + &dogfood, + ]; + args.extend_from_slice(extra); + pv(t, &args) +} + +fn json_of(r: &Run) -> Value { + serde_json::from_str(&r.stdout).unwrap_or_else(|e| panic!("stdout is JSON: {e}\n{}", show(r))) +} + +fn edit(path: &Path, f: impl FnOnce(&mut Value)) { + let mut v: Value = + serde_json::from_str(&std::fs::read_to_string(path).expect("read")).expect("json"); + f(&mut v); + std::fs::write(path, serde_json::to_string_pretty(&v).expect("ser")).expect("write"); +} + +fn models(t: &Path, host: &str) -> PathBuf { + t.join(format!("evidence/dogfood/models/0.69.1/{host}.json")) +} + +fn kernels(t: &Path, host: &str) -> PathBuf { + t.join(format!("evidence/dogfood/kernels/0.69.1/{host}.json")) +} + +/// Index of the one row for (verb, thinking, rung) in a model receipt. +fn row_index(v: &Value, verb: &str, thinking: &str, rung: &str) -> usize { + v["cells"] + .as_array() + .expect("cells") + .iter() + .position(|c| c["verb"] == verb && c["thinking"] == thinking && c["context"] == rung) + .unwrap_or_else(|| panic!("no row {verb}/{thinking}/{rung}")) +} + +fn row<'a>(v: &'a mut Value, verb: &str, thinking: &str, rung: &str) -> &'a mut Value { + let i = row_index(v, verb, thinking, rung); + &mut v["cells"][i] +} + +fn cell(host: &str, verb: &str, thinking: &str, rung: &str) -> String { + format!("release-cell/0.69.1/{host}/{MODEL}/{verb}/think-{thinking}/{rung}") +} + +/// RED, and the findings name `needle`. +fn assert_red_naming(r: &Run, needle: &str) -> Value { + assert_eq!(r.code, 1, "{}", show(r)); + let v = json_of(r); + assert_eq!(v["verdict"], "Fail", "{}", show(r)); + let msgs: Vec<&str> = v["findings"] + .as_array() + .expect("findings") + .iter() + .filter_map(|f| f["message"].as_str()) + .collect(); + assert!( + msgs.iter().any(|m| m.contains(needle)), + "no finding names {needle}:\n{}", + msgs.join("\n") + ); + v +} + +#[test] +fn the_green_release_passes_with_every_cell_named() { + let t = green(); + let r = gate(t.path(), &[]); + assert_eq!(r.code, 0, "{}", show(&r)); + let v = json_of(&r); + assert_eq!(v["verdict"], "Pass"); + assert_eq!( + v["pc_shape"], "fired", + "the plant fired from the named family" + ); + assert_eq!( + v["pc_extract"]["release-evidence"], "fired", + "the extractor's own control (PMAT-3704)" + ); + let rel = &v["release"]; + assert_eq!( + rel["cells"], 48, + "2 hosts × 4 verbs × 2 thinking × (4k, consumer-max, declared)" + ); + assert_eq!(rel["cells_with_row"], 48); + assert_eq!(rel["cell_names"].as_array().map(Vec::len), Some(48)); + assert_eq!( + rel["kernel_cells"], 4, + "(q4k_gemv@q4_k, rmsnorm@f32) × 2 hosts" + ); + assert_eq!(rel["orphan_rows"], 0); + assert_eq!( + rel["tokenizer_cells"], 1, + "one distinct model across both hosts (#3726)" + ); + let armed = v["armed_shapes"].as_array().expect("armed").len(); + assert_eq!( + armed, 9, + "--shape arms the whole family whatever armed_shapes says" + ); +} + +#[test] +fn a_missing_cell_is_red_naming_it() { + let t = green(); + edit(&models(t.path(), "lambda"), |v| { + let i = row_index(v, "chat", "on", "consumer-max"); + v["cells"].as_array_mut().expect("cells").remove(i); + }); + assert_red_naming( + &gate(t.path(), &[]), + &cell("lambda", "chat", "on", "consumer-max"), + ); +} + +#[test] +fn a_skipped_cell_is_red_naming_it() { + let t = green(); + edit(&models(t.path(), "gx10"), |v| { + row(v, "serve", "off", "4k")["verdict"] = "skip".into(); + }); + assert_red_naming(&gate(t.path(), &[]), &cell("gx10", "serve", "off", "4k")); +} + +#[test] +fn a_fallback_cell_is_red_naming_it() { + let t = green(); + edit(&models(t.path(), "lambda"), |v| { + row(v, "run", "off", "4k")["fallback"] = true.into(); + }); + assert_red_naming(&gate(t.path(), &[]), &cell("lambda", "run", "off", "4k")); +} + +#[test] +fn a_cell_that_does_not_say_whether_it_fell_back_is_red() { + let t = green(); + edit(&models(t.path(), "lambda"), |v| { + row(v, "code", "off", "4k") + .as_object_mut() + .expect("row") + .remove("fallback"); + }); + assert_red_naming(&gate(t.path(), &[]), &cell("lambda", "code", "off", "4k")); +} + +#[test] +fn a_receipt_measured_at_another_commit_is_stale_and_red() { + let t = green(); + edit(&models(t.path(), "lambda"), |v| { + v["apr_sha"] = PARENT.into() + }); + let v = assert_red_naming(&gate(t.path(), &[]), "release-host/0.69.1/lambda"); + let msgs = v["findings"].to_string(); + assert!( + msgs.contains(&cell("lambda", "run", "off", "4k")), + "every lambda cell is stale" + ); + assert!( + !msgs.contains("release-host/0.69.1/gx10"), + "gx10 is untouched" + ); +} + +#[test] +fn a_receipt_for_another_version_is_stale_and_red() { + let t = green(); + edit(&models(t.path(), "gx10"), |v| { + v["version"] = "0.69.0".into() + }); + assert_red_naming(&gate(t.path(), &[]), "release-host/0.69.1/gx10"); +} + +#[test] +fn a_think_on_row_with_an_unclosed_block_and_an_empty_answer_is_red() { + let t = green(); + edit(&models(t.path(), "lambda"), |v| { + let r = row(v, "chat", "on", "4k"); + r["think_closed"] = false.into(); + r["answer_chars"] = 0.into(); + }); + let v = assert_red_naming(&gate(t.path(), &[]), &cell("lambda", "chat", "on", "4k")); + let msgs = v["findings"].to_string(); + assert!(msgs.contains("thinkOk"), "names the unclosed block: {msgs}"); + assert!(msgs.contains("answered"), "names the empty answer: {msgs}"); +} + +#[test] +fn a_missing_think_on_cell_is_red_naming_it() { + let t = green(); + edit(&models(t.path(), "gx10"), |v| { + let i = row_index(v, "code", "on", "declared"); + v["cells"].as_array_mut().expect("cells").remove(i); + }); + assert_red_naming( + &gate(t.path(), &[]), + &cell("gx10", "code", "on", "declared"), + ); +} + +#[test] +fn an_absent_declared_length_rung_is_red_and_so_is_an_unmeasured_length() { + let t = green(); + edit(&models(t.path(), "lambda"), |v| { + let i = row_index(v, "run", "off", "declared"); + v["cells"].as_array_mut().expect("cells").remove(i); + }); + assert_red_naming( + &gate(t.path(), &[]), + &cell("lambda", "run", "off", "declared"), + ); + + // the length is a property of the FILE (one sha256): gx10 dropping it leaves lambda's measurement standing, + // but gx10's own declared cells have no size to fill and go red … + let t = green(); + edit(&models(t.path(), "gx10"), |v| { + v["inventory"][0] + .as_object_mut() + .expect("item") + .remove("context_length"); + }); + assert_red_naming( + &gate(t.path(), &[]), + &cell("gx10", "run", "off", "declared"), + ); + // … and with no host measuring it, the model itself is named + let t = green(); + for h in ["lambda", "gx10"] { + edit(&models(t.path(), h), |v| { + v["inventory"][0] + .as_object_mut() + .expect("item") + .remove("context_length"); + }); + } + assert_red_naming(&gate(t.path(), &[]), "contextLength"); +} + +#[test] +fn two_hosts_that_disagree_on_one_files_length_are_red() { + let t = green(); + edit(&models(t.path(), "gx10"), |v| { + v["inventory"][0]["context_length"] = 4096.into(); + }); + let v = assert_red_naming(&gate(t.path(), &[]), "contextLength"); + assert!(v["findings"].to_string().contains("maxCount")); +} + +#[test] +fn a_prompt_short_of_its_rung_is_red() { + let t = green(); + edit(&models(t.path(), "lambda"), |v| { + row(v, "serve", "on", "consumer-max")["prompt_tokens"] = 1000.into(); + }); + let v = assert_red_naming( + &gate(t.path(), &[]), + &cell("lambda", "serve", "on", "consumer-max"), + ); + assert!(v["findings"].to_string().contains("contextMet")); +} + +#[test] +fn two_rows_for_one_cell_are_ambiguous_and_red() { + let t = green(); + edit(&models(t.path(), "lambda"), |v| { + let dup = row(v, "run", "on", "4k").clone(); + v["cells"].as_array_mut().expect("cells").push(dup); + }); + let v = assert_red_naming(&gate(t.path(), &[]), &cell("lambda", "run", "on", "4k")); + assert!(v["findings"].to_string().contains("maxCount")); +} + +#[test] +fn a_kernel_green_on_sm_121_and_red_on_sm_89_is_named_on_lambda_before_any_model_cell() { + let t = green(); + edit(&kernels(t.path(), "lambda"), |v| { + v["rows"][0]["verdict"] = "fail".into(); + v["rows"][0]["max_err"] = 0.5.into(); + }); + // and one model cell too, so the ORDER is observable + edit(&models(t.path(), "lambda"), |v| { + row(v, "run", "off", "4k")["verdict"] = "fail".into(); + }); + let v = assert_red_naming( + &gate(t.path(), &[]), + "release-kernel/0.69.1/lambda/q4k_gemv/q4_k", + ); + let findings = v["findings"].as_array().expect("findings"); + let first = findings[0]["message"].as_str().unwrap_or(""); + assert!( + first.contains("release-kernel/0.69.1/lambda"), + "kernel first: {first}" + ); + assert!( + !v["findings"] + .to_string() + .contains("release-kernel/0.69.1/gx10"), + "gx10's kernel is green" + ); +} + +#[test] +fn a_kernel_measured_on_one_arch_only_is_a_missing_row_on_the_other() { + let t = green(); + edit(&kernels(t.path(), "gx10"), |v| { + v["rows"].as_array_mut().expect("rows").remove(1); // rmsnorm@f32 unmeasured on gx10 + }); + assert_red_naming( + &gate(t.path(), &[]), + "release-kernel/0.69.1/gx10/rmsnorm/f32", + ); +} + +#[test] +fn a_required_host_with_no_receipt_is_red_naming_the_host() { + let t = green(); + std::fs::remove_file(models(t.path(), "gx10")).expect("rm"); + assert_red_naming(&gate(t.path(), &[]), "release-host/0.69.1/gx10"); +} + +#[test] +fn an_inherited_dogfood_or_none_at_all_is_red() { + let t = green(); + edit(&t.path().join("dogfood-receipt.json"), |v| { + v["commit"] = PARENT.into(); + }); + assert_red_naming(&gate(t.path(), &[]), "release-dogfood/0.69.1"); + + let t = green(); + let contracts = s(&t.path().join("contracts")); + let r = pv( + t.path(), + &[ + "lint", + &contracts, + "--gate", + "shapes", + "--shape", + "release-readiness-v1", + "--release-version", + "0.69.1", + "--release-commit", + MC, + ], + ); + assert_red_naming(&r, "dogfoodReceipt"); +} + +#[test] +fn a_consumer_with_no_recorded_number_is_red_on_the_derived_rung() { + let t = green(); + edit(&t.path().join("evidence/release/context-rungs.json"), |v| { + v["consumers"].as_array_mut().expect("consumers").push( + serde_json::from_str::( + r#"{"consumer": "pending", "basis": "", "source": "fixture://pending"}"#, + ) + .expect("a consumer record"), + ); + }); + assert_red_naming(&gate(t.path(), &[]), "release-context/consumer-max"); +} + +#[test] +fn thinking_that_contradicts_its_template_markers_is_red() { + let t = green(); + edit(&models(t.path(), "lambda"), |v| { + // off only, though the template carries the `enable_thinking` switch (#3723's derivation says both) + v["inventory"][0]["thinking_modes"] = + serde_json::from_str::(r#"["off"]"#).expect("modes"); + }); + assert_red_naming(&gate(t.path(), &[]), "thinkingContradiction"); +} + +#[test] +fn a_v1_receipt_cannot_stand_for_a_host() { + let t = green(); + edit(&models(t.path(), "gx10"), |v| { + v["schema"] = "apr-model-ladder-receipt/v1".into(); + }); + assert_red_naming(&gate(t.path(), &[]), "release-host/0.69.1/gx10"); +} + +#[test] +fn t4_receipts_measured_at_the_bump_head_pass_only_with_receipts_commit() { + // T-4: the committed receipts were measured at the bump head; R7 proved it equal to MC modulo evidence/ + let t = green(); + let other = t.path().join("t4-receipts"); + std::fs::create_dir_all(&other).expect("mkdir"); + for h in ["lambda", "gx10"] { + let dst = other.join(format!("{h}.json")); + std::fs::rename(models(t.path(), h), &dst).expect("mv"); + edit(&dst, |v| v["apr_sha"] = BUMP.into()); + edit(&kernels(t.path(), h), |v| v["apr_sha"] = BUMP.into()); + edit(&tokenizer(t.path()), |v| v["apr_sha"] = BUMP.into()); + } + let dir = s(&other); + let r = gate(t.path(), &["--receipts", &dir, "--receipts-commit", BUMP]); + assert_eq!(r.code, 0, "{}", show(&r)); + let r = gate(t.path(), &["--receipts", &dir]); + assert_red_naming(&r, "release-host/0.69.1/lambda"); +} + +#[test] +fn no_release_subject_declines_it_never_passes() { + let t = green(); + let contracts = s(&t.path().join("contracts")); + let r = pv( + t.path(), + &[ + "lint", + &contracts, + "--gate", + "shapes", + "--shape", + "release-readiness-v1", + ], + ); + assert_eq!( + r.code, + 2, + "zero cells is a decline, not a pass: {}", + show(&r) + ); +} + +#[test] +fn a_malformed_subject_or_a_misplaced_flag_is_the_callers_error() { + let t = green(); + let contracts = s(&t.path().join("contracts")); + let partial = pv( + t.path(), + &[ + "lint", + &contracts, + "--gate", + "shapes", + "--release-version", + "0.69.1", + ], + ); + assert_eq!(partial.code, 3, "{}", show(&partial)); + let short = pv( + t.path(), + &[ + "lint", + &contracts, + "--gate", + "shapes", + "--release-version", + "0.69.1", + "--release-commit", + "225b2a9ab", + ], + ); + assert_eq!( + short.code, + 3, + "a prefix is refused, never matched: {}", + show(&short) + ); + let misplaced = pv( + t.path(), + &["lint", &contracts, "--gate", "sigma", "--shape", "x"], + ); + assert_eq!(misplaced.code, 3, "{}", show(&misplaced)); + let unknown = pv( + t.path(), + &[ + "lint", + &contracts, + "--gate", + "shapes", + "--shape", + "no-such-shape", + ], + ); + assert_eq!(unknown.code, 3, "{}", show(&unknown)); +} + +#[test] +fn pv_extract_writes_the_release_graph_only_where_it_is_told() { + let t = green(); + let contracts = s(&t.path().join("contracts")); + let out = t.path().join("release.nt"); + let dogfood = s(&t.path().join("dogfood-receipt.json")); + let args = [ + "extract", + &contracts, + "--release-version", + "0.69.1", + "--release-commit", + MC, + "--dogfood-receipt", + &dogfood, + ]; + let refused = pv(t.path(), &args); + assert_eq!(refused.code, 3, "no --out: {}", show(&refused)); + assert!( + !t.path().join("contracts/contracts.nt").exists(), + "the tracked file is untouched" + ); + let out_s = s(&out); + let mut with_out = args.to_vec(); + with_out.extend_from_slice(&["--out", &out_s]); + let r = pv(t.path(), &with_out); + assert_eq!(r.code, 0, "{}", show(&r)); + let v = json_of(&r); + assert_eq!(v["release"]["cells"], 48); + let nt = std::fs::read_to_string(&out).expect("written"); + assert!(nt.contains("release/Cell"), "the cells are in the graph"); + assert!(nt.contains("release/KernelDiffReceipt")); + assert!(!nt.contains("_:"), "no blank nodes (R-15)"); + assert!(!t.path().join("contracts/contracts.nt").exists()); +} + +fn tokenizer(t: &Path) -> PathBuf { + t.join("evidence/dogfood/tokenizer/0.69.1/parity.json") +} + +const TOK_CELL: &str = "release-tokenizer/0.69.1/tiny-qwen35.gguf"; + +#[test] +fn one_token_id_that_disagrees_with_llama_cpp_is_red_naming_the_model() { + // #3726: apr mapped every non-ASCII byte to id 0 and `apr parity` could not see it (CPU and GPU share it) + let t = green(); + edit(&tokenizer(t.path()), |v| { + v["rows"][0]["mismatches"] = 1.into() + }); + let v = assert_red_naming(&gate(t.path(), &[]), TOK_CELL); + assert!(v["findings"].to_string().contains("parityExact")); +} + +#[test] +fn a_tokenizer_comparison_over_nothing_or_a_broken_round_trip_or_no_receipt_is_red() { + let t = green(); + edit(&tokenizer(t.path()), |v| { + v["rows"][0]["tokens_compared"] = 0.into() + }); + assert_red_naming(&gate(t.path(), &[]), TOK_CELL); + + let t = green(); + edit(&tokenizer(t.path()), |v| { + v["rows"][0]["roundtrip_ok"] = false.into() + }); + let v = assert_red_naming(&gate(t.path(), &[]), TOK_CELL); + assert!(v["findings"].to_string().contains("roundTrip")); + + let t = green(); + std::fs::remove_file(tokenizer(t.path())).expect("rm"); + assert_red_naming(&gate(t.path(), &[]), TOK_CELL); +} + +// ── the memory rule (#3710, operator decision "a") ────────────────────────────────────────────────────── +// weights 16 GB + 1 MB/token KV + 1 GB workspace: `4k` (21.1 GB) fits the 24 GB card; `consumer-max` (25 GB) and +// `declared` (27 GB) do not, and are owed on lambda as honest pre-load refusals — and on gx10 (119 GB) as passes. + +const GB: u64 = 1_000_000_000; + +fn memory_split(t: &Path) { + for (h, gpu) in [("lambda", 24 * GB), ("gx10", 119 * GB)] { + edit(&models(t, h), |v| { + v["gpu_mem_total_bytes"] = gpu.into(); + let item = &mut v["inventory"][0]; + item["weights_bytes"] = (16 * GB).into(); + item["kv_bytes_per_token"] = 1_000_000u64.into(); + item["workspace_bytes"] = GB.into(); + }); + } + edit(&models(t, "lambda"), |v| { + for c in v["cells"].as_array_mut().expect("cells") { + let need = match c["context"].as_str() { + Some("consumer-max") => 25 * GB, + Some("declared") => 27 * GB, + _ => continue, + }; + c["verdict"] = "refused".into(); + c["required_bytes"] = need.into(); + c["available_bytes"] = (24 * GB).into(); + c["reason"] = + "pre-load refusal: KV for this context needs more than the device holds".into(); + } + }); +} + +#[test] +fn a_27b_that_refuses_honestly_on_lambda_and_passes_on_gx10_is_green() { + let t = green(); + memory_split(t.path()); + let r = gate(t.path(), &[]); + assert_eq!(r.code, 0, "{}", show(&r)); + let v = json_of(&r); + assert_eq!( + v["release"]["refusal_cells"], 16, + "4 verbs × 2 modes × (consumer-max, declared) on lambda" + ); + assert_eq!(v["release"]["cells"], 48, "a refusal is still an owed cell"); +} + +#[test] +fn the_gx10_cell_that_covers_the_rung_missing_is_red() { + let t = green(); + memory_split(t.path()); + edit(&models(t.path(), "gx10"), |v| { + let i = row_index(v, "chat", "off", "declared"); + v["cells"].as_array_mut().expect("cells").remove(i); + }); + assert_red_naming( + &gate(t.path(), &[]), + &cell("gx10", "chat", "off", "declared"), + ); +} + +#[test] +fn an_oom_mid_run_where_a_refusal_is_owed_is_red() { + let t = green(); + memory_split(t.path()); + edit(&models(t.path(), "lambda"), |v| { + let r = row(v, "serve", "on", "declared"); + r["verdict"] = "error".into(); + r["reason"] = "CUDA_ERROR_OUT_OF_MEMORY after 41 s of prefill".into(); + r.as_object_mut().expect("row").remove("required_bytes"); + r.as_object_mut().expect("row").remove("available_bytes"); + }); + let v = assert_red_naming( + &gate(t.path(), &[]), + &cell("lambda", "serve", "on", "declared"), + ); + assert!(v["findings"] + .to_string() + .contains("release-readiness-v1.refusal")); +} + +#[test] +fn a_refusal_whose_arithmetic_shows_no_shortfall_is_red() { + let t = green(); + memory_split(t.path()); + edit(&models(t.path(), "lambda"), |v| { + row(v, "code", "off", "consumer-max")["required_bytes"] = (20 * GB).into(); + }); + let v = assert_red_naming( + &gate(t.path(), &[]), + &cell("lambda", "code", "off", "consumer-max"), + ); + assert!(v["findings"].to_string().contains("arithmeticNamed")); +} + +#[test] +fn a_cell_that_fits_by_the_arithmetic_but_is_missing_is_red() { + let t = green(); + memory_split(t.path()); + edit(&models(t.path(), "lambda"), |v| { + let i = row_index(v, "run", "on", "4k"); + v["cells"].as_array_mut().expect("cells").remove(i); + }); + assert_red_naming(&gate(t.path(), &[]), &cell("lambda", "run", "on", "4k")); +} + +#[test] +fn a_rung_that_fits_on_no_required_host_is_red_naming_the_rung() { + let t = green(); + memory_split(t.path()); + edit(&models(t.path(), "gx10"), |v| { + v["gpu_mem_total_bytes"] = (24 * GB).into(); + }); + assert_red_naming( + &gate(t.path(), &[]), + &format!("release-coverage/0.69.1/{MODEL}/declared"), + ); +} + +#[test] +fn a_refusal_because_a_co_tenant_holds_memory_is_red() { + // required 23 GB ≤ the card's 24 GB total, refused because only 20 GB were free: a busy GPU, not the + // arithmetic — it must not shrink what the release proves (#3712, aprender-62) + let t = green(); + memory_split(t.path()); + edit(&models(t.path(), "lambda"), |v| { + let r = row(v, "chat", "off", "consumer-max"); + r["required_bytes"] = (23 * GB).into(); + r["available_bytes"] = (20 * GB).into(); + }); + let v = assert_red_naming( + &gate(t.path(), &[]), + &cell("lambda", "chat", "off", "consumer-max"), + ); + assert!(v["findings"].to_string().contains("arithmeticNamed")); +} + +#[test] +fn a_model_whose_template_always_opens_think_owes_only_the_on_cells() { + // Qwen3-style prefill that always opens (#3723): ON only — the OFF cells are not owed, so removing + // every OFF row leaves the release green, and removing an ON row does not + let t = green(); + for h in ["lambda", "gx10"] { + edit(&models(t.path(), h), |v| { + v["inventory"][0]["thinking_modes"] = + serde_json::from_str::(r#"["on"]"#).expect("modes"); + v["inventory"][0]["thinking_markers"] = + serde_json::from_str::(r#"[""]"#).expect("markers"); + let cells = v["cells"].as_array_mut().expect("cells"); + cells.retain(|c| c["thinking"] == "on"); + }); + } + let r = gate(t.path(), &[]); + assert_eq!(r.code, 0, "{}", show(&r)); + assert_eq!( + json_of(&r)["release"]["cells"], + 24, + "2 hosts × 4 verbs × ON × 3 rungs" + ); + edit(&models(t.path(), "gx10"), |v| { + let i = row_index(v, "code", "on", "4k"); + v["cells"].as_array_mut().expect("cells").remove(i); + }); + assert_red_naming(&gate(t.path(), &[]), &cell("gx10", "code", "on", "4k")); +} diff --git a/crates/aprender-contracts/src/lint/mod.rs b/crates/aprender-contracts/src/lint/mod.rs index db3b347077..88c8c70cad 100644 --- a/crates/aprender-contracts/src/lint/mod.rs +++ b/crates/aprender-contracts/src/lint/mod.rs @@ -244,6 +244,9 @@ pub enum GateExtra { /// ONT-4b2: Lean theorems extracted, and contract `lean_theorem:` references naming none of them. lean_statements: usize, lean_refs_unresolved: usize, + /// aprender#3715: what `extract:release-evidence` derived — absent unless a release subject was given. + #[serde(skip_serializing_if = "Option::is_none")] + release: Option>, }, } @@ -646,11 +649,22 @@ pub enum NamedGateOutcome { /// report a verdict whose precondition nobody checked — `UnknownGate` is the honest answer, not a silent pass. #[must_use] pub fn run_named_gate(contract_dir: &Path, name: &str) -> NamedGateOutcome { + run_named_gate_with(contract_dir, name, &shapes_gate::ShapesOptions::default()) +} + +/// [`run_named_gate`] with the shapes gate's `--shape` / `--release-*` options (aprender#3715). +pub fn run_named_gate_with( + contract_dir: &Path, + name: &str, + shapes_opts: &shapes_gate::ShapesOptions, +) -> NamedGateOutcome { match name { "relations" => { NamedGateOutcome::Relations(relations_gate::run_relations_gate(contract_dir)) } - "shapes" => NamedGateOutcome::Shapes(shapes_gate::run_shapes_gate(contract_dir)), + "shapes" => { + NamedGateOutcome::Shapes(shapes_gate::run_shapes_gate_with(contract_dir, shapes_opts)) + } "sigma" => NamedGateOutcome::Sigma(sigma_gate::run_sigma_gate(contract_dir)), "validate" => { let (contracts, parse_errors) = load_contracts(contract_dir); diff --git a/crates/aprender-contracts/src/lint/shapes_gate.rs b/crates/aprender-contracts/src/lint/shapes_gate.rs index 5739d10db9..9c772dfdf0 100644 --- a/crates/aprender-contracts/src/lint/shapes_gate.rs +++ b/crates/aprender-contracts/src/lint/shapes_gate.rs @@ -33,8 +33,10 @@ use std::path::Path; use std::time::Instant; use crate::ontology::arming::ArmedShapes; +use crate::ontology::extract::release_inputs::Subject; use crate::ontology::extract::{ - self, apr_model, code, gguf, json, lean, parity_receipt, pv_contract, ExtractFailure, + self, apr_model, code, gguf, json, lean, parity_receipt, pv_contract, release_evidence, + ExtractFailure, }; use crate::ontology::rdf::{iri, Graph, Term, RDF_TYPE}; use crate::ontology::receipts; @@ -196,9 +198,51 @@ fn w3c_checked(shapes_n: usize, focus_nodes_n: usize) -> Result`: grade only that shape FAMILY — the shape `` and every shape whose id starts with + /// `.` — and arm it whatever `armed_shapes` says: a caller that names a shape is asking for its verdict, + /// and "computed and reported" would let a release gate print violations and exit 0. + pub only: Option, + /// `--release-*`: the release subject `extract:release-evidence` reads. `None` → no release graph at all. + pub release: Option, +} + +/// The shapes `only` selects, in declaration order; a name that selects nothing is the CALLER's error (exit 3), +/// never an empty pass. +fn select_family( + declared: Vec<(NodeShape, String)>, + only: Option<&str>, +) -> Result, ShapeError> { + let Some(id) = only else { + return Ok(declared.into_iter().map(|(s, _)| s).collect()); + }; + let prefix = format!("{id}."); + let family: Vec = declared + .into_iter() + .map(|(s, _)| s) + .filter(|s| s.id == id || s.id.starts_with(&prefix)) + .collect(); + if family.iter().any(|s| s.id == id) { + Ok(family) + } else { + Err(ShapeError::Malformed { + shape: id.to_string(), + what: "--shape names no declared shape".into(), + }) + } +} + /// Run the gate over `contract_dir`. #[must_use] pub fn run_shapes_gate(contract_dir: &Path) -> ShapesOutcome { + run_shapes_gate_with(contract_dir, &ShapesOptions::default()) +} + +/// [`run_shapes_gate`] with `--shape` / `--release-*` (aprender#3715). +#[must_use] +pub fn run_shapes_gate_with(contract_dir: &Path, opts: &ShapesOptions) -> ShapesOutcome { let start = Instant::now(); let (declared, checked) = match collect_shapes(contract_dir) { Ok(x) => x, @@ -209,15 +253,19 @@ pub fn run_shapes_gate(contract_dir: &Path) -> ShapesOutcome { contracts_checked: checked, }; } - let arming = match armed_shapes_of(contract_dir) { - Ok(a) => a, + let arming = match (opts.only.as_deref(), armed_shapes_of(contract_dir)) { + (_, Err(e)) => return ShapesOutcome::Unsupported(e), + (Some(_), Ok(_)) => ArmedShapes::All, + (None, Ok(a)) => a, + }; + let shapes = match select_family(declared, opts.only.as_deref()) { + Ok(s) => s, Err(e) => return ShapesOutcome::Unsupported(e), }; - let shapes: Vec = declared.into_iter().map(|(s, _)| s).collect(); // ONE walk (R-18): every extractor, the json documents, the ladder receipts joined to the rungs — the same - // graph `pv extract` writes. - let extraction = match extract::all(contract_dir) { + // graph `pv extract` writes — plus, when a release subject is given, the release evidence (#3715). + let extraction = match extract::all_with(contract_dir, opts.release.as_ref()) { Ok(x) => x, Err(e) => return ShapesOutcome::ExtractFailed(e), }; @@ -233,6 +281,13 @@ pub fn run_shapes_gate(contract_dir: &Path) -> ShapesOutcome { }); let (mut report, plant_violations) = validate_with_plant(&graph, &shapes, &arming); + if opts.only.is_some() { + // a named family reports in the order its contract declares it: the release, its hosts, its context, + // its kernels, THEN its model cells (#3715: a kernel that diverges across arches is named before any + // model cell it breaks). Stable, so the validator's order holds within a shape. + let rank = |id: &str| shapes.iter().position(|s| s.id == id).unwrap_or(usize::MAX); + report.results.sort_by_key(|r| rank(&r.shape)); + } carry_extract_warnings(&mut report, &extraction.warnings); let pc_extract = extract_controls(); let unmeasured = needs_receipts && extraction.receipts.is_empty(); @@ -342,6 +397,7 @@ pub fn run_shapes_gate(contract_dir: &Path) -> ShapesOutcome { symbols_unresolved: extraction.code.unresolved, lean_statements: extraction.lean.statements, lean_refs_unresolved: extraction.lean.refs_unresolved.len(), + release: extraction.release.clone().map(Box::new), }), }; ShapesOutcome::Ran { @@ -399,6 +455,8 @@ fn extract_controls() -> BTreeMap { "parity-receipt", parity_receipt::positive_control(&parity_receipt::control_sample()), ), + // aprender#3715: drawn every run, subject or not — a cell owed without a receipt stays a node + ("release-evidence", release_evidence::positive_control()), ] .into_iter() .map(|(k, fired)| { diff --git a/crates/aprender-contracts/src/ontology/extract/mod.rs b/crates/aprender-contracts/src/ontology/extract/mod.rs index 3f2c54939c..da7e46f0bc 100644 --- a/crates/aprender-contracts/src/ontology/extract/mod.rs +++ b/crates/aprender-contracts/src/ontology/extract/mod.rs @@ -21,6 +21,8 @@ pub mod json; pub mod lean; pub mod parity_receipt; pub mod pv_contract; +pub mod release_evidence; +pub mod release_inputs; /// Every extractor's output over `contract_dir`, plus the input-side warnings the extractors chose to carry /// rather than hide (a torn JSONL line), plus what the model extractors and the receipt resolver counted. @@ -44,6 +46,8 @@ pub struct Extraction { pub lean: lean::LeanStats, /// ONT-4c3: the logit-parity receipts under `evidence/parity/**`, and the files this extractor refused. pub parity: parity_receipt::ParityStats, + /// aprender#3715: the release evidence — `None` unless a release subject was given (an ordinary PR has none). + pub release: Option, } /// What a walk could not do. Every variant is the DECLARATION's fault (exit 3), never a corpus verdict. @@ -53,6 +57,8 @@ pub enum ExtractFailure { Json(json::ExtractError), /// A ladder receipt under `evidence/dogfood/models/` is unreadable or carries a foreign schema. Receipt(receipts::ReceiptError), + /// aprender#3715: the release subject is malformed, or a release input is unreadable or foreign. + Release(release_inputs::ReleaseError), } impl std::fmt::Display for ExtractFailure { @@ -60,6 +66,7 @@ impl std::fmt::Display for ExtractFailure { match self { Self::Json(e) => write!(f, "{e}"), Self::Receipt(e) => write!(f, "receipt {e}"), + Self::Release(e) => write!(f, "{e}"), } } } @@ -70,6 +77,15 @@ impl std::error::Error for ExtractFailure {} /// contract dir's parent — the repo root, the same base `ont:file` uses); then the `gguf` and `apr-model` /// entities and the ladder receipts joined to the rungs. `Err` is a declaration's fault. pub fn all(contract_dir: &Path) -> Result { + all_with(contract_dir, None) +} + +/// [`all`], plus — when `release` names a release subject — `extract:release-evidence` (aprender#3715): the +/// release's receipts as the graph `release-readiness-v1` grades. Same one walk (R-18). +pub fn all_with( + contract_dir: &Path, + release: Option<&release_inputs::Subject>, +) -> Result { let mut out = Extraction { graph: pv_contract::extract(contract_dir), ..Extraction::default() @@ -91,6 +107,12 @@ pub fn all(contract_dir: &Path) -> Result { out.code = code::extract(contract_dir, &mut out.graph); out.lean = lean::extract(contract_dir, &mut out.graph); out.parity = parity_receipt::extract(root, &mut out.graph); + if let Some(subject) = release { + out.release = Some( + release_evidence::extract(&mut out.graph, contract_dir, subject) + .map_err(ExtractFailure::Release)?, + ); + } Ok(out) } diff --git a/crates/aprender-contracts/src/ontology/extract/release_evidence.rs b/crates/aprender-contracts/src/ontology/extract/release_evidence.rs new file mode 100644 index 0000000000..2db3277a29 --- /dev/null +++ b/crates/aprender-contracts/src/ontology/extract/release_evidence.rs @@ -0,0 +1,1119 @@ +//! ONT-001 §3.7 — `extract:release-evidence` (aprender#3715): the release's receipts become a graph in which +//! every obligation is a NODE, so absence is a missing edge that `minCount 1` rejects — never a row nobody read. +//! +//! Operator, 2026-09-21: *"you need to improve our dogfood process and pv SHACL so leaks don't exist pre-release."* +//! Every 0.69.0 leak was absence read as conformance (a rung marked optional, a GPU error read as a skip, an +//! inherited dogfood, models on a host that were not on the ladder). So the universe is DERIVED here, from what +//! was measured, and each (thing × host × verb × context) the release must prove gets a node of its own: +//! +//! | node | IRI | from | +//! |---|---|---| +//! | `release:Release` | `release-subject/` | the CLI's `--release-*` flags (never inferred) | +//! | `release:DogfoodReceipt` | `release-dogfood/` | `--dogfood-receipt`: the file R5 judged | +//! | `release:Host` | `release-host//` | every `ladder.hosts[]` entry with `required: true` | +//! | `release:HostReceipt` | `release-receipt///` | the host's `apr-model-ladder-receipt` (v2, #3712) | +//! | `release:Model` | `model/` (shared with `extract:gguf`) | the host's measured `inventory[]` ∪ the ladder's cuda rungs listed for it | +//! | `release:Verb` | `release-verb/` | run, chat, serve, code | +//! | `release:ContextRung` | `release-context/` | `evidence/release/context-rungs.json`; `consumer-max` is COMPUTED from the consumer records, and `declared` is each model's own GGUF `context_length` | +//! | `release:Cell` | `release-cell/////think-/` | model × verb × thinking × rung, per host — the focus of `release-readiness-v1` | +//! | `release:CellReceipt` | `release-row////` | each `cells[]` row that keys onto a cell | +//! | `release:RefusalCell` | as `release:Cell` | a cell that does not FIT the host by the declared memory arithmetic: owed as an honest pre-load refusal (#3710 rule) | +//! | `release:RungCoverage` | `release-coverage///` | every (model, rung) — owed on ≥ 1 required host | +//! | `release:KernelCell` | `release-kernel////` | every (kernel, quant) on any required host's dispatch path × every required host | +//! | `release:KernelDiffReceipt` | `release-krow////` | each kernel-diff row | +//! | `release:TokenizerCell` | `release-tokenizer//` | every distinct model in any required host's universe (#3726) | +//! | `release:TokenizerParityReceipt` | `release-trow///` | each tokenizer-parity row keyed onto a model | +//! +//! The cell universe per model (#3710, operator: "MORE than enough tokens … chat with and without thinking, ditto +//! run, ditto code"): every verb × thinking ∈ {on, off} (only `off` when the host MEASURED no thinking mode) × +//! every declared rung up to the model's own `context_length`, plus `declared` itself. A model whose length was +//! not measured owes every rung, and the `.model` shape names the missing length. +//! +//! Rust computes only what SHACL-Core cannot say (the universe, the join, `fresh`, `contextMet`, `thinkOk`, +//! `answered`, `withinBound`); +//! the shapes in `contracts/release-readiness-v1.yaml` say what a pass IS, and the fixtures under +//! `tests/fixtures/ont/release-*` falsify both. Nothing here runs unless a release subject is given: an ordinary +//! PR has no release, and inventing one would grade a tree against a commit it is not. + +use std::collections::{BTreeMap, BTreeSet}; +use std::path::Path; + +use crate::ontology::extract::gguf::{self, Rung}; +use crate::ontology::extract::pv_contract::{self, scalar}; +use crate::ontology::extract::release_inputs::{ + self as inputs, Consumer, ContextRung, Dogfood, KernelReceipt, ReleaseError, Subject, + TokReceipt, +}; +use crate::ontology::rdf::{iri, iri_path, Graph, Term, ONT_BASE, RDF_TYPE}; +use crate::ontology::receipts::{self, CellRow, Receipt}; + +/// The verbs every model must pass on every required host (#3710: "run, chat, serve and code"). +pub const VERBS: [&str; 4] = ["run", "chat", "serve", "code"]; +/// Both thinking modes (#3710 bar raise: "chat with and without thinking, ditto run, ditto code"). +pub const THINKING: [&str; 2] = ["on", "off"]; +/// The per-model rung: the model's own declared context length, read from its GGUF on the host. +pub use crate::ontology::extract::release_inputs::DECLARED; + +/// The vocabulary root: `https://ont.paiml.dev/v1alpha1/release/` (`release:` in a shape). +#[must_use] +pub fn rel(name: &str) -> String { + format!("{ONT_BASE}release/{name}") +} + +/// What one run derived, for the gate's report. +#[derive(Debug, Clone, Default, PartialEq, Eq, serde::Serialize)] +pub struct ReleaseStats { + pub required_hosts: usize, + pub model_receipts: usize, + pub kernel_receipts: usize, + /// Distinct (host, model) pairs in the universe. + pub models: usize, + pub cells: usize, + /// Cells with at least one keyed row (a row is not a pass — the shape decides that). + pub cells_with_row: usize, + pub kernel_cells: usize, + /// One per distinct model (sha256) in any required host's universe (#3726). + pub tokenizer_cells: usize, + /// Cells owed as an honest pre-load refusal because they do not fit the host (counted inside `cells`). + pub refusal_cells: usize, + /// (model, rung) pairs that must be owed on ≥ 1 required host (#3710 rule). + pub rung_coverage: usize, + pub context_rungs: usize, + /// Rows that keyed onto no cell (a model outside the universe, an unknown verb or rung): counted, not graded. + pub orphan_rows: usize, + /// Every cell, named `///think-/` — so a Pass reads "N/N cells", each named + /// (#3715 done_when 5), and a reader can see which cells a verdict covered, not only how many. + pub cell_names: Vec, +} + +/// A `ladder.hosts[]` entry with `required: true`. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct HostDecl { + pub id: String, + pub cc: String, +} + +/// `ladder.cells.long_rungs_for` (#3712 amendment 2): the families that owe every long rung, and one +/// representative file per other architecture. Absent from the ladder → every model owes the long rungs. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct LongRungsFor { + pub families: BTreeSet, + /// `general.architecture` → the one file basename that owes the long rungs for that arch. + pub representatives: BTreeMap, +} + +impl LongRungsFor { + fn owes(&self, arch: Option<&str>, file: &str) -> bool { + arch.is_some_and(|a| { + self.families.contains(a) || self.representatives.get(a).is_some_and(|f| f == file) + }) + } +} + +/// What the ladder contract(s) declare for the release. +#[derive(Debug, Clone, Default)] +pub struct Ladder { + pub hosts: Vec, + pub rungs: Vec, + pub long_rungs_for: Option, +} + +/// One model the release must prove on one host. +#[derive(Debug, Clone, PartialEq, Eq)] +struct ModelInfo { + file: String, + arch: Option, + quant: Option, + context_length: Option, + thinking_modes: Option>, + thinking_markers: Option>, + /// The producer's echo, compared with `owes_long`. + owes_long_echo: Option, + /// Derived here, from the ladder contract. + owes_long: bool, + /// (weights, KV bytes per token, workspace) — the declared memory arithmetic, when all three were measured. + mem: Option<(u64, u64, u64)>, + kv_dtype: Option, +} + +impl ModelInfo { + /// Does a `tokens`-long context fit on a host with `gpu` bytes (#3710 rule, operator "a")? `None` when any + /// term is unmeasured — and an unknown fit leaves the cell OWED, the stricter reading. + fn fits(&self, gpu: Option, tokens: Option) -> Option { + Some(self.required_bytes(tokens?)? <= gpu?) + } + + fn required_bytes(&self, tokens: u64) -> Option { + let (w, kv, ws) = self.mem?; + Some( + w.saturating_add(kv.saturating_mul(tokens)) + .saturating_add(ws), + ) + } + + /// `off` always; `on` unless the host MEASURED that the model has no thinking mode. + /// The modes the model owes: its measured `thinking_modes` (#3723); BOTH when unmeasured — and both when the + /// measured list names no valid mode, so a malformed list cannot make a model vanish (the `.model` shape names + /// it as a contradiction instead). + fn modes(&self) -> Vec<&'static str> { + let owed: Vec<&'static str> = match &self.thinking_modes { + Some(ms) => THINKING + .iter() + .copied() + .filter(|t| ms.iter().any(|m| m == t)) + .collect(), + None => THINKING.to_vec(), + }; + if owed.is_empty() { + THINKING.to_vec() + } else { + owed + } + } + + /// The rungs this model owes: every declared rung whose size fits its declared length (all of them while the + /// length is unmeasured, or the rung's own size is), then `declared`, whose target IS the length. + fn rungs<'r>(&self, rungs: &'r [ContextRung]) -> Vec<(&'r str, Option)> { + let fits = |r: &&ContextRung| match (r.tokens, self.context_length) { + (Some(t), Some(l)) => t <= l, + _ => true, + }; + let mut out: Vec<(&str, Option)> = rungs + .iter() + .filter(|r| self.owes_long || !r.long) + .filter(fits) + .map(|r| (r.id.as_str(), r.tokens)) + .collect(); + if self.owes_long { + out.push((DECLARED, self.context_length)); + } + out + } + + /// `thinking_modes` must be what its template evidence can prove (#3723's derivation): `enable_thinking` + /// among the markers ⇔ both modes; no marker at all ⇒ `["off"]`; a list naming no valid mode is itself a + /// contradiction. (`["on"]` — a prompt that always opens `` — is the producer's to prove; pv checks only + /// what the markers decide.) `None` when both are unmeasured or they agree. + fn thinking_contradiction(&self) -> Option { + let modes = self.thinking_modes.as_ref()?; + let valid: Vec<&String> = modes + .iter() + .filter(|m| THINKING.contains(&m.as_str())) + .collect(); + let both = THINKING.iter().all(|t| modes.iter().any(|m| m == t)); + let markers = self.thinking_markers.as_deref().unwrap_or_default(); + let switch = markers.iter().any(|m| m == "enable_thinking"); + let bad = valid.is_empty() + || valid.len() != modes.len() + || switch != both + || (self.thinking_markers.is_some() && markers.is_empty() && modes != &["off"]); + bad.then(|| { + format!( + "thinking_modes = [{}] but thinking_markers = [{}]", + modes.join(", "), + markers.join(", ") + ) + }) + } +} + +/// One required host, with everything derived for it. +struct HostView<'a> { + decl: &'a HostDecl, + node: String, + receipts: Vec<&'a Receipt>, + kernels: Vec<&'a KernelReceipt>, + /// sha256 → model, the host's universe. + models: BTreeMap, + /// The host's measured GPU memory (the fit arithmetic's right-hand side). + gpu_mem: Option, +} + +/// The required hosts, the rungs and `cells.long_rungs_for`, from every contract that carries a `ladder:` block. +#[must_use] +pub fn ladder_of(contract_dir: &Path) -> Ladder { + let mut out = Ladder::default(); + for (stem, _rel, doc) in pv_contract::documents(contract_dir) { + let Some(l) = doc.get("ladder") else { + continue; + }; + let decls = l.get("hosts").and_then(serde_yaml::Value::as_sequence); + for h in decls.into_iter().flatten() { + if h.get("required").and_then(serde_yaml::Value::as_bool) == Some(true) { + out.hosts.push(HostDecl { + id: scalar(h.get("id")).unwrap_or_default(), + cc: scalar(h.get("cc")).unwrap_or_default(), + }); + } + } + out.rungs.extend(gguf::rungs_of(&stem, &doc)); + if let Some(lr) = l.get("cells").and_then(|c| c.get("long_rungs_for")) { + out.long_rungs_for = Some(long_rungs_for(lr)); + } + } + out.hosts.sort_by(|a, b| a.id.cmp(&b.id)); + out.hosts.dedup_by(|a, b| a.id == b.id); + out +} + +fn long_rungs_for(v: &serde_yaml::Value) -> LongRungsFor { + let families = v + .get("families") + .and_then(serde_yaml::Value::as_sequence) + .into_iter() + .flatten() + .filter_map(|f| scalar(Some(f))) + .collect(); + let representatives = v + .get("representatives") + .and_then(serde_yaml::Value::as_mapping) + .into_iter() + .flatten() + .filter_map(|(k, f)| Some((scalar(Some(k))?, scalar(Some(f))?))) + .collect(); + LongRungsFor { + families, + representatives, + } +} + +/// Read the release's evidence and write its graph into `g`. `Err` is a declaration's fault (exit 3). +pub fn extract( + g: &mut Graph, + contract_dir: &Path, + subject: &Subject, +) -> Result { + let root = super::repo_root(contract_dir); + let ladder = ladder_of(contract_dir); + let (ctx_rungs, consumers) = inputs::read_context_rungs(&root)?.unwrap_or_default(); + let model_receipts = + receipts::read_dir(&subject.model_dir(&root), &root).map_err(ReleaseError::Receipt)?; + let kernel_receipts = inputs::read_kernel_receipts(&subject.kernel_dir(&root), &root)?; + let tok_receipts = inputs::read_tokenizer_receipts(&subject.tokenizer_dir(&root), &root)?; + let dogfood = subject + .dogfood_receipt + .as_deref() + .map(inputs::read_dogfood) + .transpose()?; + Ok(build( + g, + subject, + &Inputs { + ladder: &ladder, + ctx_rungs: &ctx_rungs, + consumers: &consumers, + model_receipts: &model_receipts, + kernel_receipts: &kernel_receipts, + tok_receipts: &tok_receipts, + dogfood: dogfood.as_ref(), + }, + )) +} + +/// Everything [`build`] reads, already parsed — so the positive control drives the same code in memory. +pub struct Inputs<'a> { + pub ladder: &'a Ladder, + pub ctx_rungs: &'a [ContextRung], + pub consumers: &'a [Consumer], + pub model_receipts: &'a [Receipt], + pub kernel_receipts: &'a [KernelReceipt], + pub tok_receipts: &'a [TokReceipt], + pub dogfood: Option<&'a Dogfood>, +} + +/// The release graph from parsed inputs: pure, no filesystem (R-15). +pub fn build(g: &mut Graph, subject: &Subject, i: &Inputs<'_>) -> ReleaseStats { + let hosts = &i.ladder.hosts; + let mut stats = ReleaseStats { + required_hosts: hosts.len(), + model_receipts: i.model_receipts.len(), + kernel_receipts: i.kernel_receipts.len(), + context_rungs: i.ctx_rungs.len(), + ..ReleaseStats::default() + }; + emit_release(g, subject, hosts, i.ctx_rungs, i.dogfood); + emit_vocabulary_nodes(g, i.ctx_rungs, i.consumers); + let views: Vec> = hosts + .iter() + .map(|h| host_view(h, subject, i.model_receipts, i.kernel_receipts, i.ladder)) + .collect(); + let mut coverage = Coverage::new(); + for v in &views { + emit_host(g, subject, v, &mut stats); + emit_cells(g, subject, v, i.ctx_rungs, &mut coverage, &mut stats); + } + emit_coverage(g, subject, &coverage, &mut stats); + emit_kernels(g, subject, &views, &mut stats); + emit_tokenizer(g, subject, &views, i.tok_receipts, &mut stats); + stats +} + +/// The positive control (R-3, PMAT-3704): drawn on EVERY gate run, with or without a release subject. One +/// required host holds one model that owes one cell; the SAMPLE receipt carries that cell's fresh Pass row, the +/// PLANTED one omits it. Fires iff the sample's cell has exactly one fresh row AND the planted cell still exists +/// with ZERO rows — i.e. the extractor still materializes absence as a node for `minCount 1` to reject, which is +/// the whole reason this shape exists. An extractor that emitted only cells it had rows for would turn absence +/// back into silence, and this control would go `not-fired`. +#[must_use] +pub fn positive_control() -> bool { + let Ok(subject) = Subject::new("0.0.0-pc", PC_COMMIT) else { + return false; + }; + let (Ok(sample), Ok(planted)) = ( + receipts::parse("__pc_sample__.json", &pc_receipt(PC_ROW)), + receipts::parse("__pc_planted__.json", &pc_receipt("")), + ) else { + return false; + }; + let cell = iri_path( + "release-cell", + &["0.0.0-pc", "pc-host", "pc.gguf", "run", "think-off", "pc"], + ); + let rows_of = |r: &Receipt| { + let mut g = Graph::new(); + let ladder = Ladder { + hosts: vec![HostDecl { + id: "pc-host".into(), + cc: "sm_0".into(), + }], + ..Ladder::default() + }; + let rung = ContextRung { + id: "pc".into(), + tokens: Some(1), + long: false, + derived_from: vec!["pc".into()], + unmeasured_consumers: Vec::new(), + }; + build( + &mut g, + &subject, + &Inputs { + ladder: &ladder, + ctx_rungs: std::slice::from_ref(&rung), + consumers: &[], + model_receipts: std::slice::from_ref(r), + kernel_receipts: &[], + tok_receipts: &[], + dogfood: None, + }, + ); + let is_cell = g + .objects(&cell, RDF_TYPE) + .iter() + .any(|t| t.as_iri() == Some(rel("Cell").as_str())); + let fresh: Vec = g + .objects(&cell, &rel("row")) + .iter() + .filter_map(|t| t.as_iri()) + .map(|row| { + g.objects(row, &rel("fresh")) + .iter() + .any(|t| t.as_literal().map(|(v, _)| v) == Some("true")) + }) + .collect(); + (is_cell, fresh) + }; + let (sample_cell, sample_rows) = rows_of(&sample); + let (planted_cell, planted_rows) = rows_of(&planted); + sample_cell && sample_rows == [true] && planted_cell && planted_rows.is_empty() +} + +const PC_COMMIT: &str = "1111111111111111111111111111111111111111"; +const PC_ROW: &str = r#"{"sha256":"cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc","file":"pc.gguf","verb":"run","thinking":"off","context":"pc","prompt_tokens":2,"max_tokens":8,"answer_chars":1,"verdict":"pass","backend":"cuda","fallback":false,"rc":0}"#; + +fn pc_receipt(cells: &str) -> String { + format!( + r#"{{"schema":"apr-model-ladder-receipt/v2","host":"pc-host","version":"0.0.0-pc","sha":"1","apr_sha":"{PC_COMMIT}","inventory":[{{"file":"pc.gguf","sha256":"cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc","arch":"pc","context_length":10,"thinking_modes":["off"]}}],"cells":[{cells}],"rungs":[]}}"# + ) +} + +fn emit_release( + g: &mut Graph, + subject: &Subject, + hosts: &[HostDecl], + rungs: &[ContextRung], + dogfood: Option<&Dogfood>, +) { + let v = subject.version.as_str(); + let n = iri_path("release-subject", &[v]); + g.insert(n.clone(), RDF_TYPE, Term::iri(rel("Release"))); + g.insert(n.clone(), rel("version"), Term::string(v)); + g.insert(n.clone(), rel("commit"), Term::string(&subject.commit)); + g.insert( + n.clone(), + rel("measuredCommit"), + Term::string(subject.measured_commit()), + ); + for h in hosts { + g.insert( + n.clone(), + rel("requiredHost"), + Term::iri(iri_path("release-host", &[v, &h.id])), + ); + } + for r in rungs { + g.insert( + n.clone(), + rel("contextRung"), + Term::iri(iri("release-context", &r.id)), + ); + } + if let Some(d) = dogfood { + let dn = iri_path("release-dogfood", &[v]); + g.insert(dn.clone(), RDF_TYPE, Term::iri(rel("DogfoodReceipt"))); + g.insert(dn.clone(), rel("file"), Term::string(&d.file)); + g.insert(dn.clone(), rel("verdict"), Term::string(&d.verdict)); + g.insert(dn.clone(), rel("commit"), Term::string(&d.commit)); + g.insert(dn.clone(), rel("version"), Term::string(&d.version)); + let fresh = d.commit == subject.commit && d.version == subject.version; + g.insert(dn.clone(), rel("fresh"), Term::boolean(fresh)); + g.insert(n, rel("dogfoodReceipt"), Term::iri(dn)); + } +} + +/// The verbs, the context rungs and the consumers they were derived from. +fn emit_vocabulary_nodes(g: &mut Graph, rungs: &[ContextRung], consumers: &[Consumer]) { + for verb in VERBS { + let n = iri("release-verb", verb); + g.insert(n.clone(), RDF_TYPE, Term::iri(rel("Verb"))); + g.insert(n, rel("name"), Term::string(verb)); + } + // `declared` has no one size — each model's own length is its target — so it is not a `release:ContextRung` + // (whose shape requires a `tokens`); the model's `contextLength` carries it, and `.model` requires that. + let declared = iri("release-context", DECLARED); + g.insert(declared.clone(), RDF_TYPE, Term::iri(rel("DeclaredRung"))); + g.insert(declared.clone(), rel("rungId"), Term::string(DECLARED)); + g.insert( + declared, + rel("derivedFrom"), + Term::string("each model's own GGUF context_length, measured on the host (#3710)"), + ); + for r in rungs { + let n = iri("release-context", &r.id); + g.insert(n.clone(), RDF_TYPE, Term::iri(rel("ContextRung"))); + g.insert(n.clone(), rel("rungId"), Term::string(&r.id)); + if let Some(t) = r.tokens { + g.insert(n.clone(), rel("tokens"), Term::integer(t)); + } + for d in &r.derived_from { + g.insert(n.clone(), rel("derivedFrom"), Term::string(d)); + } + for c in &r.unmeasured_consumers { + g.insert(n.clone(), rel("unmeasuredConsumer"), Term::string(c)); + } + } + for c in consumers { + let n = iri("release-consumer", &c.name); + g.insert(n.clone(), RDF_TYPE, Term::iri(rel("Consumer"))); + if let Some(t) = c.max_prompt_tokens { + g.insert(n.clone(), rel("maxPromptTokens"), Term::integer(t)); + } + g.insert(n.clone(), rel("basis"), Term::string(&c.basis)); + g.insert(n, rel("source"), Term::string(&c.source)); + } +} + +/// The host's receipts and its universe: every inventory model with a measured hash, plus every ladder rung that +/// claims cuda and lists this host (or lists none). Keyed by sha256, so a copy under a second name is one model. +fn host_view<'a>( + decl: &'a HostDecl, + subject: &Subject, + model_receipts: &'a [Receipt], + kernel_receipts: &'a [KernelReceipt], + ladder: &Ladder, +) -> HostView<'a> { + let receipts: Vec<&Receipt> = model_receipts + .iter() + .filter(|r| r.host == decl.id) + .collect(); + let kernels: Vec<&KernelReceipt> = kernel_receipts + .iter() + .filter(|r| r.host == decl.id) + .collect(); + let mut models: BTreeMap = BTreeMap::new(); + for item in receipts.iter().flat_map(|r| &r.inventory) { + if let Some(sha) = &item.sha256 { + models.entry(sha.clone()).or_insert_with(|| ModelInfo { + file: item.file.clone(), + arch: item.arch.clone(), + quant: item.quant.clone(), + context_length: item.context_length, + thinking_modes: item.thinking_modes.clone(), + thinking_markers: item.thinking_markers.clone(), + owes_long_echo: item.owes_long_rungs, + owes_long: true, + mem: item + .weights_bytes + .zip(item.kv_bytes_per_token) + .zip(item.workspace_bytes) + .map(|((w, kv), ws)| (w, kv, ws)), + kv_dtype: item.kv_dtype.clone(), + }); + } + } + let listed = |r: &&Rung| r.hosts.is_empty() || r.hosts.iter().any(|h| h == &decl.id); + for r in ladder + .rungs + .iter() + .filter(listed) + .filter(|r| r.backends.iter().any(|b| b == "cuda")) + { + models + .entry(r.sha256.to_ascii_lowercase()) + .or_insert_with(|| ModelInfo { + file: r.gguf.clone(), + arch: Some(r.arch.clone()), + quant: None, + context_length: None, + thinking_modes: None, + thinking_markers: None, + owes_long_echo: None, + owes_long: true, + mem: None, + kv_dtype: None, + }); + } + if let Some(lr) = &ladder.long_rungs_for { + for m in models.values_mut() { + m.owes_long = lr.owes(m.arch.as_deref(), &m.file); + } + } + let gpu_mem = receipts.iter().find_map(|r| r.gpu_mem_total_bytes); + HostView { + decl, + node: iri_path("release-host", &[&subject.version, &decl.id]), + receipts, + kernels, + models, + gpu_mem, + } +} + +/// Is this receipt of THIS release — its version, and the sha its `apr` was built from? +fn fresh(version: &str, apr_sha: Option<&str>, subject: &Subject) -> bool { + version == subject.version && apr_sha == Some(subject.measured_commit()) +} + +fn emit_host(g: &mut Graph, subject: &Subject, v: &HostView<'_>, stats: &mut ReleaseStats) { + let h = &v.node; + g.insert(h.clone(), RDF_TYPE, Term::iri(rel("Host"))); + g.insert(h.clone(), rel("hostId"), Term::string(&v.decl.id)); + g.insert(h.clone(), rel("cc"), Term::string(&v.decl.cc)); + for r in &v.receipts { + let n = iri_path("release-receipt", &[&subject.version, &v.decl.id, &r.file]); + g.insert(n.clone(), RDF_TYPE, Term::iri(rel("HostReceipt"))); + g.insert(n.clone(), rel("file"), Term::string(&r.file)); + g.insert(n.clone(), rel("schema"), Term::string(&r.schema)); + g.insert(n.clone(), rel("version"), Term::string(&r.version)); + if let Some(s) = &r.apr_sha { + g.insert(n.clone(), rel("aprSha"), Term::string(s)); + } + let ok = fresh(&r.version, r.apr_sha.as_deref(), subject); + g.insert(n.clone(), rel("fresh"), Term::boolean(ok)); + g.insert(h.clone(), rel("hostReceipt"), Term::iri(n)); + for item in r.inventory.iter().filter(|i| i.sha256.is_none()) { + g.insert( + h.clone(), + rel("unmeasuredModel"), + Term::string(format!("{}: {}", r.file, item.file)), + ); + } + for c in r.cells.iter().filter(|c| c.sha256.is_none()) { + g.insert( + h.clone(), + rel("unkeyedRow"), + Term::string(format!( + "{}: {} {} think-{} {} verdict={}", + r.file, c.file, c.verb, c.thinking, c.context, c.verdict + )), + ); + } + } + for (sha, m) in &v.models { + stats.models += 1; + let n = iri("model", sha); + g.insert(n.clone(), RDF_TYPE, Term::iri(rel("Model"))); + g.insert(n.clone(), rel("sha256"), Term::string(sha)); + g.insert(n.clone(), rel("file"), Term::string(&m.file)); + if let Some(a) = &m.arch { + g.insert(n.clone(), rel("arch"), Term::string(a)); + } + if let Some(q) = &m.quant { + g.insert(n.clone(), rel("quant"), Term::string(q)); + } + if let Some(l) = m.context_length { + g.insert(n.clone(), rel("contextLength"), Term::integer(l)); + } + emit_model_evidence(g, &n, m); + g.insert(h.clone(), rel("holds"), Term::iri(n)); + } +} + +/// A model's thinking evidence and long-rung obligation, and every disagreement between what the producer +/// claimed and what pv derives (#3712 amendments 1, 2). +fn emit_model_evidence(g: &mut Graph, n: &str, m: &ModelInfo) { + let n = n.to_string(); + for t in m.thinking_modes.iter().flatten() { + g.insert(n.clone(), rel("thinkingMode"), Term::string(t)); + } + for mk in m.thinking_markers.iter().flatten() { + g.insert(n.clone(), rel("thinkingMarker"), Term::string(mk)); + } + if let Some(c) = m.thinking_contradiction() { + g.insert(n.clone(), rel("thinkingContradiction"), Term::string(c)); + } + g.insert(n.clone(), rel("owesLongRungs"), Term::boolean(m.owes_long)); + if let Some(k) = &m.kv_dtype { + g.insert(n.clone(), rel("kvDtype"), Term::string(k)); + } + if let Some(echo) = m.owes_long_echo.filter(|e| *e != m.owes_long) { + g.insert( + n, + rel("longRungsMismatch"), + Term::string(format!( + "{}: receipt says owes_long_rungs={echo}, the ladder contract derives {}", + m.file, m.owes_long + )), + ); + } +} + +/// One cell's key: (verb, thinking mode, rung id, the rung's target size for this model). +type CellKey<'r> = (&'static str, &'static str, &'r str, Option); + +/// One `release:Cell` per (model, verb, thinking, rung) on this host, and an edge to every row that keys onto it. +/// (model sha256, model file, rung id) → the hosts on which that rung is OWED (it fits, or its fit is unknown). +type Coverage = BTreeMap<(String, String, String), BTreeSet>; + +fn emit_cells( + g: &mut Graph, + subject: &Subject, + v: &HostView<'_>, + rungs: &[ContextRung], + coverage: &mut Coverage, + stats: &mut ReleaseStats, +) { + let mut keyed: BTreeSet<(String, usize)> = BTreeSet::new(); + for (sha, m) in &v.models { + let owed = m.rungs(rungs); + for (id, t) in &owed { + let hosts = coverage + .entry((sha.clone(), m.file.clone(), (*id).to_string())) + .or_default(); + if m.fits(v.gpu_mem, *t) != Some(false) { + hosts.insert(v.node.clone()); + } + } + let modes = m.modes(); + let keys: Vec> = VERBS + .iter() + .flat_map(|verb| modes.iter().map(move |mode| (*verb, *mode))) + .flat_map(|(verb, mode)| owed.iter().map(move |(id, t)| (verb, mode, *id, *t))) + .collect(); + for key in keys { + let cell = emit_cell_node(g, subject, v, sha, m, key); + stats.cells += 1; + if m.fits(v.gpu_mem, key.3) == Some(false) { + stats.refusal_cells += 1; + } + let (verb, mode, rung, _) = key; + stats.cell_names.push(format!( + "{}/{}/{verb}/think-{mode}/{rung}", + v.decl.id, m.file + )); + let rows = rows_for(g, subject, v, sha, key, &mut keyed); + if !rows.is_empty() { + stats.cells_with_row += 1; + } + for r in rows { + g.insert(cell.clone(), rel("row"), Term::iri(r)); + } + } + } + let all: usize = v.receipts.iter().map(|r| r.cells.len()).sum(); + stats.orphan_rows += all - keyed.len(); +} + +fn emit_cell_node( + g: &mut Graph, + subject: &Subject, + v: &HostView<'_>, + sha: &str, + m: &ModelInfo, + (verb, mode, rung, target): CellKey<'_>, +) -> String { + let think = format!("think-{mode}"); + let cell = iri_path( + "release-cell", + &[&subject.version, &v.decl.id, &m.file, verb, &think, rung], + ); + // a cell that does not FIT this host by the declared arithmetic is owed as an honest pre-load refusal + // instead (#3710 rule): a different class, a different shape — never an OOM mid-run + let class = if m.fits(v.gpu_mem, target) == Some(false) { + "RefusalCell" + } else { + "Cell" + }; + g.insert(cell.clone(), RDF_TYPE, Term::iri(rel(class))); + if let Some(req) = target.and_then(|t| m.required_bytes(t)) { + g.insert(cell.clone(), rel("requiredBytes"), Term::integer(req)); + } + g.insert(cell.clone(), rel("host"), Term::iri(v.node.clone())); + g.insert(cell.clone(), rel("model"), Term::iri(iri("model", sha))); + g.insert( + cell.clone(), + rel("verb"), + Term::iri(iri("release-verb", verb)), + ); + g.insert(cell.clone(), rel("thinking"), Term::string(mode)); + g.insert( + cell.clone(), + rel("context"), + Term::iri(iri("release-context", rung)), + ); + cell +} + +/// Emit every row of this host's receipts that keys onto (sha, verb, thinking, rung); return their IRIs. +fn rows_for( + g: &mut Graph, + subject: &Subject, + v: &HostView<'_>, + sha: &str, + key: CellKey<'_>, + keyed: &mut BTreeSet<(String, usize)>, +) -> Vec { + let (verb, mode, rung, _) = key; + let mut out = Vec::new(); + for r in &v.receipts { + for (i, c) in r.cells.iter().enumerate() { + let hit = c.sha256.as_deref() == Some(sha) + && c.verb == verb + && c.thinking == mode + && c.context == rung; + if hit { + keyed.insert((r.file.clone(), i)); + out.push(emit_row(g, subject, r, i, c, key, v.gpu_mem)); + } + } + } + out +} + +/// Did the row fill its rung? A fixed rung: the prompt alone reaches it. `declared`: prompt plus the output +/// budget fill the model's own length. An unknown on either side is `false` — an unmeasured size met nothing. +fn context_met(c: &CellRow, (_, _, rung, target): CellKey<'_>) -> bool { + let Some(t) = target else { + return false; + }; + match (rung == DECLARED, c.prompt_tokens, c.max_tokens) { + (false, Some(p), _) => p >= t, + (true, Some(p), Some(o)) => p.saturating_add(o) >= t, + _ => false, + } +} + +fn emit_row( + g: &mut Graph, + subject: &Subject, + r: &Receipt, + i: usize, + c: &CellRow, + key: CellKey<'_>, + gpu_total: Option, +) -> String { + let n = iri_path( + "release-row", + &[&subject.version, &r.host, &r.file, &i.to_string()], + ); + g.insert(n.clone(), RDF_TYPE, Term::iri(rel("CellReceipt"))); + g.insert( + n.clone(), + rel("verdict"), + Term::string(c.verdict.to_ascii_lowercase()), + ); + g.insert(n.clone(), rel("backend"), Term::string(&c.backend)); + if let Some(fb) = c.fallback { + g.insert(n.clone(), rel("fallback"), Term::boolean(fb)); + } + let ok = fresh(&r.version, r.apr_sha.as_deref(), subject); + g.insert(n.clone(), rel("fresh"), Term::boolean(ok)); + g.insert( + n.clone(), + rel("contextMet"), + Term::boolean(context_met(c, key)), + ); + // thinking ON must CLOSE its block (#3720: an unclosed block is reported, never an empty answer) + let think_ok = c.thinking == "off" || c.think_closed == Some(true); + g.insert(n.clone(), rel("thinkOk"), Term::boolean(think_ok)); + let answered = c.answer_chars.is_some_and(|a| a > 0); + // a refusal NAMES its arithmetic: what it needs is more than what the host has + // honest only against the TOTAL: a refusal because a co-tenant holds memory (free < required ≤ total) is + // not the arithmetic, it is a busy GPU silently shrinking what the release proves (#3712, 62) + let named = matches!((c.required_bytes, gpu_total), (Some(r), Some(t)) if r > t); + g.insert(n.clone(), rel("arithmeticNamed"), Term::boolean(named)); + g.insert(n.clone(), rel("answered"), Term::boolean(answered)); + emit_row_detail(g, &n, r, c); + n +} + +/// What the row measured, carried for the reader: never graded here. +fn emit_row_detail(g: &mut Graph, n: &str, r: &Receipt, c: &CellRow) { + let n = n.to_string(); + if let Some(p) = c.prompt_tokens { + g.insert(n.clone(), rel("promptTokens"), Term::integer(p)); + } + if let Some(o) = c.max_tokens { + g.insert(n.clone(), rel("maxTokens"), Term::integer(o)); + } + if let Some(t) = c.ttft_ms { + g.insert(n.clone(), rel("ttftMs"), Term::double(t)); + } + if let Some(rc) = c.rc { + g.insert(n.clone(), rel("rc"), Term::signed(rc)); + } + if !c.reason.is_empty() { + g.insert(n.clone(), rel("reason"), Term::string(&c.reason)); + } + if let Some(s) = &r.apr_sha { + g.insert(n.clone(), rel("aprSha"), Term::string(s)); + } + g.insert(n, rel("receiptFile"), Term::string(&r.file)); +} + +/// Every rung a model declares must be owed — and so Pass — on at least one required host (#3710 rule): the +/// 27B's 262,144 cells refuse on the 24 GB 4090 and are owed on gx10. A rung that fits nowhere is named here. +fn emit_coverage(g: &mut Graph, subject: &Subject, coverage: &Coverage, stats: &mut ReleaseStats) { + for ((sha, file, rung), hosts) in coverage { + stats.rung_coverage += 1; + let n = iri_path("release-coverage", &[&subject.version, file, rung]); + g.insert(n.clone(), RDF_TYPE, Term::iri(rel("RungCoverage"))); + g.insert(n.clone(), rel("model"), Term::iri(iri("model", sha))); + g.insert( + n.clone(), + rel("context"), + Term::iri(iri("release-context", rung)), + ); + for h in hosts { + g.insert(n.clone(), rel("owedOn"), Term::iri(h.clone())); + } + } +} + +/// The kernel differential (#3715 addendum): the universe K is every kernel on ANY required host's dispatch +/// path, and each (kernel, host) — each GPU arch — is a `release:KernelCell`. A kernel measured on one arch only +/// is therefore a missing row on the other, and one that passes on sm_121 and fails on sm_89 is a failing row on +/// the sm_89 node: the class that made qwen2.5-coder red on lambda alone. +fn emit_kernels( + g: &mut Graph, + subject: &Subject, + views: &[HostView<'_>], + stats: &mut ReleaseStats, +) { + let universe: BTreeSet<(&str, &str)> = views + .iter() + .flat_map(|v| &v.kernels) + .flat_map(|r| &r.dispatch) + .flat_map(|d| &d.kernels) + .map(|(k, q)| (k.as_str(), q.as_str())) + .collect(); + for v in views { + emit_kernel_host(g, subject, v); + for key in &universe { + stats.kernel_cells += 1; + emit_kernel_cell(g, subject, v, *key); + } + } +} + +/// The host's kernel receipts, its dispatch kernels, and every held model the dispatch lists do not cover. +fn emit_kernel_host(g: &mut Graph, subject: &Subject, v: &HostView<'_>) { + let h = &v.node; + let mut covered: BTreeSet<&str> = BTreeSet::new(); + for r in &v.kernels { + let n = iri_path("release-kreceipt", &[&subject.version, &v.decl.id, &r.file]); + g.insert(n.clone(), RDF_TYPE, Term::iri(rel("KernelReceiptFile"))); + g.insert(n.clone(), rel("file"), Term::string(&r.file)); + g.insert(n.clone(), rel("sm"), Term::string(&r.sm)); + let ok = fresh(&r.version, r.apr_sha.as_deref(), subject); + g.insert(n.clone(), rel("fresh"), Term::boolean(ok)); + g.insert(h.clone(), rel("kernelReceipt"), Term::iri(n)); + for d in &r.dispatch { + if let Some(s) = &d.sha256 { + covered.insert(s); + } + for (k, q) in &d.kernels { + g.insert( + h.clone(), + rel("dispatchKernel"), + Term::string(kernel_label(k, q)), + ); + } + } + } + for (sha, m) in &v.models { + if !covered.contains(sha.as_str()) { + g.insert( + h.clone(), + rel("modelWithoutDispatch"), + Term::string(&m.file), + ); + } + } +} + +/// `q4k_gemv@q4_k`, or the bare kernel when it carries no quant (an f32 elementwise kernel). +fn kernel_label(kernel: &str, quant: &str) -> String { + if quant.is_empty() { + kernel.to_string() + } else { + format!("{kernel}@{quant}") + } +} + +fn emit_kernel_cell( + g: &mut Graph, + subject: &Subject, + v: &HostView<'_>, + (kernel, quant): (&str, &str), +) { + // kernel and quant as their own segments: `release-kernel////`, `-` for none + let q = if quant.is_empty() { "-" } else { quant }; + let cell = iri_path("release-kernel", &[&subject.version, &v.decl.id, kernel, q]); + g.insert(cell.clone(), RDF_TYPE, Term::iri(rel("KernelCell"))); + g.insert(cell.clone(), rel("kernel"), Term::string(kernel)); + g.insert(cell.clone(), rel("quant"), Term::string(quant)); + g.insert(cell.clone(), rel("host"), Term::iri(v.node.clone())); + g.insert(cell.clone(), rel("cc"), Term::string(&v.decl.cc)); + for r in &v.kernels { + let rows = r.rows.iter().enumerate(); + for (i, row) in rows.filter(|(_, x)| x.kernel == kernel && x.quant == quant) { + let n = emit_krow(g, subject, r, i, row); + g.insert(cell.clone(), rel("row"), Term::iri(n)); + } + } +} + +fn emit_krow( + g: &mut Graph, + subject: &Subject, + r: &KernelReceipt, + i: usize, + row: &inputs::KernelRow, +) -> String { + let n = iri_path( + "release-krow", + &[&subject.version, &r.host, &r.file, &i.to_string()], + ); + g.insert(n.clone(), RDF_TYPE, Term::iri(rel("KernelDiffReceipt"))); + g.insert(n.clone(), rel("verdict"), Term::string(&row.verdict)); + g.insert(n.clone(), rel("reference"), Term::string(&row.reference)); + if let Some(e) = row.max_err { + g.insert(n.clone(), rel("maxErr"), Term::double(e)); + } + if let Some(b) = row.bound { + g.insert(n.clone(), rel("bound"), Term::double(b)); + } + if let Some(m) = row.index_mismatch { + g.insert(n.clone(), rel("indexMismatch"), Term::integer(m)); + } + if let Some(t) = row.tie_margin { + g.insert(n.clone(), rel("tieMargin"), Term::double(t)); + } + g.insert( + n.clone(), + rel("withinBound"), + Term::boolean(row.within_bound()), + ); + if !row.bound_source.is_empty() { + g.insert( + n.clone(), + rel("boundSource"), + Term::string(&row.bound_source), + ); + } + let ok = fresh(&r.version, r.apr_sha.as_deref(), subject); + g.insert(n.clone(), rel("fresh"), Term::boolean(ok)); + if !row.reason.is_empty() { + g.insert(n.clone(), rel("reason"), Term::string(&row.reason)); + } + n +} + +/// Tokenizer parity (aprender#3726): apr's token ids against the pinned llama.cpp, one cell per distinct model +/// in any required host's universe — host-independent, because the tokenizer is. It is the cell that sees what +/// every apr-vs-apr gate cannot: CPU and GPU share the tokenizer, so `apr parity` passes a tokenizer that maps +/// every non-ASCII byte to id 0. +fn emit_tokenizer( + g: &mut Graph, + subject: &Subject, + views: &[HostView<'_>], + receipts: &[TokReceipt], + stats: &mut ReleaseStats, +) { + let models: BTreeMap<&str, &ModelInfo> = views + .iter() + .flat_map(|v| v.models.iter().map(|(sha, m)| (sha.as_str(), m))) + .collect(); + for (sha, m) in models { + stats.tokenizer_cells += 1; + let cell = iri_path("release-tokenizer", &[&subject.version, &m.file]); + g.insert(cell.clone(), RDF_TYPE, Term::iri(rel("TokenizerCell"))); + g.insert(cell.clone(), rel("model"), Term::iri(iri("model", sha))); + if let Some(a) = &m.arch { + g.insert(cell.clone(), rel("family"), Term::string(a)); + } + for r in receipts { + let rows = r.rows.iter().enumerate(); + for (i, row) in rows.filter(|(_, x)| x.sha256.as_deref() == Some(sha)) { + let n = emit_trow(g, subject, r, i, row); + g.insert(cell.clone(), rel("row"), Term::iri(n)); + } + } + } +} + +fn emit_trow( + g: &mut Graph, + subject: &Subject, + r: &TokReceipt, + i: usize, + row: &inputs::TokRow, +) -> String { + let n = iri_path("release-trow", &[&subject.version, &r.file, &i.to_string()]); + g.insert( + n.clone(), + RDF_TYPE, + Term::iri(rel("TokenizerParityReceipt")), + ); + g.insert(n.clone(), rel("verdict"), Term::string(&row.verdict)); + g.insert(n.clone(), rel("parityExact"), Term::boolean(row.exact())); + g.insert( + n.clone(), + rel("roundTrip"), + Term::boolean(row.roundtrip_ok == Some(true)), + ); + let opt = |g: &mut Graph, p: &str, v: &str| { + if !v.is_empty() { + g.insert(n.clone(), rel(p), Term::string(v)); + } + }; + opt(g, "corpusSha", &row.corpus_sha256); + opt(g, "comparator", &row.comparator); + opt(g, "comparatorSha", &row.comparator_sha); + opt(g, "reason", &row.reason); + if let Some(m) = row.mismatches { + g.insert(n.clone(), rel("mismatches"), Term::integer(m)); + } + if let Some(t) = row.tokens_compared { + g.insert(n.clone(), rel("tokensCompared"), Term::integer(t)); + } + let ok = fresh(&r.version, r.apr_sha.as_deref(), subject); + g.insert(n.clone(), rel("fresh"), Term::boolean(ok)); + n +} + +#[cfg(test)] +#[path = "release_evidence_tests.rs"] +mod tests; diff --git a/crates/aprender-contracts/src/ontology/extract/release_evidence_tests.rs b/crates/aprender-contracts/src/ontology/extract/release_evidence_tests.rs new file mode 100644 index 0000000000..e2fcce47cf --- /dev/null +++ b/crates/aprender-contracts/src/ontology/extract/release_evidence_tests.rs @@ -0,0 +1,458 @@ +//! aprender#3715 — `extract:release-evidence` in isolation: the universe, the join, and what is computed. The +//! shapes' verdicts over the same graphs are `tests/fixtures/ont/release-*` on the CLI +//! (`crates/aprender-contracts-cli/tests/ont_release_readiness.rs`). + +use super::*; +use crate::ontology::extract::release_inputs::{derive_rungs, Consumer}; + +const MC: &str = "1111111111111111111111111111111111111111"; +const BUMP: &str = "2222222222222222222222222222222222222222"; +const SHA_A: &str = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"; +const SHA_B: &str = "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"; + +/// A throwaway repo: a ladder contract with two required hosts, a context-rung file, and whatever receipts +/// the case writes. Returns (tempdir, contract dir). +fn repo(ladder_rungs: &str) -> (tempfile::TempDir, std::path::PathBuf) { + let t = tempfile::tempdir().expect("tempdir"); + let c = t.path().join("contracts"); + std::fs::create_dir_all(&c).expect("contracts dir"); + std::fs::write( + c.join("ladder.yaml"), + format!( + "name: ladder\nentity: {{type: gguf}}\nladder:\n hosts:\n - {{id: lambda, cc: sm_89, required: true}}\n - {{id: gx10, cc: sm_121, required: true}}\n - {{id: yoga, cc: sm_86, required: false}}\n rungs:\n{ladder_rungs}" + ), + ) + .expect("ladder"); + let ev = t.path().join("evidence/release"); + std::fs::create_dir_all(&ev).expect("evidence dir"); + std::fs::write( + ev.join("context-rungs.json"), + r#"{"schema":"apr-release-context-rungs/v1", + "consumers":[{"consumer":"rah","max_prompt_tokens":1000,"basis":"measured","source":"u1"}], + "rungs":[{"id":"golden","tokens":0,"derived_from":["g"]},{"id":"consumer-max"}]}"#, + ) + .expect("rungs"); + (t, c) +} + +fn write_receipt(root: &Path, host: &str, body: &str) { + let d = root.join("evidence/dogfood/models/0.69.1"); + std::fs::create_dir_all(&d).expect("receipt dir"); + std::fs::write(d.join(format!("{host}.json")), body).expect("receipt"); +} + +fn receipt(host: &str, apr_sha: &str, inventory: &str, cells: &str) -> String { + format!( + r#"{{"schema":"apr-model-ladder-receipt/v2","host":"{host}","version":"0.69.1","sha":"x", + "apr_sha":"{apr_sha}","cc":"8.9","gpu":"g","inventory":[{inventory}],"cells":[{cells}],"rungs":[]}}"# + ) +} + +fn cell(sha: &str, verb: &str, ctx: &str) -> String { + cell_t(sha, verb, "off", ctx) +} + +fn cell_t(sha: &str, verb: &str, thinking: &str, ctx: &str) -> String { + format!( + r#"{{"sha256":"{sha}","file":"m.gguf","verb":"{verb}","thinking":"{thinking}","context":"{ctx}", + "prompt_tokens":2000,"max_tokens":512,"answer_chars":12, + "verdict":"pass","backend":"cuda","fallback":false,"rc":0}}"# + ) +} + +fn subject() -> Subject { + Subject::new("0.69.1", MC).expect("subject") +} + +#[test] +fn a_subject_refuses_a_short_or_empty_sha_and_an_empty_version() { + assert!( + Subject::new("0.69.1", "225b2a9ab").is_err(), + "a prefix is refused, never matched" + ); + assert!(Subject::new("", MC).is_err()); + assert!(subject().with_receipts_commit("abc").is_err()); + let s = subject().with_receipts_commit(BUMP).expect("40 hex"); + assert_eq!(s.measured_commit(), BUMP); + assert_eq!(subject().measured_commit(), MC); +} + +#[test] +fn consumer_max_is_the_max_over_the_records_and_names_a_consumer_with_no_number() { + let consumers = vec![ + Consumer { + name: "rah".into(), + max_prompt_tokens: Some(148_000), + basis: "measured".into(), + source: "u1".into(), + }, + Consumer { + name: "rmedia".into(), + max_prompt_tokens: Some(128_000), + basis: "plan".into(), + source: "u2".into(), + }, + Consumer { + name: "arbiter".into(), + max_prompt_tokens: None, + basis: String::new(), + source: "u3".into(), + }, + ]; + let rung = ContextRung { + id: "consumer-max".into(), + tokens: Some(7), + long: true, + derived_from: vec![], + unmeasured_consumers: vec![], + }; + let out = derive_rungs(vec![rung], &consumers); + assert_eq!( + out[0].tokens, + Some(148_000), + "a declared number is ignored: the rung is derived" + ); + assert_eq!(out[0].derived_from.len(), 3); + assert_eq!(out[0].unmeasured_consumers, vec!["arbiter".to_string()]); +} + +#[test] +fn the_universe_is_the_inventory_plus_the_cuda_rungs_listed_for_the_host_and_every_cell_gets_a_node( +) { + // rung a: cuda, every host · rung b: cpu only (not a cuda obligation) · rung c: cuda, gx10 only + let (t, c) = repo(&format!( + " - {{id: a, sha256: {SHA_A}, arch: qwen2, gguf: a.gguf, backends: [cpu, cuda], required: true}}\n - {{id: b, sha256: {}, arch: qwen2, gguf: b.gguf, backends: [cpu], required: true}}\n - {{id: c, sha256: {}, arch: qwen3, gguf: c.gguf, backends: [cuda], hosts: [gx10], required: true}}\n", + "c".repeat(64), + "d".repeat(64) + )); + // lambda holds b (inventory) and an unlisted model e + let inv = format!(r#"{{"file":"e.gguf","sha256":"{SHA_B}"}},{{"file":"nohash.gguf"}}"#); + write_receipt( + t.path(), + "lambda", + &receipt("lambda", MC, &inv, &cell(SHA_A, "run", "golden")), + ); + let mut g = Graph::new(); + let st = extract(&mut g, &c, &subject()).expect("extracts"); + assert_eq!(st.required_hosts, 2, "yoga is not required"); + // lambda: a (rung) + e (inventory) = 2 · gx10: a + c = 2 (no receipt, but the rungs listed for it stay) + assert_eq!(st.models, 4); + // no measured length or thinking mode: every rung (golden, consumer-max) + declared, both modes + assert_eq!(st.cells, 4 * 4 * 2 * 3, "models × verbs × thinking × rungs"); + assert_eq!(st.cells_with_row, 1); + let lambda = iri_path("release-host", &["0.69.1", "lambda"]); + let unmeasured = g.objects(&lambda, &rel("unmeasuredModel")); + assert_eq!( + unmeasured.len(), + 1, + "an inventory row with no hash is named, never dropped" + ); + let gx10 = iri_path("release-host", &["0.69.1", "gx10"]); + assert!( + g.objects(&gx10, &rel("hostReceipt")).is_empty(), + "no receipt is a missing edge" + ); + assert!(!g.to_ntriples().contains("_:"), "no blank nodes (R-15)"); +} + +#[test] +fn a_row_keys_by_sha_verb_and_rung_and_carries_fresh_and_context_met() { + let (t, c) = repo(&format!( + " - {{id: a, sha256: {SHA_A}, arch: qwen2, gguf: a.gguf, backends: [cuda], hosts: [lambda], required: true}}\n" + )); + let rows = [ + cell(SHA_A, "chat", "consumer-max"), + cell(SHA_A, "chat", "no-such-rung"), + cell(SHA_B, "chat", "golden"), + ] + .join(","); + let inv = format!(r#"{{"file":"a.gguf","sha256":"{SHA_A}"}}"#); + write_receipt(t.path(), "lambda", &receipt("lambda", BUMP, &inv, &rows)); + let mut g = Graph::new(); + let st = extract(&mut g, &c, &subject()).expect("extracts"); + assert_eq!(st.cells_with_row, 1); + assert_eq!( + st.orphan_rows, 2, + "an unknown rung and a model outside the universe key onto no cell" + ); + let cellnode = iri_path( + "release-cell", + &[ + "0.69.1", + "lambda", + "a.gguf", + "chat", + "think-off", + "consumer-max", + ], + ); + let row = g.objects(&cellnode, &rel("row"))[0] + .as_iri() + .expect("iri") + .to_string(); + let lit = |p: &str| { + g.objects(&row, &rel(p))[0] + .as_literal() + .map(|(v, _)| v.to_string()) + .expect("literal") + }; + assert_eq!( + lit("fresh"), + "false", + "measured at BUMP, released at MC, no --receipts-commit" + ); + assert_eq!( + lit("contextMet"), + "true", + "2000 prompt tokens ≥ consumer-max 1000" + ); + let mut g2 = Graph::new(); + extract( + &mut g2, + &c, + &subject().with_receipts_commit(BUMP).expect("sha"), + ) + .expect("extracts"); + let fresh2 = g2.objects(&row, &rel("fresh"))[0] + .as_literal() + .map(|(v, _)| v.to_string()); + assert_eq!( + fresh2.as_deref(), + Some("true"), + "T-4: R7 proved BUMP ≡ MC and said so" + ); +} + +#[test] +fn a_kernel_on_one_hosts_dispatch_path_is_an_obligation_on_every_required_host() { + let (t, c) = repo(&format!( + " - {{id: a, sha256: {SHA_A}, arch: qwen2, gguf: a.gguf, backends: [cuda], required: true}}\n" + )); + let kd = t.path().join("evidence/dogfood/kernels/0.69.1"); + std::fs::create_dir_all(&kd).expect("kernel dir"); + std::fs::write( + kd.join("gx10.json"), + format!( + r#"{{"schema":"apr-kernel-diff-receipt/v1","host":"gx10","version":"0.69.1","apr_sha":"{MC}","sm":"sm_121", + "dispatch":[{{"sha256":"{SHA_A}","file":"a.gguf","kernels":[{{"kernel":"q4k_gemv","quant":"q4_k"}}]}}], + "rows":[{{"kernel":"q4k_gemv","quant":"q4_k","reference":"cpu","max_err":0.001,"bound":0.004,"bound_source":"pair","verdict":"pass"}}, + {{"kernel":"q4k_gemv","quant":"q6_k","reference":"cpu","max_err":0.001,"bound":0.004,"bound_source":"pair","verdict":"pass"}}]}}"# + ), + ) + .expect("kernel receipt"); + let mut g = Graph::new(); + let st = extract(&mut g, &c, &subject()).expect("extracts"); + assert_eq!(st.kernel_cells, 2, "one kernel × two required hosts"); + let lambda = iri_path("release-kernel", &["0.69.1", "lambda", "q4k_gemv", "q4_k"]); + let gx10 = iri_path("release-kernel", &["0.69.1", "gx10", "q4k_gemv", "q4_k"]); + assert!( + g.objects(&lambda, &rel("row")).is_empty(), + "measured on gx10 only: lambda's row is missing" + ); + assert_eq!( + g.objects(&gx10, &rel("row")).len(), + 1, + "the q6_k row is a different cell (#3712, eb)" + ); + let host = iri_path("release-host", &["0.69.1", "lambda"]); + assert_eq!( + g.objects(&host, &rel("modelWithoutDispatch")).len(), + 1, + "lambda holds a.gguf and has no dispatch list for it" + ); +} + +#[test] +fn a_foreign_context_or_kernel_schema_is_refused_by_name() { + let (t, c) = repo(""); + std::fs::write( + t.path().join("evidence/release/context-rungs.json"), + r#"{"schema":"something/v9"}"#, + ) + .expect("rewrite"); + let e = extract(&mut Graph::new(), &c, &subject()).expect_err("refused"); + assert!(e.to_string().contains("something/v9"), "{e}"); +} + +#[test] +fn a_measured_length_and_thinking_mode_shape_the_cells_and_a_think_on_row_must_close_and_answer() { + let (t, c) = repo(""); + // e.gguf: 1500-token context (golden 0 fits, consumer-max 1000 fits), no thinking mode + // f.gguf: 800-token context (consumer-max 1000 does NOT fit), thinking mode unmeasured → both modes + let inv = format!( + r#"{{"file":"e.gguf","sha256":"{SHA_A}","context_length":1500,"thinking_modes":["off"],"thinking_markers":[]}}, + {{"file":"f.gguf","sha256":"{SHA_B}","context_length":800}}"# + ); + let unclosed = cell_t(SHA_B, "chat", "on", "golden").replace( + r#""answer_chars":12"#, + r#""answer_chars":0,"think_closed":false"#, + ); + let declared_row = cell_t(SHA_A, "run", "off", "declared"); // 2000 + 512 ≥ 1500 + let rows = [unclosed, declared_row].join(","); + write_receipt(t.path(), "lambda", &receipt("lambda", MC, &inv, &rows)); + let mut g = Graph::new(); + let st = extract(&mut g, &c, &subject()).expect("extracts"); + // lambda: e = 4 verbs × 1 mode × 3 rungs = 12 · f = 4 × 2 × 2 rungs (golden, declared) = 16 + // gx10: no receipt and no rung listed → no models, no cells (its Host node carries the missing receipt) + assert_eq!(st.cells, 12 + 16); + let on = |file: &str| { + iri_path( + "release-cell", + &["0.69.1", "lambda", file, "chat", "think-on", "golden"], + ) + }; + assert!( + g.objects(&on("e.gguf"), RDF_TYPE).is_empty(), + "no thinking mode measured → no ON cell" + ); + let row = g.objects(&on("f.gguf"), &rel("row"))[0] + .as_iri() + .expect("iri") + .to_string(); + let lit = |n: &str, p: &str| { + g.objects(n, &rel(p))[0] + .as_literal() + .map(|(v, _)| v.to_string()) + }; + assert_eq!( + lit(&row, "thinkOk").as_deref(), + Some("false"), + "an unclosed think block" + ); + assert_eq!( + lit(&row, "answered").as_deref(), + Some("false"), + "an empty answer" + ); + let big = iri_path( + "release-cell", + &[ + "0.69.1", + "lambda", + "f.gguf", + "chat", + "think-off", + "consumer-max", + ], + ); + assert!( + g.objects(&big, &rel("model")).is_empty(), + "a rung past the declared length is not owed" + ); + let decl = iri_path( + "release-cell", + &["0.69.1", "lambda", "e.gguf", "run", "think-off", "declared"], + ); + let drow = g.objects(&decl, &rel("row"))[0] + .as_iri() + .expect("iri") + .to_string(); + assert_eq!( + lit(&drow, "contextMet").as_deref(), + Some("true"), + "prompt + budget fill the declared length" + ); +} + +#[test] +fn a_rung_file_that_redeclares_the_per_model_rung_is_refused() { + let (t, c) = repo(""); + std::fs::write( + t.path().join("evidence/release/context-rungs.json"), + r#"{"schema":"apr-release-context-rungs/v1","consumers":[],"rungs":[{"id":"declared","tokens":4096}]}"#, + ) + .expect("rewrite"); + let e = extract(&mut Graph::new(), &c, &subject()).expect_err("refused"); + assert!(e.to_string().contains("declared"), "{e}"); +} + +#[test] +fn long_rungs_are_owed_per_the_ladders_long_rungs_for_and_an_echo_that_disagrees_is_named() { + // qwen35 is a family that owes everything; for qwen2 only the representative file does + let (t, c) = repo( + " cells:\n long_rungs_for: {families: [qwen35], representatives: {qwen2: rep.gguf}}\n", + ); + std::fs::write( + t.path().join("evidence/release/context-rungs.json"), + r#"{"schema":"apr-release-context-rungs/v1", + "consumers":[{"consumer":"rah","max_prompt_tokens":1000,"basis":"measured","source":"u1"}], + "rungs":[{"id":"4k","tokens":40,"derived_from":["g"]},{"id":"consumer-max","long":true}]}"#, + ) + .expect("rungs"); + let inv = format!( + r#"{{"file":"big.gguf","sha256":"{SHA_A}","arch":"qwen35","context_length":262144,"thinking_modes":["on","off"],"thinking_markers":["","enable_thinking"]}}, + {{"file":"small.gguf","sha256":"{SHA_B}","arch":"qwen2","context_length":32768,"thinking_modes":["off"],"thinking_markers":["","enable_thinking"],"owes_long_rungs":true}}"# + ); + write_receipt(t.path(), "lambda", &receipt("lambda", MC, &inv, "")); + let mut g = Graph::new(); + let st = extract(&mut g, &c, &subject()).expect("extracts"); + // big: 4 verbs × 2 modes × (4k, consumer-max, declared) = 24 · small: 4 × 1 × (4k) = 4 + assert_eq!(st.cells, 24 + 4); + let small = iri("model", SHA_B); + let lit = |p: &str| { + g.objects(&small, &rel(p)) + .first() + .and_then(|t| t.as_literal()) + .map(|(v, _)| v.to_string()) + }; + assert_eq!( + lit("owesLongRungs").as_deref(), + Some("false"), + "qwen2 and not the representative" + ); + assert!( + lit("longRungsMismatch").is_some_and(|m| m.contains("small.gguf")), + "the echo said true" + ); + assert!( + lit("thinkingContradiction").is_some_and(|m| m.contains("")), + "off only, though the template carries the enable_thinking switch" + ); + let big = iri("model", SHA_A); + assert!(g.objects(&big, &rel("thinkingContradiction")).is_empty()); + assert!( + g.objects(&big, &rel("longRungsMismatch")).is_empty(), + "no echo, nothing to disagree with" + ); +} + +#[test] +fn a_discrete_kernel_is_within_bound_only_on_zero_mismatches_or_near_ties() { + use crate::ontology::extract::release_inputs::KernelRow; + let row = |mism: Option, tie: Option, err: Option| KernelRow { + kernel: "topk".into(), + quant: "f32".into(), + reference: "cpu".into(), + max_err: err, + index_mismatch: mism, + tie_margin: tie, + bound: Some(0.01), + bound_source: "pair".into(), + verdict: "pass".into(), + reason: String::new(), + }; + assert!(row(Some(0), None, None).within_bound(), "same expert sets"); + assert!( + row(Some(3), Some(0.002), None).within_bound(), + "three near-ties" + ); + assert!( + !row(Some(1), Some(0.5), Some(0.0)).within_bound(), + "a real divergence; max_err is moot" + ); + assert!( + !row(Some(1), None, None).within_bound(), + "a mismatch with no margin measured" + ); + assert!( + !row(None, None, None).within_bound(), + "nothing measured is outside every bound" + ); +} + +#[test] +fn the_positive_control_fires_without_any_release_subject_or_file() { + // PMAT-3704 R-3: drawn on every gate run — a sample cell with one fresh row, and a planted cell with none + // that must still be a node (the mutant that emits only measured cells turns this false; measured) + assert!(positive_control()); +} diff --git a/crates/aprender-contracts/src/ontology/extract/release_inputs.rs b/crates/aprender-contracts/src/ontology/extract/release_inputs.rs new file mode 100644 index 0000000000..ccf04c0eaa --- /dev/null +++ b/crates/aprender-contracts/src/ontology/extract/release_inputs.rs @@ -0,0 +1,494 @@ +//! aprender#3715 — the inputs `extract:release-evidence` reads, each refused BY NAME when unreadable: the release +//! subject (the CLI's `--release-*` flags), the consumer-derived context rungs, the dogfood receipt R5 judged, and +//! the per-host kernel-diff receipts. The per-host MODEL receipts are `crate::ontology::receipts` (one reader for +//! the ladder join and the release matrix). +//! +//! Every reader here is pure and deterministic (R-15): files in byte order, no clock, no network, no git. What a +//! file does not say is carried as `None`, never defaulted to the passing value — an absent `fallback` is not +//! `false`, an absent `max_err` is not within any bound. + +use std::path::{Path, PathBuf}; + +/// The context-rung declaration, relative to the repository root. +pub const CONTEXT_RUNGS_FILE: &str = "evidence/release/context-rungs.json"; +pub const CONTEXT_SCHEMA: &str = "apr-release-context-rungs/v1"; +/// The per-host kernel-diff receipts: `evidence/dogfood/kernels//.json` (#3715 addendum). +pub const KERNEL_EVIDENCE_DIR: &str = "evidence/dogfood/kernels"; +pub const KERNEL_SCHEMA: &str = "apr-kernel-diff-receipt/v1"; +/// The tokenizer-parity receipts (aprender#3726): apr's token ids against the pinned llama.cpp comparator. +pub const TOKENIZER_EVIDENCE_DIR: &str = "evidence/dogfood/tokenizer"; +pub const TOKENIZER_SCHEMA: &str = "apr-tokenizer-parity-receipt/v1"; +/// The rung whose token count is DERIVED as the max over the consumer records, never written down. +pub const CONSUMER_MAX: &str = "consumer-max"; +/// Reserved: the per-model rung whose size is the model's own GGUF `context_length` (added by the extractor). +pub const DECLARED: &str = "declared"; + +/// What a release-readiness run is asked about. Built from `--release-version` / `--release-commit` and friends. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Subject { + pub version: String, + /// The release commit (MC), 40 lowercase hex: the dogfood receipt's `commit` must equal it. + pub commit: String, + /// T-4 only: the sha the committed receipts were measured at, after R7 proved the tree equal modulo + /// `evidence/`. `None` → the receipts must carry [`Subject::commit`]. + pub receipts_commit: Option, + /// `--receipts`: the model-receipt dir. `None` → `evidence/dogfood/models//`. + pub receipts_dir: Option, + /// `--kernel-receipts`: `None` → `evidence/dogfood/kernels//`. + pub kernel_receipts_dir: Option, + /// `--dogfood-receipt`: the exact file rule_r5 judged. `None` → the release has no dogfood receipt (a violation). + pub dogfood_receipt: Option, + /// `--tokenizer-receipts`: `None` → `evidence/dogfood/tokenizer//`. + pub tokenizer_receipts_dir: Option, +} + +impl Subject { + /// Refuse a subject no receipt could ever match: an empty version, or a commit that is not 40 hex. A short sha + /// is refused rather than prefix-matched — two releases can share a prefix, and the gate would not know. + pub fn new(version: &str, commit: &str) -> Result { + if version.trim().is_empty() { + return Err(ReleaseError::Subject("--release-version is empty".into())); + } + Ok(Self { + version: version.trim().to_string(), + commit: full_sha("--release-commit", commit)?, + receipts_commit: None, + receipts_dir: None, + kernel_receipts_dir: None, + dogfood_receipt: None, + tokenizer_receipts_dir: None, + }) + } + + /// Set `--receipts-commit` (T-4), under the same 40-hex rule. + pub fn with_receipts_commit(mut self, sha: &str) -> Result { + self.receipts_commit = Some(full_sha("--receipts-commit", sha)?); + Ok(self) + } + + /// The sha every model and kernel receipt's `apr_sha` must equal. + #[must_use] + pub fn measured_commit(&self) -> &str { + self.receipts_commit.as_deref().unwrap_or(&self.commit) + } + + #[must_use] + pub fn model_dir(&self, root: &Path) -> PathBuf { + self.receipts_dir.clone().unwrap_or_else(|| { + root.join(crate::ontology::receipts::EVIDENCE_DIR) + .join(&self.version) + }) + } + + #[must_use] + pub fn tokenizer_dir(&self, root: &Path) -> PathBuf { + self.tokenizer_receipts_dir + .clone() + .unwrap_or_else(|| root.join(TOKENIZER_EVIDENCE_DIR).join(&self.version)) + } + + #[must_use] + pub fn kernel_dir(&self, root: &Path) -> PathBuf { + self.kernel_receipts_dir + .clone() + .unwrap_or_else(|| root.join(KERNEL_EVIDENCE_DIR).join(&self.version)) + } +} + +fn full_sha(flag: &str, s: &str) -> Result { + let s = s.trim().to_ascii_lowercase(); + if s.len() == 40 && s.bytes().all(|b| b.is_ascii_hexdigit()) { + Ok(s) + } else { + Err(ReleaseError::Subject(format!( + "{flag} {s:?} is not a full 40-hex commit (a prefix is refused, never matched)" + ))) + } +} + +/// Why the release evidence could not be read. Every variant is the DECLARATION's fault (exit 3). +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum ReleaseError { + Subject(String), + Receipt(crate::ontology::receipts::ReceiptError), + Input { file: String, what: String }, +} + +impl std::fmt::Display for ReleaseError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Subject(w) => write!(f, "release subject: {w}"), + Self::Receipt(e) => write!(f, "receipt {e}"), + Self::Input { file, what } => write!(f, "{file}: {what}"), + } + } +} + +impl std::error::Error for ReleaseError {} + +fn input_err(file: &str, what: impl Into) -> ReleaseError { + ReleaseError::Input { + file: file.to_string(), + what: what.into(), + } +} + +/// Read `path` as JSON carrying `schema == want`; any other schema is refused by name. +fn read_json(path: &Path, name: &str, want: &str) -> Result { + let text = + std::fs::read_to_string(path).map_err(|e| input_err(name, format!("unreadable: {e}")))?; + let v: serde_json::Value = + serde_json::from_str(&text).map_err(|e| input_err(name, format!("not JSON: {e}")))?; + let schema = v + .get("schema") + .and_then(serde_json::Value::as_str) + .unwrap_or(""); + if schema != want { + return Err(input_err( + name, + format!("schema {schema:?} is not {want} — refused by name"), + )); + } + Ok(v) +} + +fn str_of(v: &serde_json::Value, key: &str) -> Option { + v.get(key) + .and_then(serde_json::Value::as_str) + .map(str::to_string) +} + +fn list<'a>(v: &'a serde_json::Value, key: &str) -> impl Iterator { + v.get(key) + .and_then(serde_json::Value::as_array) + .into_iter() + .flatten() +} + +/// One consumer's measured (or planned) maximum prompt, quoted from its own record on #3710 / #3716. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Consumer { + pub name: String, + /// `None` = the consumer has not recorded a number yet — which makes `consumer-max` underived. + pub max_prompt_tokens: Option, + /// `measured` or `plan`, as the consumer's own record says. + pub basis: String, + pub source: String, +} + +/// One context rung, as declared. `tokens` is `None` for the derived rung until [`derive_rungs`] fills it. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ContextRung { + pub id: String, + pub tokens: Option, + /// A LONG rung is owed only by the models `ladder.cells.long_rungs_for` names (#3712 amendment 2); a short + /// rung is owed by every model it fits. + pub long: bool, + pub derived_from: Vec, + /// Consumers the derived rung could not include because they recorded no number. + pub unmeasured_consumers: Vec, +} + +/// The context-rung declaration as read: the rungs (with `consumer-max` derived) and the consumer records. +pub type ContextDecl = (Vec, Vec); + +/// Read the context-rung declaration. `Ok(None)` when the file is absent — which the shape turns into a +/// violation (a release with no context rung proves nothing about context); a present-but-foreign file is exit 3. +pub fn read_context_rungs(root: &Path) -> Result, ReleaseError> { + let path = root.join(CONTEXT_RUNGS_FILE); + if !path.exists() { + return Ok(None); + } + let v = read_json(&path, CONTEXT_RUNGS_FILE, CONTEXT_SCHEMA)?; + let consumers: Vec = list(&v, "consumers") + .map(|c| Consumer { + name: str_of(c, "consumer").unwrap_or_default(), + max_prompt_tokens: c + .get("max_prompt_tokens") + .and_then(serde_json::Value::as_u64), + basis: str_of(c, "basis").unwrap_or_default(), + source: str_of(c, "source").unwrap_or_default(), + }) + .collect(); + if let Some(r) = list(&v, "rungs").find(|r| str_of(r, "id").as_deref() == Some(DECLARED)) { + return Err(input_err( + CONTEXT_RUNGS_FILE, + format!("rung {r} redeclares `{DECLARED}`, which is each model's own GGUF context_length and is never written down"), + )); + } + let rungs = list(&v, "rungs") + .map(|r| ContextRung { + id: str_of(r, "id").unwrap_or_default(), + tokens: r.get("tokens").and_then(serde_json::Value::as_u64), + long: r + .get("long") + .and_then(serde_json::Value::as_bool) + .unwrap_or(false), + derived_from: list(r, "derived_from") + .filter_map(serde_json::Value::as_str) + .map(str::to_string) + .collect(), + unmeasured_consumers: Vec::new(), + }) + .collect(); + Ok(Some((derive_rungs(rungs, &consumers), consumers))) +} + +/// `consumer-max` is COMPUTED here: the max over every consumer record that states a number, derived from every +/// consumer's source. A declared `tokens` on it is ignored, and a consumer with no number is carried by name so +/// the shape can refuse the rung (a max over a subset is a smaller number wearing the full name). +#[must_use] +pub fn derive_rungs(rungs: Vec, consumers: &[Consumer]) -> Vec { + rungs + .into_iter() + .map(|mut r| { + if r.id == CONSUMER_MAX { + r.tokens = consumers.iter().filter_map(|c| c.max_prompt_tokens).max(); + r.derived_from = consumers + .iter() + .filter(|c| !c.source.is_empty()) + .map(|c| format!("{}: {}", c.name, c.source)) + .collect(); + r.unmeasured_consumers = consumers + .iter() + .filter(|c| c.max_prompt_tokens.is_none()) + .map(|c| c.name.clone()) + .collect(); + } + r + }) + .collect() +} + +/// The dogfood receipt R5 judged (`scripts/dogfood.sh`: `{verdict, commit, version, phase, deferred, …}`). +/// Only verdict, commit and version are read: the pre-publish deferral whitelist is R5's, and a second copy of it +/// here would be a second declaration. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Dogfood { + pub file: String, + pub verdict: String, + pub commit: String, + pub version: String, +} + +pub fn read_dogfood(path: &Path) -> Result { + let name = path.display().to_string(); + let text = + std::fs::read_to_string(path).map_err(|e| input_err(&name, format!("unreadable: {e}")))?; + let v: serde_json::Value = + serde_json::from_str(&text).map_err(|e| input_err(&name, format!("not JSON: {e}")))?; + Ok(Dogfood { + file: name, + verdict: str_of(&v, "verdict").unwrap_or_default(), + commit: str_of(&v, "commit") + .unwrap_or_default() + .to_ascii_lowercase(), + version: str_of(&v, "version").unwrap_or_default(), + }) +} + +/// One inventory model's dispatch path on one host: the kernels `apr parity --per-op` reports for it. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Dispatch { + pub sha256: Option, + pub file: String, + /// (kernel, quant). A GEMV is keyed on its quant — qwen3moe runs `ffn_down_exps` as Q4_K in 24 layers and + /// Q6_K in 24 (#3712, aprender-eb) — so a kernel listed as a bare string carries quant `""`. + pub kernels: Vec<(String, String)>, +} + +/// One kernel-vs-reference measurement. +#[derive(Debug, Clone, PartialEq)] +pub struct KernelRow { + pub kernel: String, + pub quant: String, + pub reference: String, + pub max_err: Option, + /// A DISCRETE kernel (router top-k): token×layer slots whose selected sets differ, and the largest CPU logit + /// gap between rank k and k+1 among them. `max_err` means nothing when the two sides chose different sets. + pub index_mismatch: Option, + pub tie_margin: Option, + pub bound: Option, + pub bound_source: String, + pub verdict: String, + pub reason: String, +} + +/// One kernel-diff receipt file (`apr-kernel-diff-receipt/v1`). +#[derive(Debug, Clone, PartialEq)] +pub struct KernelReceipt { + pub file: String, + pub host: String, + pub version: String, + pub apr_sha: Option, + pub sm: String, + pub dispatch: Vec, + pub rows: Vec, +} + +/// Every `*.json` under `dir`, in byte order (none when the dir is absent — the shape names the host). +pub fn read_kernel_receipts(dir: &Path, root: &Path) -> Result, ReleaseError> { + let mut files = Vec::new(); + walk_json(dir, &mut files); + files.sort(); + files + .iter() + .map(|f| { + let name = f + .strip_prefix(root) + .unwrap_or(f) + .to_string_lossy() + .replace('\\', "/"); + let v = read_json(f, &name, KERNEL_SCHEMA)?; + Ok(parse_kernel_receipt(&name, &v)) + }) + .collect() +} + +fn walk_json(dir: &Path, out: &mut Vec) { + let Ok(rd) = std::fs::read_dir(dir) else { + return; + }; + for e in rd.flatten() { + let p = e.path(); + if p.is_dir() { + walk_json(&p, out); + } else if p.extension().and_then(|x| x.to_str()) == Some("json") { + out.push(p); + } + } +} + +fn parse_kernel_receipt(file: &str, v: &serde_json::Value) -> KernelReceipt { + KernelReceipt { + file: file.to_string(), + host: str_of(v, "host").unwrap_or_default(), + version: str_of(v, "version").unwrap_or_default(), + apr_sha: str_of(v, "apr_sha").map(|s| s.to_ascii_lowercase()), + sm: str_of(v, "sm").unwrap_or_default(), + dispatch: list(v, "dispatch") + .map(|d| Dispatch { + sha256: str_of(d, "sha256").map(|s| s.to_ascii_lowercase()), + file: str_of(d, "file").unwrap_or_default(), + kernels: list(d, "kernels").filter_map(kernel_key).collect(), + }) + .collect(), + rows: list(v, "rows") + .map(|r| KernelRow { + kernel: str_of(r, "kernel").unwrap_or_default(), + quant: str_of(r, "quant").unwrap_or_default().to_ascii_lowercase(), + reference: str_of(r, "reference").unwrap_or_default(), + max_err: r.get("max_err").and_then(serde_json::Value::as_f64), + index_mismatch: r.get("index_mismatch").and_then(serde_json::Value::as_u64), + tie_margin: r.get("tie_margin").and_then(serde_json::Value::as_f64), + bound: r.get("bound").and_then(serde_json::Value::as_f64), + bound_source: str_of(r, "bound_source").unwrap_or_default(), + verdict: str_of(r, "verdict") + .unwrap_or_default() + .to_ascii_lowercase(), + reason: str_of(r, "reason").unwrap_or_default(), + }) + .collect(), + } +} + +/// A dispatch entry: `"kernel"` or `{"kernel": …, "quant": …}`. +fn kernel_key(k: &serde_json::Value) -> Option<(String, String)> { + match k { + serde_json::Value::String(s) => Some((s.clone(), String::new())), + serde_json::Value::Object(_) => Some(( + str_of(k, "kernel")?, + str_of(k, "quant").unwrap_or_default().to_ascii_lowercase(), + )), + _ => None, + } +} + +impl KernelRow { + /// Within the measured bound. Continuous: `max_err ≤ bound`. Discrete (`index_mismatch` present): no mismatch, + /// or every mismatch a near-tie (`tie_margin ≤ bound`). Anything unmeasured is outside. + #[must_use] + pub fn within_bound(&self) -> bool { + match self.index_mismatch { + Some(0) => true, + Some(_) => matches!((self.tie_margin, self.bound), (Some(t), Some(b)) if t <= b), + None => matches!((self.max_err, self.bound), (Some(e), Some(b)) if e <= b), + } + } +} + +/// One model's tokenizer comparison (#3726): apr's ids vs the pinned llama.cpp on a named corpus. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct TokRow { + pub sha256: Option, + pub file: String, + pub family: String, + pub corpus_sha256: String, + pub comparator: String, + pub comparator_sha: String, + pub tokens_compared: Option, + pub mismatches: Option, + /// `decode(encode(x)) == x` over the corpus. + pub roundtrip_ok: Option, + pub verdict: String, + pub reason: String, +} + +impl TokRow { + /// Identical ids over a corpus that is not empty: zero tokens compared is not parity, it is no measurement. + #[must_use] + pub fn exact(&self) -> bool { + self.mismatches == Some(0) && self.tokens_compared.is_some_and(|n| n > 0) + } +} + +/// One tokenizer-parity receipt file. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct TokReceipt { + pub file: String, + pub version: String, + pub apr_sha: Option, + pub rows: Vec, +} + +/// Every `*.json` under `dir` (none when absent — every model's tokenizer cell is then a missing row). +pub fn read_tokenizer_receipts(dir: &Path, root: &Path) -> Result, ReleaseError> { + let mut files = Vec::new(); + walk_json(dir, &mut files); + files.sort(); + files + .iter() + .map(|f| { + let name = f + .strip_prefix(root) + .unwrap_or(f) + .to_string_lossy() + .replace('\\', "/"); + let v = read_json(f, &name, TOKENIZER_SCHEMA)?; + Ok(TokReceipt { + file: name, + version: str_of(&v, "version").unwrap_or_default(), + apr_sha: str_of(&v, "apr_sha").map(|s| s.to_ascii_lowercase()), + rows: list(&v, "rows").map(parse_tok_row).collect(), + }) + }) + .collect() +} + +fn parse_tok_row(r: &serde_json::Value) -> TokRow { + TokRow { + sha256: str_of(r, "sha256").map(|s| s.to_ascii_lowercase()), + file: str_of(r, "file").unwrap_or_default(), + family: str_of(r, "family").unwrap_or_default(), + corpus_sha256: str_of(r, "corpus_sha256").unwrap_or_default(), + comparator: str_of(r, "comparator").unwrap_or_default(), + comparator_sha: str_of(r, "comparator_sha").unwrap_or_default(), + tokens_compared: r.get("tokens_compared").and_then(serde_json::Value::as_u64), + mismatches: r.get("mismatches").and_then(serde_json::Value::as_u64), + roundtrip_ok: r.get("roundtrip_ok").and_then(serde_json::Value::as_bool), + verdict: str_of(r, "verdict") + .unwrap_or_default() + .to_ascii_lowercase(), + reason: str_of(r, "reason").unwrap_or_default(), + } +} diff --git a/crates/aprender-contracts/src/ontology/rdf.rs b/crates/aprender-contracts/src/ontology/rdf.rs index 901dc93f06..f7949781de 100644 --- a/crates/aprender-contracts/src/ontology/rdf.rs +++ b/crates/aprender-contracts/src/ontology/rdf.rs @@ -221,6 +221,19 @@ pub fn iri(kind: &str, id: &str) -> String { format!("{ONT_BASE}{kind}/{}", percent_encode(id)) } +/// A multi-segment instance IRI, `///…`, each segment percent-encoded on its own so a `/` +/// inside one (a file name, a host) cannot fake a boundary — and the path still reads as the cell it names +/// (`release-cell/0.69.1/lambda//chat/consumer-max`, aprender#3715). +#[must_use] +pub fn iri_path(kind: &str, segments: &[&str]) -> String { + let mut out = format!("{ONT_BASE}{kind}"); + for s in segments { + out.push('/'); + out.push_str(&percent_encode(s)); + } + out +} + /// A vocabulary term: `https://ont.paiml.dev/v1alpha1/`. `ont:Contract`, `ont:id`, `ont:depends_on`, …. #[must_use] pub fn ont(name: &str) -> String { diff --git a/crates/aprender-contracts/src/ontology/receipts.rs b/crates/aprender-contracts/src/ontology/receipts.rs index 1e23034dae..f92cea8b07 100644 --- a/crates/aprender-contracts/src/ontology/receipts.rs +++ b/crates/aprender-contracts/src/ontology/receipts.rs @@ -1,9 +1,12 @@ //! ONT-001 §5 ONT-4c1 — `resolves: receipt`: the tracked model-capability-ladder receipts, read once, joined //! to the rungs by sha256, and materialized as edges a shape can count. //! -//! One reader, one schema: `evidence/dogfood/models//.json` with -//! `schema: apr-model-ladder-receipt/v1` (aprender `scripts/model_ladder.sh`). Any other `schema` value is -//! refused BY NAME — a receipt this reader does not understand is not silently read as empty. A rung row with +//! One reader, two versions of one schema: `evidence/dogfood/models//.json` with +//! `schema: apr-model-ladder-receipt/v1` or `/v2` (aprender `scripts/model_ladder.sh`; v2 is #3712's, and adds the +//! host's measured `inventory[]`, the full `apr_sha`, and — with the widened 0.69.1 bar — `cells[]`, one row per +//! (model × verb × context rung), which the release-readiness resolver reads, #3715). Any other `schema` value is +//! refused BY NAME — a receipt this reader does not understand is not silently read as empty. v2 rows are v1 rows +//! plus `file`/`inventory_only`, so the rung join below reads both versions the same way. A rung row with //! `sha256` equal to the contract's is a **witness**; a row WITHOUT `sha256` is not a witness (the 0.68.1 //! receipts committed before aprender#3507 have none — "no measurement", never a match); a row with a //! different hex is a **hex mismatch**, materialized so a shape rejects it naming rung, receipt and both hexes. @@ -32,6 +35,10 @@ use crate::ontology::extract::gguf::{model, Rung}; use crate::ontology::rdf::{iri, Graph, Term, RDF_TYPE}; pub const SCHEMA: &str = "apr-model-ladder-receipt/v1"; +/// #3712's successor: v1 plus `inventory[]`, `apr_sha` and `cells[]`. +pub const SCHEMA_V2: &str = "apr-model-ladder-receipt/v2"; +/// Every schema this reader understands, in version order. +pub const SCHEMAS: [&str; 2] = [SCHEMA, SCHEMA_V2]; /// Where the receipts live, relative to the repository root. pub const EVIDENCE_DIR: &str = "evidence/dogfood/models"; @@ -50,17 +57,87 @@ pub struct Row { pub backends: BTreeMap, } -/// One receipt file. +/// One model the host HOLDS (v2 `inventory[]`): the universe the release must prove, measured on the host. #[derive(Debug, Clone, PartialEq, Eq)] +pub struct InventoryItem { + pub file: String, + /// `None` when the row carries no hash: no measurement, never a match. + pub sha256: Option, + pub arch: Option, + pub quant: Option, + /// The GGUF's own `*.context_length`, read from the file on the host (#3710: "not picked here"). + pub context_length: Option, + /// The thinking modes the model's chat template supports, DERIVED from `tokenizer.chat_template` (#3723, + /// the cop's rule): `enable_thinking` → `["on","off"]`; a generation prompt that always opens `` → + /// `["on"]`; otherwise `["off"]`. `None` (no template read) owes BOTH modes. + pub thinking_modes: Option>, + /// The evidence for `thinking_modes`: the markers found in `tokenizer.chat_template`, and that template's hash. + pub thinking_markers: Option>, + pub chat_template_sha256: Option, + /// The producer's ECHO of whether this model owes the long rungs; pv re-derives it from the ladder contract + /// and a disagreement is a violation (#3712 amendment 2). + pub owes_long_rungs: Option, + /// The declared memory arithmetic (#3710 rule, operator "a"): weights + KV per token (at the chosen + /// precision) + workspace, compared with the host's measured `gpu_mem_total_bytes`. + pub weights_bytes: Option, + pub kv_bytes_per_token: Option, + pub workspace_bytes: Option, + /// The KV precision `kv_bytes_per_token` was computed at (#3712, 62: device KV was measured as f32). + pub kv_dtype: Option, +} + +/// One (model × verb × thinking × context rung) row of v2 `cells[]` (#3712 / #3715). +#[derive(Debug, Clone, PartialEq)] +pub struct CellRow { + pub sha256: Option, + pub file: String, + pub verb: String, + /// `on` / `off` (#3710, operator: "chat with and without thinking, ditto run, ditto code"). + pub thinking: String, + pub context: String, + pub prompt_tokens: Option, + /// The output budget the request asked for; with `prompt_tokens`, what fills the declared context. + pub max_tokens: Option, + /// Thinking ON: did the think block close? `None` on a thinking-OFF row (and on a row that did not say). + pub think_closed: Option, + /// Characters of answer AFTER any think block. `Some(0)` is the empty answer #3720 forbids. + pub answer_chars: Option, + pub ttft_ms: Option, + /// A pre-load refusal's arithmetic: what the cell needs, and what the host has. `verdict: "refused"`. + pub required_bytes: Option, + pub available_bytes: Option, + pub verdict: String, + pub backend: String, + /// `None` when the row does not say — which is not `false`. + pub fallback: Option, + pub rc: Option, + pub reason: String, +} + +/// One receipt file. +#[derive(Debug, Clone, PartialEq)] pub struct Receipt { - /// Path relative to the repository root, `/`-separated. + /// Path relative to the repository root, `/`-separated (or as given, for a receipt dir outside the tree). pub file: String, + pub schema: String, pub host: String, pub version: String, pub sha: String, + /// v2: the full 40-hex commit the measured `apr` was built from. `None` on v1. + pub apr_sha: Option, pub cc: String, pub gpu: String, + /// v2: the host's measured TOTAL GPU memory (unified on GB10), the right-hand side of the fit arithmetic. + /// Owed iff required ≤ total; a refusal is honest only when required > total (#3712, 62): a co-tenant that + /// holds memory must not shrink the owed set. + pub gpu_mem_total_bytes: Option, + /// v2: free GPU memory when measured: carried for the reader, never part of the fit. + pub gpu_mem_free_bytes: Option, pub rows: Vec, + /// v2 `inventory[]`; empty on v1 (which cannot say what the host holds). + pub inventory: Vec, + /// v2 `cells[]`; empty when the measuring side wrote none. + pub cells: Vec, } /// A receipt file this reader refuses, by name. @@ -79,11 +156,16 @@ impl std::fmt::Display for ReceiptError { impl std::error::Error for ReceiptError {} /// Read every `*.json` under `/evidence/dogfood/models/**`, in byte order. A file whose `schema` is not -/// [`SCHEMA`] is an error naming the file and the schema it carries. +/// one of [`SCHEMAS`] is an error naming the file and the schema it carries. pub fn read_all(root: &Path) -> Result, ReceiptError> { - let dir = root.join(EVIDENCE_DIR); + read_dir(&root.join(EVIDENCE_DIR), root) +} + +/// Read every `*.json` under `dir/**`, in byte order, naming each file relative to `root` when it lies under it +/// (a T-1 receipt dir such as `$AP/models-t1` does not, and is named as given). +pub fn read_dir(dir: &Path, root: &Path) -> Result, ReceiptError> { let mut files = Vec::new(); - walk(&dir, &mut files); + walk(dir, &mut files); files.sort(); let mut out = Vec::new(); for f in files { @@ -127,9 +209,10 @@ pub fn parse(file: &str, text: &str) -> Result { .get("schema") .and_then(serde_json::Value::as_str) .unwrap_or(""); - if schema != SCHEMA { + if !SCHEMAS.contains(&schema) { return Err(err(format!( - "schema {schema:?} is not {SCHEMA} — refused by name" + "schema {schema:?} is not {} — refused by name", + SCHEMAS.join(" or ") ))); } let s = |k: &str| { @@ -140,70 +223,115 @@ pub fn parse(file: &str, text: &str) -> Result { }) .unwrap_or_default() }; - let mut rows = Vec::new(); - for r in v - .get("rungs") - .and_then(serde_json::Value::as_array) - .into_iter() - .flatten() - { - let b = |k: &str| { - r.get(k) - .and_then(serde_json::Value::as_bool) - .unwrap_or(false) - }; - let cap = r.get("capability_match"); - let capability_passed = cap - .and_then(|c| c.get("passed")) - .and_then(serde_json::Value::as_bool) - .unwrap_or(false) - && !cap - .and_then(|c| c.get("skipped")) - .and_then(serde_json::Value::as_bool) - .unwrap_or(false); - let mut backends = BTreeMap::new(); - if let Some(bs) = r.get("backends").and_then(serde_json::Value::as_object) { - for (name, st) in bs { - let ran = st - .get("ran") - .and_then(serde_json::Value::as_bool) - .unwrap_or(false); - let fallback = st - .get("fallback") - .and_then(serde_json::Value::as_bool) - .unwrap_or(false); - backends.insert(name.clone(), (ran, fallback)); - } - } - rows.push(Row { - id: r - .get("id") - .and_then(serde_json::Value::as_str) - .unwrap_or_default() - .to_string(), - present: b("present"), - sha_ok: b("sha_ok"), - sha256: r - .get("sha256") - .and_then(serde_json::Value::as_str) - .map(str::to_ascii_lowercase), - required: b("required"), - green: b("green"), - capability_passed, - backends, - }); - } Ok(Receipt { file: file.to_string(), + schema: schema.to_string(), host: s("host"), version: s("version"), sha: s("sha"), + apr_sha: str_of(&v, "apr_sha").map(|x| x.to_ascii_lowercase()), cc: s("cc"), gpu: s("gpu"), - rows, + gpu_mem_total_bytes: v + .get("gpu_mem_total_bytes") + .and_then(serde_json::Value::as_u64), + gpu_mem_free_bytes: v + .get("gpu_mem_free_bytes") + .and_then(serde_json::Value::as_u64), + rows: list(&v, "rungs").map(parse_row).collect(), + inventory: list(&v, "inventory").map(parse_inventory_item).collect(), + cells: list(&v, "cells").map(parse_cell).collect(), }) } +/// The elements of array `key`, or none. +fn list<'a>(v: &'a serde_json::Value, key: &str) -> impl Iterator { + v.get(key) + .and_then(serde_json::Value::as_array) + .into_iter() + .flatten() +} + +fn str_of(v: &serde_json::Value, key: &str) -> Option { + v.get(key) + .and_then(serde_json::Value::as_str) + .map(str::to_string) +} + +fn bool_of(v: &serde_json::Value, key: &str) -> bool { + v.get(key) + .and_then(serde_json::Value::as_bool) + .unwrap_or(false) +} + +/// `passed ∧ ¬skipped`: a skipped gate reports `passed: true`, and that is not a pass. +fn gate_passed(cap: Option<&serde_json::Value>) -> bool { + cap.is_some_and(|c| bool_of(c, "passed") && !bool_of(c, "skipped")) +} + +fn parse_row(r: &serde_json::Value) -> Row { + let mut backends = BTreeMap::new(); + if let Some(bs) = r.get("backends").and_then(serde_json::Value::as_object) { + for (name, st) in bs { + backends.insert(name.clone(), (bool_of(st, "ran"), bool_of(st, "fallback"))); + } + } + Row { + id: str_of(r, "id").unwrap_or_default(), + present: bool_of(r, "present"), + sha_ok: bool_of(r, "sha_ok"), + sha256: str_of(r, "sha256").map(|x| x.to_ascii_lowercase()), + required: bool_of(r, "required"), + green: bool_of(r, "green"), + capability_passed: gate_passed(r.get("capability_match")), + backends, + } +} + +fn parse_inventory_item(r: &serde_json::Value) -> InventoryItem { + InventoryItem { + file: str_of(r, "file").unwrap_or_default(), + sha256: str_of(r, "sha256").map(|x| x.to_ascii_lowercase()), + arch: str_of(r, "arch"), + quant: str_of(r, "quant"), + context_length: r.get("context_length").and_then(serde_json::Value::as_u64), + thinking_modes: strings(r, "thinking_modes"), + thinking_markers: strings(r, "thinking_markers"), + chat_template_sha256: str_of(r, "chat_template_sha256"), + owes_long_rungs: r + .get("owes_long_rungs") + .and_then(serde_json::Value::as_bool), + weights_bytes: r.get("weights_bytes").and_then(serde_json::Value::as_u64), + kv_bytes_per_token: r + .get("kv_bytes_per_token") + .and_then(serde_json::Value::as_u64), + workspace_bytes: r.get("workspace_bytes").and_then(serde_json::Value::as_u64), + kv_dtype: str_of(r, "kv_dtype"), + } +} + +fn parse_cell(r: &serde_json::Value) -> CellRow { + CellRow { + sha256: str_of(r, "sha256").map(|x| x.to_ascii_lowercase()), + file: str_of(r, "file").unwrap_or_default(), + verb: str_of(r, "verb").unwrap_or_default(), + thinking: str_of(r, "thinking").unwrap_or_default(), + context: str_of(r, "context").unwrap_or_default(), + prompt_tokens: r.get("prompt_tokens").and_then(serde_json::Value::as_u64), + max_tokens: r.get("max_tokens").and_then(serde_json::Value::as_u64), + think_closed: r.get("think_closed").and_then(serde_json::Value::as_bool), + answer_chars: r.get("answer_chars").and_then(serde_json::Value::as_u64), + ttft_ms: r.get("ttft_ms").and_then(serde_json::Value::as_f64), + required_bytes: r.get("required_bytes").and_then(serde_json::Value::as_u64), + available_bytes: r.get("available_bytes").and_then(serde_json::Value::as_u64), + verdict: str_of(r, "verdict").unwrap_or_default(), + backend: str_of(r, "backend").unwrap_or_default(), + fallback: r.get("fallback").and_then(serde_json::Value::as_bool), + rc: r.get("rc").and_then(serde_json::Value::as_i64), + reason: str_of(r, "reason").unwrap_or_default(), + } +} + /// What resolution found, for the gate's report. #[derive(Debug, Clone, Default, PartialEq, Eq)] pub struct ResolveStats { @@ -344,6 +472,16 @@ fn emit_receipt_node(g: &mut Graph, rec: &Receipt, row: &Row, rung: &Rung) -> St node } +/// An array of strings at `key`, or `None` when the key is absent (which is not an empty list). +fn strings(r: &serde_json::Value, key: &str) -> Option> { + r.get(key).and_then(serde_json::Value::as_array).map(|a| { + a.iter() + .filter_map(serde_json::Value::as_str) + .map(str::to_string) + .collect() + }) +} + #[cfg(test)] mod tests { use super::*; @@ -378,6 +516,42 @@ mod tests { let e = parse("x.json", r#"{"schema":"something/v9","rungs":[]}"#).expect_err("refused"); assert!(e.what.contains("something/v9"), "{e}"); assert!(e.what.contains(SCHEMA), "{e}"); + assert!(e.what.contains(SCHEMA_V2), "{e}"); + } + + #[test] + fn a_v2_receipt_joins_its_rungs_like_v1_and_carries_inventory_apr_sha_and_cells() { + // #3712's v2: the rung rows are v1 rows plus `file`/`inventory_only`, so the ladder join is unchanged — + // the arm that goes RED when v2 is refused (the collision #3715 closes) or read as empty. + let row = OK.replace( + r#""id":"a","#, + r#""id":"a","file":"a.gguf","inventory_only":false,"#, + ); + let text = format!( + r#"{{"schema":"apr-model-ladder-receipt/v2","host":"lambda","version":"0.69.1","sha":"abc", + "apr_sha":"ABCDEF0123456789ABCDEF0123456789ABCDEF01","cc":"8.9","gpu":"g", + "inventory":[{{"file":"a.gguf","sha256":"AAAA","bytes":4}}], + "cells":[{{"sha256":"aaaa","file":"a.gguf","verb":"chat","context":"golden","prompt_tokens":40, + "verdict":"pass","backend":"cuda","fallback":false,"rc":0,"reason":""}}], + "rungs":[{row}]}}"# + ); + let rec = parse("evidence/dogfood/models/0.69.1/lambda.json", &text).expect("v2 parses"); + assert_eq!(rec.schema, SCHEMA_V2); + assert_eq!( + rec.apr_sha.as_deref(), + Some("abcdef0123456789abcdef0123456789abcdef01") + ); + assert_eq!(rec.inventory.len(), 1); + assert_eq!(rec.inventory[0].sha256.as_deref(), Some("aaaa")); + assert_eq!(rec.cells.len(), 1); + assert_eq!(rec.cells[0].verb, "chat"); + assert_eq!(rec.cells[0].fallback, Some(false)); + let mut g = Graph::new(); + let st = resolve(&mut g, &[rung("a", "aaaa", &[], true)], &[rec]); + assert_eq!((st.witnesses, st.green_on), (1, 1)); + let v3 = + parse("x.json", r#"{"schema":"apr-model-ladder-receipt/v3"}"#).expect_err("v3 refused"); + assert!(v3.what.contains("v3"), "{v3}"); } #[test] diff --git a/docs/audits/impl-PMAT-3715-receipt.md b/docs/audits/impl-PMAT-3715-receipt.md new file mode 100644 index 0000000000..6c8a228cdf --- /dev/null +++ b/docs/audits/impl-PMAT-3715-receipt.md @@ -0,0 +1,97 @@ +# PMAT-3715 implementation receipt: `release-readiness-v1` (aprender#3715) + +**Read this before judging scope.** `pmat work status PMAT-3715` prints only the ticket's title. The ticket's scope +is the ISSUE, and the issue's author (the cop, aprender-04, posting as `noahgift`) widened it on the issue itself. +Below, every criterion this diff implements is **quoted verbatim, with its URL**, so each part of the diff can be +checked against the words that asked for it. Nothing below is a paraphrase standing in for a criterion. + +## 1. The issue body's done_when (https://github.com/paiml/aprender/issues/3715) + +> - [ ] **the evidence graph**: `pv extract` ingests the release receipts (per-host model inventory, ladder and matrix receipts from #3712/#3708, dogfood receipt) as nodes: `:Model` (sha256, quant, arch), `:Host`, `:Verb` (run/chat/serve/code), `:ContextRung` (token counts DERIVED from the consumer-traffic records on #3710, never invented), `:Receipt` (verdict, backend, fallback, apr version plus sha) +> - [ ] **the shape** `release-readiness-v1`: focus = every `:Model` in the inventory of every required `:Host`. For each (verb, context rung) exactly one `:Receipt` with `verdict=Pass`, `backend=cuda`, `fallback=false`, `apr_sha = release commit`. A missing, skipped, stale-sha or fallback receipt is a violation naming the cell. +> - [ ] **armed with a red-turning falsifier**: fixtures where each single cell is missing, skipped, falls back, or is stale, and each one turns the gate RED (the ONT-001 rule: a shape that has never gone red isn't armed) +> - [ ] **wired where the decision is made**: autopilot's T-1 step and `check_publish_preflight.sh` both run `pv lint --gate shapes --shape release-readiness-v1` on the release commit's evidence, and a violation STOPs before the tag and refuses at T-4 +> - [ ] **proof**: on the v0.69.0 evidence the shape reports the actual leaks (qwen3-8b lambda, qwen2.5-coder lambda, qwen3moe both hosts, chat/serve/code cells absent). On 0.69.1 it passes with N/N cells named. + +## 2. Scope ADDED on the issue by its author, each implemented here + +**2a. Kernel differential**, https://github.com/paiml/aprender/issues/3715#issuecomment-5763396082: +> **Added cell (operator approved, verbatim "yes, to all", 2026-09-21): kernel differential per GPU architecture.** +> - [ ] The evidence graph gains `:KernelDiffReceipt` nodes: (kernel, sm arch ∈ {sm_89 lambda, sm_121 gx10}, reference ∈ {cpu, cuda-oxide twin}, max_err, bound, verdict). The bound is DERIVED from a measured known-good/known-bad pair (the C14 basis pattern), never picked. +> - [ ] `release-readiness-v1` requires a Pass receipt for every kernel on the release's dispatch path (the kernels `apr parity --per-op` reports for each inventory model) on EACH GPU arch. A kernel that behaves differently on sm_89 and sm_121 is a violation BEFORE any model-level cell is judged. +> - [ ] The falsifier fixture is a kernel receipt that passes on sm_121 and fails on sm_89, and it turns the gate RED. + +→ `release:KernelCell` / `release:KernelDiffReceipt`, shape `release-readiness-v1.kernel`, reported before any cell. The fixture is `a_kernel_green_on_sm_121_and_red_on_sm_89_is_named_on_lambda_before_any_model_cell`. The (kernel, quant) key and the discrete-kernel `index_mismatch`/`tie_margin` rule are the producer's amendments, agreed on #3712 (issuecomment-5763633743). + +**2b. Thinking × context dimensions**, https://github.com/paiml/aprender/issues/3715#issuecomment-5763490841: +> **Dimensions added (operator, verbatim):** "lets ensure this release has MORE than enough tokens for our needs (overdo it), and our verbs are dogfooded, i.e chat with and without thinking, ditto run, ditto code, etc,". Each cell is now (model, host, verb ∈ {run, chat, serve, code}, **thinking ∈ {on, off}**, **context rung ∈ {4k, 8k, 20k, 60k, 148k, the model's declared context_length}**). The declared length is READ from the GGUF, not picked. Thinking ON must close its block and answer correctly, or REPORT an unclosed block as an error, never an empty answer. Full statement on #3710. + +The full statement, https://github.com/paiml/aprender/issues/3710#issuecomment-5763489958, adds: +> Each Qwen3.5 model must be CORRECT … at every rung up to the **model's own declared context length** (GGUF `*.context_length`, read from the file, not picked here). + +→ the cell key (verb, `think-on|off`, rung); `thinkOk` / `answered` on every row; the `declared` rung sized by the model's own `context_length`; a rung above the model's length is not owed. + +**2c. Which thinking modes a model owes**, https://github.com/paiml/aprender/issues/3723#issuecomment-5763858250 (the cop accepted this derivation there): +> - the template honours `enable_thinking` → **both** modes are supported (ON and OFF cells owed) +> - else the generation prompt ALWAYS opens `` → **ON only** (an OFF request is refused by name) +> - else → **OFF only** (an ON request is refused by name) + +→ `inventory[].thinking_modes`, and a `thinkingContradiction` whenever the markers disprove the modes. + +**2d. Tokenizer parity**, #3726's own done_when, https://github.com/paiml/aprender/issues/3726: +> a **tokenizer-parity gate** … that fails on any id mismatch against the pinned llama.cpp … It becomes a required cell per model in the release shape (#3715). + +→ `release:TokenizerCell`, shape `release-readiness-v1.tokenizer`; one mismatching id turns it RED. + +**2e. The memory rule**, operator decision "a", https://github.com/paiml/aprender/issues/3710#issuecomment-5763753907: +> - A (model, host, context-rung) cell is **owed on a host only if it fits** by the declared memory arithmetic (weights + KV at the chosen precision + workspace ≤ that host's GPU memory, measured). +> - Every rung a model declares must be **owed and Pass on at least one required host.** +> - On a host where the cell doesn't fit, a DIFFERENT cell is owed: **an honest pre-load refusal** (non-zero, naming the arithmetic), never an OOM mid-run and never a silent truncation. + +→ `release:RefusalCell` (shape `.refusal`) and `release:RungCoverage` (shape `.coverage`). The rule that a refusal is honest only against TOTAL memory, never free, is the producer's amendment on #3712 (issuecomment-5763795158 thread). + +**2f. No waivers.** #3710 allows "a FAIL for the release unless the operator waives that cell by name", but every agent session posts as `noahgift`, so a comment-based waiver is forgeable by any of them. The cop's ruling, recorded in the body of https://github.com/paiml/aprender/issues/3722: *"(c). #3715 ships with no waiver path."* The diff therefore has no waiver input. + +**2g. The extractor's positive control: #3704's rule, which now applies to this new extractor.** origin/main was +merged in because #3706 (PMAT-3704, merged 52f43da71) ties `pc_extract` to Σ: the merged tree would have been RED +with `release-evidence` implemented and uncontrolled. The criteria of https://github.com/paiml/aprender/issues/3704, +quoted verbatim, as they now bind `release-evidence`: +> - A test that fails when an extractor the shapes gate runs has no `pc_extract` entry — so the next extractor cannot ship without its control. +> - A not-fired parity control makes the gate refuse (as the other four already do), mutation-proved. + +→ `release_evidence::positive_control()` is drawn on EVERY gate run, with no release subject and no file. It runs +a sample cell with one fresh Pass row and a planted cell with no row through the real `build()`. It fires iff the +sample's cell has exactly one fresh row AND the planted cell is still a `release:Cell` node with ZERO rows, which +is to say absence is still materialized for `minCount 1`. `extract_controls()` carries it as `release-evidence`. +#3706's own Σ-tie test (`every_implemented_entity_type_in_sigma_has_an_extract_control_and_it_fires`) passes. +**Mutant, measured:** an extractor that emits only the cells it has rows for → `pv lint contracts --gate shapes` +exits **2**, `shapes: positive control pc_extract.release-evidence did not fire`, `decline: PositiveControlFailed`. +Restored → Pass, with all eight controls `fired`. + +## 3. Points a reviewer may otherwise read as out of scope + +- **"All models, not just Q4_K."** The universe is the host's measured inventory, exactly as done_when 2 says ("focus = every `:Model` in the inventory"). The inventory is defined by the ladder contract's `inventory.patterns`: `*q4_k*.gguf`, `*q4k*.gguf`, `*q4_k*.apr`, `*q4k*.apr` (#3712). That makes it Q4_K by construction. pv does not re-filter by quant, because that would be a second declaration of the universe. Ladder rungs join the inventory only when they claim `cuda`. +- **`receipts.rs` now reads v2.** #3712's `apr-model-ladder-receipt/v2` is the receipt done_when 1 ingests. Without this change, the first committed v2 receipt turns `pv lint --gate shapes` into exit 3 on every PR. A v3 is still refused by name (`a_v2_receipt_joins_its_rungs_like_v1_…`). +- **`--shape` forces arming.** done_when 4's invocation is `pv lint --gate shapes --shape release-readiness-v1`. A named shape that was only "computed and reported" would print its violations and exit 0 at T-1. In ordinary PR lint the family is unarmed and has 0 focus nodes, because there is no release subject. +- **The ont4b shape ratchet goes 9 → 18.** That test hard-codes the count so that a new shape must be named; the nine are named in it. + +## 4. Measured at the PR head + +| check | result | +|---|---| +| `cargo test -p aprender-contracts --lib` (merged head) | 1701 passed, 0 failed | +| `cargo test -p aprender-contracts-cli` (every target) | all green; `ont_release_readiness` has 34 cases | +| `cargo clippy -p aprender-contracts --lib --tests -- -D warnings`, and the same for `-cli` | clean | +| `pv lint contracts` (full) | PASS, 0 errors; no warning from `release-readiness-v1.yaml` | +| `pv extract contracts --check` | fresh (`contracts.nt`, `shapes.ttl` regenerated) | +| `bash scripts/check_ont_ratchet.sh --check` | PASS, after the `--write` restamp | +| `bash scripts/check_explicit_test_commands.sh` | PASS (new `.cmd` 450) | +| mutant: `in: [pass, skip]` on the cell verdict | `a_skipped_cell_is_red_naming_it` FAILS, so it is caught | +| mutant: `fresh()` always true | 3 staleness cases FAIL, so it is caught | +| mutant: emit only cells that have rows | `pv lint contracts --gate shapes` exit 2, `pc_extract.release-evidence did not fire` | +| 0.69.0 proof (`evidence/release/proof-0.69.0/`) | exit 1, 795 findings, naming every leak in done_when 5 | + +## 5. What this PR does NOT do (so it carries `Refs #3715`, not `Closes`) + +- done_when 4's call sites: autopilot T-1 and `check_publish_preflight.sh` T-4. Those are aprender-f0's (#3717/#3708). This PR provides the exact invocation and exit contract they call, agreed on #3712 (issuecomment-5763344947): 0 Pass · 1 Fail · 2 decline · 3 caller error, and every non-zero STOPs. +- done_when 5's second half, "on 0.69.1 it passes with N/N cells named". It needs the producer (#3712 `cells[]`, the memory arithmetic, and kernel and tokenizer receipts) and the model fixes. `release.cell_names` is how it will name the cells. diff --git a/docs/audits/quorum-PMAT-3715.json b/docs/audits/quorum-PMAT-3715.json new file mode 100644 index 0000000000..a916791f9d --- /dev/null +++ b/docs/audits/quorum-PMAT-3715.json @@ -0,0 +1,187 @@ +{ + "ticket": "PMAT-3715", + "base": "origin/main", + "base_resolved": "origin/main", + "base_note": "no origin/origin/main exists; judged against the local ref", + "head": "84289cb7ba38adf96c0a1c5f836eb84ccb8d0646", + "diff_sha256": "9df958d287befbf36a6b50be55984468291262bf8a3cf2cf9507b584e2f5676b", + "width": 3, + "executor": "agy", + "prompt_mode": "file", + "prompt_bytes": 291117, + "author": { + "model": "claude-opus-5", + "family": "claude", + "source": "flag" + }, + "agreed": true, + "lanes": [ + { + "lane": 1, + "status": "SUCCESS", + "verdict": "PASS", + "summary": "The PR accurately and completely implements the PMAT-3715 release-readiness shape and its requested scope additions. I verified the graph extraction logic, the generation of KernelDiffReceipt and TokenizerParityReceipt nodes, the parsing of the GGUF `context_length` for declared rungs, and the correct verification of thinking mode consistency against `enable_thinking` and `thinking_markers`. The release-readiness-v1 shape rigorously checks for missing, skipped, or stale receipts and requires `fallback=false` without providing any waiver path. Furthermore, the `positive_control()` has been successfully added to `extract_controls()` to ensure the gate cannot ship without its control, perfectly fulfilling the ticket requirements without introducing any forbidden behavior.", + "findings": [], + "raw_bytes": 2809, + "err_bytes": 100, + "envelope_status": "SUCCESS", + "verdict_source": "structured_output", + "grounding_check": "parity", + "model": "gemini-3.1-pro-high", + "model_measured": "gemini-3.1-pro-high", + "model_source": "measured", + "family": "gemini", + "role": "independent", + "fallback": { + "judged_by": "gemini-3.1-pro-high", + "exhausted": false, + "attempts": [ + { + "model": "gemini-3.1-pro-high", + "family": "gemini", + "outcome": "answered" + } + ], + "skipped": [] + } + }, + { + "lane": 2, + "status": "SUCCESS", + "verdict": "PASS", + "summary": "The diff perfectly implements the requirements of PMAT-3715 and the receipt claims. The SHACL shape enforces the required matrix of cells, and all edge cases (missing, skipped, fallback, stale receipts, kernel architecture mismatches, tokenizer parity) are verified in the test suite to turn the gate red and name the cell. There are no doctrinal violations, weakened gates, or unauthorized scopes. The gate works exactly as requested.", + "findings": [], + "raw_bytes": 3819, + "err_bytes": 100, + "envelope_status": "SUCCESS", + "verdict_source": "structured_output", + "grounding_check": "parity", + "model": "gemini-3.1-pro-low", + "model_measured": "gemini-3.1-pro-low", + "model_source": "measured", + "family": "gemini", + "role": "independent", + "fallback": { + "judged_by": "gemini-3.1-pro-low", + "exhausted": false, + "attempts": [ + { + "model": "gemini-3.1-pro-low", + "family": "gemini", + "outcome": "answered" + } + ], + "skipped": [] + } + }, + { + "lane": 3, + "status": "SUCCESS", + "verdict": "PASS", + "summary": "The audit prompt file was fully read, the repository builds, and all tests pass. No issues found.", + "findings": [], + "raw_bytes": 1862, + "err_bytes": 100, + "envelope_status": "SUCCESS", + "verdict_source": "structured_output", + "model": "gpt-oss-120b-medium", + "model_measured": "gpt-oss-120b-medium", + "model_source": "measured", + "family": "openai", + "role": "independent", + "fallback": { + "judged_by": "gpt-oss-120b-medium", + "exhausted": false, + "attempts": [ + { + "model": "gpt-oss-120b-medium", + "family": "openai", + "outcome": "capacity", + "reason": "Our servers are experiencing high traffic right now, please try again in a minute. (UNAVAILABLE (code 503): No capacity available for model gpt-oss-120b-medium on the server)" + }, + { + "model": "gpt-oss-120b-medium", + "family": "openai", + "outcome": "answered" + } + ], + "skipped": [] + } + } + ], + "dissent": [], + "dedup": [], + "uncovered": [], + "coverage_source": "lanes", + "partial": false, + "partial_reasons": [ + "lane 3: grounding NOT COMPARED — structured_output holds a verdict but no single verdict object could be read out of .response (1862 raw bytes), so agy's labels stand unchecked for this lane (PMAT-082)", + "lane 3: judged by gpt-oss-120b-medium after falling through gpt-oss-120b-medium (capacity: Our servers are experiencing high traffic right now, please try again in a minute. (UNAVAILABLE (code 503): No capacity available for model gpt-oss-120b-medium on the server)) (PMAT-321)" + ], + "fallback": { + "same_family_width": 1, + "chain": [ + { + "model": "gemini-3.1-pro-high", + "family": "gemini", + "disposition": "configured" + }, + { + "model": "gemini-3.1-pro-low", + "family": "gemini", + "disposition": "configured" + }, + { + "model": "gpt-oss-120b-medium", + "family": "openai", + "disposition": "configured" + }, + { + "model": "qwen3.5", + "family": "qwen", + "disposition": "not-run", + "why": "no quorum.local_lane in the config — the aprender lane has no model to load" + }, + { + "model": "claude-opus-4-6-thinking", + "family": "claude", + "disposition": "width", + "why": "same family as the author: at most 1 lane, recorded role width, counted toward no floor (R-15a)" + }, + { + "model": "claude-sonnet-4-6", + "family": "claude", + "disposition": "width", + "why": "same family as the author: at most 1 lane, recorded role width, counted toward no floor (R-15a)" + } + ], + "precheck": [ + { + "family": "gemini", + "model": "gemini-3.1-pro-high", + "probe": 1, + "outcome": "live" + }, + { + "family": "openai", + "model": "gpt-oss-120b-medium", + "probe": 1, + "outcome": "live" + } + ], + "prah": { + "source": "install-receipt", + "path": "/home/noah/.claude/skills/paiml-implement/bin/prah" + } + }, + "auto_merge": { + "checked": true, + "was_armed": false, + "disarmed": false, + "note": "auto-merge not armed" + }, + "lint": { + "ok": true, + "output": "receipt complete: kind=artifact lanes=3 author=claude-opus-5/claude" + } +} diff --git a/docs/roadmaps/entries/PMAT-3715.yaml b/docs/roadmaps/entries/PMAT-3715.yaml new file mode 100644 index 0000000000..2ddf533ddf --- /dev/null +++ b/docs/roadmaps/entries/PMAT-3715.yaml @@ -0,0 +1,22 @@ +- id: PMAT-3715 + github_issue: 3715 + item_type: task + title: 'pv SHACL release-readiness shape: the tag is REFUSED unless every (Q4_K model in host inventory × {lambda,gx10} × {run,chat,serve,code} × consumer-context) cell has a Pass receipt — a missing cell is a violation, never a skip' + status: in_progress + priority: critical + assigned_to: aprender-97 + created: 2026-09-21T15:55:00Z + updated: 2026-09-21T15:55:00Z + spec: null + acceptance_criteria: + - 'Issue #3715 done_when 1, verbatim: "**the evidence graph**: `pv extract` ingests the release receipts (per-host model inventory, ladder and matrix receipts from #3712/#3708, dogfood receipt) as nodes: `:Model` (sha256, quant, arch), `:Host`, `:Verb` (run/chat/serve/code), `:ContextRung` (token counts DERIVED from the consumer-traffic records on #3710, never invented), `:Receipt` (verdict, backend, fallback, apr version plus sha)"' + - 'done_when 2, verbatim: "**the shape** `release-readiness-v1`: focus = every `:Model` in the inventory of every required `:Host`. For each (verb, context rung) exactly one `:Receipt` with `verdict=Pass`, `backend=cuda`, `fallback=false`, `apr_sha = release commit`. A missing, skipped, stale-sha or fallback receipt is a violation naming the cell."' + - 'done_when 3, verbatim: "**armed with a red-turning falsifier**: fixtures where each single cell is missing, skipped, falls back, or is stale, and each one turns the gate RED (the ONT-001 rule: a shape that has never gone red isn''t armed)"' + - 'done_when 4, verbatim: "**wired where the decision is made**: autopilot''s T-1 step and `check_publish_preflight.sh` both run `pv lint --gate shapes --shape release-readiness-v1` on the release commit''s evidence, and a violation STOPs before the tag and refuses at T-4"' + - 'done_when 5, verbatim: "**proof**: on the v0.69.0 evidence the shape reports the actual leaks (qwen3-8b lambda, qwen2.5-coder lambda, qwen3moe both hosts, chat/serve/code cells absent). On 0.69.1 it passes with N/N cells named."' + phases: [] + subtasks: [] + estimated_effort: null + labels: + - kind:code + notes: 'KNOWN REMAINDER (cop ruling, verbatim): release_evidence.rs `VERBS` is a hand-coded list; #3745 S2 replaces it with the derived cell set via `Inputs.cells`. 0.69.1 does not tag while it exists. QUORUM: round 3 AGREED at 84289cb7ba38adf96c0a1c5f836eb84ccb8d0646 (gemini-3.1-pro-high, gemini-3.1-pro-low, gpt-oss-120b-medium all PASS; docs/audits/quorum-PMAT-3715.json). Assigned by the cop (aprender-04) to aprender-97 on the 0.69.1 all-hands (EPIC #3710). The receipt FORMAT is shared and was agreed on #3712 before building: with aprender-62 (measuring side: issuecomment-5763332078, -5763540875, -5763596544, memory fields -5763795158) and aprender-f0 (T-1/T-4 wiring, issuecomment-5763344947); kernel-diff with aprender-37/aprender-eb (-5763412455, -5763633743, -5763653200); tokenizer parity on #3726 (-5763745060). SCOPE ADDED ON THE ISSUE by its author (cop), each built: kernel differential per GPU arch (#3715 issuecomment-5763396082); thinking {on,off} x context rungs {4k,8k,20k,60k,consumer-max,declared} (-5763490841, #3710 -5763489958); tokenizer parity per model (#3726 done_when 3); the memory-fit rule, operator decision "a" (#3710 -5763753907); three-state thinking_modes (#3723). WAIVERS: none, cop ruling (c) - a comment-based waiver is forgeable by every agent session (all post as noahgift); the operator-only surface is #3722. DONE HERE: done_when 1 (pv extract --release-* --out ingests inventory, cells, kernel/tokenizer/dogfood receipts as Release/Host/Model/Verb/ContextRung/Cell/CellReceipt/... nodes); done_when 2 (the nine-shape release-readiness-v1 family, armed by --shape); done_when 3 (34 red-turning CLI cases in crates/aprender-contracts-cli/tests/ont_release_readiness.rs, wired as ci/explicit-test-commands.d/450; two mutants caught: skip-as-pass turns 1 red, always-fresh turns 3 red); the pv side of done_when 4 (the invocation and exit contract f0 calls: 0 Pass, 1 Fail naming cells, 2 decline, 3 caller error); the first half of done_when 5 (evidence/release/proof-0.69.0: RED, 795 findings naming qwen3-8b lambda verdict=fail, qwen2.5-coder lambda and qwen3moe both hosts unmeasured/unkeyed, 752 of 768 cells absent, the NO-GO inherited dogfood). NOT DONE HERE (so the PR carries Refs, not Closes): the autopilot T-1 / check_publish_preflight T-4 call sites (aprender-f0, #3717/#3708), and "on 0.69.1 it passes with N/N cells named", which needs the producer (#3712 cells[]) and the model fixes to land. Also fixes a collision that existed independently: receipts.rs refused every schema but apr-model-ladder-receipt/v1 by name, so the first committed v2 receipt would have turned `pv lint --gate shapes` exit 3 on every PR.' diff --git a/evidence/release/context-rungs.json b/evidence/release/context-rungs.json new file mode 100644 index 0000000000..c4f107df9a --- /dev/null +++ b/evidence/release/context-rungs.json @@ -0,0 +1,54 @@ +{ + "schema": "apr-release-context-rungs/v1", + "_doc": "aprender#3715. The context rungs release-readiness-v1 requires of every (model, host, verb, thinking) cell. `consumer-max` carries no number: pv COMPUTES it as the max over `consumers[]`, and a consumer with no number is a named violation. The per-model `declared` rung is each model's own GGUF context_length and is never written here (pv refuses it). A `long` rung (and `declared`) is owed only by the models the ladder's `cells.long_rungs_for` names (#3712 amendment 2); a rung larger than a model's measured context_length is not owed by that model (cop ruling, 2026-09-21: 'correct at every rung up to the model's own declared context length', #3710).", + "consumers": [ + { + "consumer": "rah", + "max_prompt_tokens": 148000, + "basis": "measured", + "source": "https://github.com/paiml/aprender/issues/3716#issuecomment-5763300198", + "quote": "max (PMAT-216) | 454,761 | diff alone 120,745 measured; whole ≈ 148k" + }, + { + "consumer": "rmedia", + "max_prompt_tokens": 128000, + "basis": "plan", + "source": "https://github.com/paiml/aprender/issues/3716#issuecomment-5763407650", + "quote": "Ask: 64k usable context minimum, 128k to cover the specialization-scale case." + }, + { + "consumer": "arbiter", + "max_prompt_tokens": 7000, + "basis": "plan", + "source": "https://github.com/paiml/aprender/issues/3716#issuecomment-5763449814", + "quote": "Planned synthesis input = template + top-8 chunks: 9,150 and 21,703 chunk characters for those two questions ... That is ≈3k–7k tokens" + } + ], + "rungs": [ + { + "id": "4k", + "tokens": 4096, + "derived_from": ["https://github.com/paiml/aprender/issues/3710#issuecomment-5763489958 (operator: \"MORE than enough tokens for our needs (overdo it)\"; required rungs 4k · 8k · 20k · 60k · 148k · declared)"] + }, + { + "id": "8k", + "tokens": 8192, + "derived_from": ["https://github.com/paiml/aprender/issues/3710#issuecomment-5763489958 (operator: \"overdo it\"; required rungs 4k · 8k · 20k · 60k · 148k · declared)"] + }, + { + "id": "20k", + "tokens": 20480, + "derived_from": ["https://github.com/paiml/aprender/issues/3716#issuecomment-5763300198 (RAH p50 quorum brief: whole brief ≈ 20k; sized up to 20480 with the binary rungs, #3712 aprender-62)"] + }, + { + "id": "60k", + "tokens": 61440, + "long": true, + "derived_from": ["https://github.com/paiml/aprender/issues/3716#issuecomment-5763300198 (RAH p90 quorum brief: whole ≈ 60k; sized up to 61440, #3712 aprender-62)"] + }, + { + "id": "consumer-max", + "long": true + } + ] +} diff --git a/evidence/release/proof-0.69.0/README.md b/evidence/release/proof-0.69.0/README.md new file mode 100644 index 0000000000..7857cc76df --- /dev/null +++ b/evidence/release/proof-0.69.0/README.md @@ -0,0 +1,48 @@ +# `release-readiness-v1` on v0.69.0's own evidence (aprender#3715 done_when 5) + +The question: had this shape been the gate at 0.69.0's T-1, would it have stopped the train, and would it have +named the leaks that actually shipped to the publish step? **Yes: RED, 795 findings.** Every leak the issue lists +is named. `gate-summary.json` holds the verdict, the counts, every non-cell finding verbatim, and the cell findings +aggregated per host/model. Re-run the command below for all 795 findings. + +## What went in: 0.69.0's measurements, translated, none invented + +| file | from | measured | +|---|---|---| +| `models/{lambda,gx10}.json` | `rel-069-state/ladder/0.69.0/.json` (the v1 ladder receipt, apr 0.69.0 (225b2a9ab)), `q4k-sweep-.jsonl` (the Q4_K sweep at the same binary), and the listing of the ladder contract's inventory dirs on each host (2026-09-21T16:20:36Z, lambda locally, gx10 over the operator-authorized SSH) | yes, by the 0.69.0 train and the sweep | +| `dogfood-receipt.json` | `rel-069-state/preflight-wt/.dogfood/receipt-20260921T121638Z.json`, byte for byte: the receipt 0.69.0's T-1 "inherited" its GO from (#3708) | yes | +| `translate.py` | the translation itself. Its docstring is the rule set | — | + +The rules `translate.py` follows, so that the translation cannot manufacture a pass: +- The **inventory** is what each host holds. A `sha256` is carried **only** where the 0.69.0 ladder hashed that exact file on that host. Everything else has no hash, and pv names it `unmeasuredModel`: the release never identified the file, so it never proved it. +- The ladder row becomes the `(run, think-off, 4k)` row, with `verdict` = its green bit and `fallback` from `backends.cuda`. `prompt_tokens` and `answer_chars` are **absent** because 0.69.0 ran golden prompts and recorded neither. +- A sweep row for a file with no measured hash is carried **unkeyed**: pv names it `unkeyedRow`, with its verdict. +- **No row** exists for chat, serve, code, think-on, or any other rung, because 0.69.0 never ran them. There is **no** kernel-diff or tokenizer-parity receipt, because neither existed. + +## What the shape says + +``` +pv lint contracts --gate shapes --shape release-readiness-v1 \ + --release-version 0.69.0 --release-commit 225b2a9abbd171b3249794722f8da5fc8027e565 \ + --receipts evidence/release/proof-0.69.0/models \ + --kernel-receipts --tokenizer-receipts \ + --dogfood-receipt evidence/release/proof-0.69.0/dogfood-receipt.json +→ exit 1, verdict Fail, 795 findings; cells 768 (16 with a row), 16 models, 8 tokenizer cells +``` + +| leak (#3715 done_when 5) | how the shape names it | +|---|---| +| **qwen3-8b on lambda** | `release-cell/0.69.0/lambda/Qwen3-8B-Q4_K_M.gguf/run/think-off/4k`: `verdict: "fail"`. It is the only ladder model with a failing verdict; the same file on gx10 does not fail | +| **qwen2.5-coder on lambda** (0.5b, 7b) | `release-host/0.69.0/lambda`: `unmeasuredModel` for both files, and `unkeyedRow … verdict=fail` for the sweep rows (GPU gibberish, and a fallback to CPU with rc 14) | +| **qwen3moe on both hosts** | `unmeasuredModel` on lambda (`Qwen3-30B-A3B-Instruct-2507`, `Qwen3-Coder-30B-A3B-Instruct`, the first with `unkeyedRow … verdict=fail`: no gates emitted) and on gx10 (`Qwen3-Coder-30B-A3B-Instruct`, which the gx10 sweep never reached) | +| **chat / serve / code absent** | 752 of 768 cells have no row (`minCount`): for each of the 16 (host, model) pairs, 47 of 48 cells are missing, meaning everything except the one golden `run` | +| **the inherited dogfood** (#3708) | `release-subject/0.69.0`: the receipt says `verdict: "NO-GO"` for commit `5f9bd9344` at version `0.68.2`. It is not GO and not fresh. The autopilot's inherited "GO" was derived from it by excluding its version row | +| no kernel differential, no tokenizer parity | both hosts `kernelReceipt minCount 1`, every model's `release-tokenizer/…` cell missing | + +Every ladder model's one golden row also fails `contextMet` and `answered`, because 0.69.0 never sized a +prompt or measured an answer. That is the "overdo it" bar (#3710) applied to a release that predates it, and it +is honest: nothing measured at 4k tokens was ever recorded. + +**What is NOT shown here:** that 0.69.1 passes. That is the other half of done_when 5 ("on 0.69.1 it passes with +N/N cells named"). It needs the producer (#3712's cells[], inventory arithmetic, kernel and tokenizer receipts) +and the fixes to land. The gate will report it at T-1 through `release.cell_names`. diff --git a/evidence/release/proof-0.69.0/dogfood-receipt.json b/evidence/release/proof-0.69.0/dogfood-receipt.json new file mode 100644 index 0000000000..1491f9a4a1 --- /dev/null +++ b/evidence/release/proof-0.69.0/dogfood-receipt.json @@ -0,0 +1,239 @@ +{ + "crate": "aprender", + "version": "0.68.2", + "timestamp": "20260921T121638Z", + "commit": "5f9bd9344f07f6c6388f03f04fa61e0d93d3b411", + "gates": [ + { + "gate": "git-clean", + "result": "PASS", + "note": "working tree clean" + }, + { + "gate": "declared:check_multiplatform_dogfood", + "result": "DEFER", + "note": "scripts/check_multiplatform_dogfood.sh: 0.68.2 is not on crates.io yet, so no host can have run `cargo install aprender --version 0.68.2`; owed by the post-publish dogfood as evidence/dogfood/0.68.2/{" + }, + { + "gate": "declared:check_model_ladder", + "result": "PASS", + "note": "scripts/check_model_ladder.sh exit=0" + }, + { + "gate": "declared:check_verifier_pinning", + "result": "PASS", + "note": "scripts/check_verifier_pinning.sh exit=0" + }, + { + "gate": "declared:check_dogfood_shim", + "result": "PASS", + "note": "scripts/check_dogfood_shim.sh exit=0" + }, + { + "gate": "declared:check_bench_receipt", + "result": "PASS", + "note": "scripts/check_bench_receipt.sh exit=0" + }, + { + "gate": "declared:check_parity_receipt", + "result": "PASS", + "note": "scripts/check_parity_receipt.sh exit=0" + }, + { + "gate": "declared:check_no_claim_literals", + "result": "PASS", + "note": "scripts/check_no_claim_literals.sh exit=0" + }, + { + "gate": "declared:check_perf041_marker", + "result": "PASS", + "note": "scripts/check_perf041_marker.sh exit=0" + }, + { + "gate": "dogfood-gates", + "result": "PASS", + "note": "8 declared gate(s) discovered and all green" + }, + { + "gate": "version-unpublished", + "result": "FAIL", + "note": "aprender 0.68.2 is ALREADY in the crates.io index \u2014 bump the version" + }, + { + "gate": "changelog", + "result": "PASS", + "note": "CHANGELOG has 0.68.2" + }, + { + "gate": "feature-scope", + "result": "INFO", + "note": "clippy/test run with: --features cli" + }, + { + "gate": "fmt", + "result": "PASS", + "note": "Warning: can't set `wrap_comments = true`, unstable features are only available in nightly channel." + }, + { + "gate": "clippy", + "result": "PASS", + "note": " Compiling thiserror-impl v2.0.18" + }, + { + "gate": "test", + "result": "PASS", + "note": " Compiling thiserror-impl v2.0.18" + }, + { + "gate": "coverage", + "result": "PASS", + "note": "warning: aprender-compute@0.68.2: provable-contracts binding.yaml not found at /mnt/nvme-raid0/agent-wt/rel-069-state/pr" + }, + { + "gate": "security", + "result": "PASS", + "note": "advisories ok" + }, + { + "gate": "security-2nd-source", + "result": "PASS", + "note": "GitHub Advisory DB (independent of RustSec): 0 open Dependabot alerts" + }, + { + "gate": "pmat-pin", + "result": "PASS", + "note": "fleet pmat is the correct verifier for a non-pmat crate (exported to declared gates)" + }, + { + "gate": "release-binary", + "result": "PASS", + "note": "built: /mnt/nvme-raid0/agent-wt/rel-069/target/release/apr" + }, + { + "gate": "deterministic-tools", + "result": "PASS", + "note": "pv 0.68.2, bashrs 7.4.1, pmat 3.41.1, probador 1.0.3" + }, + { + "gate": "pv-contracts", + "result": "PASS", + "note": "1878 contract(s) valid (positive control fired)" + }, + { + "gate": "pv-lint", + "result": "PASS", + "note": "Summary: 0 errors, 11531 warnings, 0 suppressed, 10054 new" + }, + { + "gate": "pv-bindings", + "result": "REPORT", + "note": "aprender: 58/60 binding functions verified in source; 2 ghost(s) \u2014 NOT gating: pv 0.49.0 misses pub(super) fn and is module-blind (paiml/provable-contracts, re-arm when fixed). Triage each ghost by ha" + }, + { + "gate": "bashrs", + "result": "PASS", + "note": "361 file(s) linted (receipt confirmed), 0 SEC/DET/IDEM errors (1 SC10xx suppressed \u2014 bashrs#226; 11 other)" + }, + { + "gate": "renacer", + "result": "SKIP", + "note": "no renacer.toml \u2014 golden tracing not configured for this crate" + }, + { + "gate": "pmat-verify", + "result": "PASS", + "note": "}" + }, + { + "gate": "pmat-comply", + "result": "PASS", + "note": "CB-200 at or under its recorded baseline \u2014 debt held flat, NOT a clean tree; 16 other fail(s), 91 skip(s) of which 3 are Error-severity (#1008). Run `pmat comply check` (without --failures-only) for t" + }, + { + "gate": "reachability", + "result": "WARN", + "note": "unreachable files/tests: 984 13213 (a file the build never compiles cannot be tested)" + }, + { + "gate": "contracts-exit-integrity", + "result": "PASS", + "note": "`contracts:` recipe propagates failure (no bare for-loop, no `|| true`)" + }, + { + "gate": "contracts", + "result": "PASS", + "note": "lint-provenance: self-test ok (markdown + yaml, marked passes, unmarked fails, exempt keys pass, count non-zero)" + }, + { + "gate": "publish-dry-run", + "result": "DEFER", + "note": "a workspace root cannot dry-run before its members are on the registry; discharged by the cascade's own per-tier publish and the post-publish dogfood" + }, + { + "gate": "release-assets", + "result": "SKIP", + "note": "assets exist only after the tag; measured by the post-publish dogfood (DOGFOOD_PHASE=post-publish), not by this phase" + }, + { + "gate": "cli-surface", + "result": "PASS", + "note": "111 advertised subcommand(s) answer --help" + }, + { + "gate": "transport-decl", + "result": "PASS", + "note": "declared: cli http mcp " + }, + { + "gate": "transport-absence", + "result": "PASS", + "note": "no undeclared mcp/http surface advertised by the binary" + }, + { + "gate": "interface-parity", + "result": "PASS", + "note": "3 transport(s), 4 e2e test(s) against the spawned release binary" + }, + { + "gate": "transport-invariance", + "result": "SKIP", + "note": "no [package.metadata.unified_surface] \u2014 declare list/cli/http/probe to enable the simultaneous cross-transport check" + }, + { + "gate": "surface-derivation", + "result": "REPORT", + "note": "no manifest declared in [package.metadata.transports] \u2014 the interface may be hand-rolled per transport; nothing here can tell single-sourcing from three lists that agree (declare manifest = { path, re" + }, + { + "gate": "probador-suite", + "result": "PASS", + "note": "PASSED 4 tests in 73.88s (4 passed, 0 failed, 0 skipped)" + }, + { + "gate": "dogfood-use", + "result": "PASS", + "note": " 7 pass / 0 fail / 0 skip" + }, + { + "gate": "model-parity-falsifier", + "result": "PASS", + "note": "the must-RED twin turns C14 RED (tests/fixtures/parity/defective/one-position-at-0.5.json)" + }, + { + "gate": "model-parity", + "result": "REPORT", + "note": "C14 UNMEASURED on noah-Lambda-Vector: C14: measured=5 rc=1 \u2014 proven on lambda and gx10 by make fleet-verify ROW=release, never here" + }, + { + "gate": "clean-room", + "result": "MANUAL", + "note": "run `make -C ../infra/machines/clean-room clean-room-aprender` (MANDATORY release gate)" + } + ], + "phase": "pre-publish", + "deferred": [ + "declared:check_multiplatform_dogfood", + "publish-dry-run" + ], + "verdict": "NO-GO" +} diff --git a/evidence/release/proof-0.69.0/gate-summary.json b/evidence/release/proof-0.69.0/gate-summary.json new file mode 100644 index 0000000000..6d597c6ae2 --- /dev/null +++ b/evidence/release/proof-0.69.0/gate-summary.json @@ -0,0 +1,180 @@ +{ + "_doc": "Summary of `pv lint contracts --gate shapes --shape release-readiness-v1 \u2026` over this directory (the command is in README.md; re-run it for the full 795-finding report). Per-cell findings are aggregated per host/model; every other finding is verbatim.", + "exit": 1, + "verdict": "Fail", + "findings": 795, + "release": { + "required_hosts": 2, + "model_receipts": 2, + "kernel_receipts": 0, + "models": 16, + "cells": 768, + "cells_with_row": 16, + "kernel_cells": 0, + "tokenizer_cells": 8, + "refusal_cells": 0, + "rung_coverage": 48, + "context_rungs": 5, + "orphan_rows": 10 + }, + "cells_by_host_model": { + "gx10/Qwen3-1.7B-Q4_K_M.gguf": { + "missing": 47, + "failing": 1, + "failing_on": [ + "answered=false", + "contextMet=false" + ] + }, + "gx10/Qwen3-8B-Q4_K_M.gguf": { + "missing": 47, + "failing": 1, + "failing_on": [ + "answered=false", + "contextMet=false" + ] + }, + "gx10/Qwen3.5-0.8B-Q4_K_M.gguf": { + "missing": 47, + "failing": 1, + "failing_on": [ + "answered=false", + "contextMet=false" + ] + }, + "gx10/Qwen3.5-27B-Q4_K_M.gguf": { + "missing": 47, + "failing": 1, + "failing_on": [ + "answered=false", + "contextMet=false" + ] + }, + "gx10/Qwen3.5-2B-Q4_K_M.gguf": { + "missing": 47, + "failing": 1, + "failing_on": [ + "answered=false", + "contextMet=false" + ] + }, + "gx10/Qwen3.5-4B-Q4_K_M.gguf": { + "missing": 47, + "failing": 1, + "failing_on": [ + "answered=false", + "contextMet=false" + ] + }, + "gx10/Qwen3.5-9B-Q4_K_M.gguf": { + "missing": 47, + "failing": 1, + "failing_on": [ + "answered=false", + "contextMet=false" + ] + }, + "gx10/qwen2.5-1.5b-instruct-q4_k_m.gguf": { + "missing": 47, + "failing": 1, + "failing_on": [ + "answered=false", + "contextMet=false" + ] + }, + "lambda/Qwen3-1.7B-Q4_K_M.gguf": { + "missing": 47, + "failing": 1, + "failing_on": [ + "answered=false", + "contextMet=false" + ] + }, + "lambda/Qwen3-8B-Q4_K_M.gguf": { + "missing": 47, + "failing": 1, + "failing_on": [ + "answered=false", + "contextMet=false", + "verdict=fail" + ] + }, + "lambda/Qwen3.5-0.8B-Q4_K_M.gguf": { + "missing": 47, + "failing": 1, + "failing_on": [ + "answered=false", + "contextMet=false" + ] + }, + "lambda/Qwen3.5-27B-Q4_K_M.gguf": { + "missing": 47, + "failing": 1, + "failing_on": [ + "answered=false", + "contextMet=false" + ] + }, + "lambda/Qwen3.5-2B-Q4_K_M.gguf": { + "missing": 47, + "failing": 1, + "failing_on": [ + "answered=false", + "contextMet=false" + ] + }, + "lambda/Qwen3.5-4B-Q4_K_M.gguf": { + "missing": 47, + "failing": 1, + "failing_on": [ + "answered=false", + "contextMet=false" + ] + }, + "lambda/Qwen3.5-9B-Q4_K_M.gguf": { + "missing": 47, + "failing": 1, + "failing_on": [ + "answered=false", + "contextMet=false" + ] + }, + "lambda/qwen2.5-1.5b-instruct-q4_k_m.gguf": { + "missing": 47, + "failing": 1, + "failing_on": [ + "answered=false", + "contextMet=false" + ] + } + }, + "other_findings": [ + "ont:release-subject/0.69.0 violates shape `release-readiness-v1.release` (node): value https://ont.paiml.dev/v1alpha1/release-dogfood/0.69.0 fails the nested shape: ont:release/verdict: \"NO-GO\" is not one of [\"GO\"]; ont:release/fresh: \"false\"^^xsd:boolean is not one of [\"true\"^^xsd:boolean]", + "ont:release-host/0.69.0/gx10 violates shape `release-readiness-v1.host` (minCount): has 0 value(s) of ont:release/dispatchKernel, minCount is 1", + "ont:release-host/0.69.0/gx10 violates shape `release-readiness-v1.host` (minCount): has 0 value(s) of ont:release/kernelReceipt, minCount is 1", + "ont:release-host/0.69.0/gx10 violates shape `release-readiness-v1.host` (maxCount): has 8 value(s) of ont:release/modelWithoutDispatch, maxCount is 0: Qwen3-1.7B-Q4_K_M.gguf, Qwen3-8B-Q4_K_M.gguf, Qwen3.5-0.8B-Q4_K_M.gguf, Qwen3.5-27B-Q4_K_M.gguf, Qwen3.5-2B-Q4_K_M.gguf", + "ont:release-host/0.69.0/gx10 violates shape `release-readiness-v1.host` (maxCount): has 5 value(s) of ont:release/unkeyedRow, maxCount is 0: evidence/release/proof-0.69.0/models/gx10.json: qwen2.5-coder-1.5b-instruct-q4_k_m.apr run think-off 4k verdict=pass, evidence/release/proof-0.69.0/models/gx10.json: qwen2.5-coder-1.5b-instruct-q4_k_m.gguf run think-off 4k verdict=pass, evidence/release/proof-0.69.0/models/gx10.json: qwen2.5-coder-1.5b-instruct-q4k.apr run think-off 4k verdict=pass, evidence/release/proof-0.69.0/models/gx10.json: qwen2.5-coder-32b-instruct-q4_k_m.gguf run think-off 4k verdict=pass, evidence/release/proof-0.69.0/models/gx10.json: qwen2.5-coder-7b-instruct-q4_k_m.gguf run think-off 4k verdict=pass", + "ont:release-host/0.69.0/gx10 violates shape `release-readiness-v1.host` (maxCount): has 6 value(s) of ont:release/unmeasuredModel, maxCount is 0: evidence/release/proof-0.69.0/models/gx10.json: Qwen3-Coder-30B-A3B-Instruct-Q4_K_M.gguf, evidence/release/proof-0.69.0/models/gx10.json: qwen2.5-coder-1.5b-instruct-q4_k_m.apr, evidence/release/proof-0.69.0/models/gx10.json: qwen2.5-coder-1.5b-instruct-q4_k_m.gguf, evidence/release/proof-0.69.0/models/gx10.json: qwen2.5-coder-1.5b-instruct-q4k.apr, evidence/release/proof-0.69.0/models/gx10.json: qwen2.5-coder-32b-instruct-q4_k_m.gguf", + "ont:release-host/0.69.0/lambda violates shape `release-readiness-v1.host` (minCount): has 0 value(s) of ont:release/dispatchKernel, minCount is 1", + "ont:release-host/0.69.0/lambda violates shape `release-readiness-v1.host` (minCount): has 0 value(s) of ont:release/kernelReceipt, minCount is 1", + "ont:release-host/0.69.0/lambda violates shape `release-readiness-v1.host` (maxCount): has 8 value(s) of ont:release/modelWithoutDispatch, maxCount is 0: Qwen3-1.7B-Q4_K_M.gguf, Qwen3-8B-Q4_K_M.gguf, Qwen3.5-0.8B-Q4_K_M.gguf, Qwen3.5-27B-Q4_K_M.gguf, Qwen3.5-2B-Q4_K_M.gguf", + "ont:release-host/0.69.0/lambda violates shape `release-readiness-v1.host` (maxCount): has 5 value(s) of ont:release/unkeyedRow, maxCount is 0: evidence/release/proof-0.69.0/models/lambda.json: Qwen3-30B-A3B-Instruct-2507-Q4_K_M.gguf run think-off 4k verdict=fail, evidence/release/proof-0.69.0/models/lambda.json: qwen2.5-coder-0.5b-instruct-q4_k_m.gguf run think-off 4k verdict=fail, evidence/release/proof-0.69.0/models/lambda.json: qwen2.5-coder-1.5b-instruct-q4k-v2.apr run think-off 4k verdict=pass, evidence/release/proof-0.69.0/models/lambda.json: qwen2.5-coder-1.5b-instruct-q4k.apr run think-off 4k verdict=pass, evidence/release/proof-0.69.0/models/lambda.json: qwen2.5-coder-7b-instruct-q4_k_m.gguf run think-off 4k verdict=fail", + "ont:release-host/0.69.0/lambda violates shape `release-readiness-v1.host` (maxCount): has 10 value(s) of ont:release/unmeasuredModel, maxCount is 0: evidence/release/proof-0.69.0/models/lambda.json: Qwen3-30B-A3B-Instruct-2507-Q4_K_M.gguf, evidence/release/proof-0.69.0/models/lambda.json: Qwen3-Coder-30B-A3B-Instruct-Q4_K_M.gguf, evidence/release/proof-0.69.0/models/lambda.json: Qwen3.5-4B-UD-Q4_K_XL.gguf, evidence/release/proof-0.69.0/models/lambda.json: qwen2.5-coder-0.5b-instruct-q4_k_m.gguf, evidence/release/proof-0.69.0/models/lambda.json: qwen2.5-coder-1.5b-instruct-q4_k_m.gguf", + "ont:model/00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4 (rung qwen35-4b-q4km) violates shape `release-readiness-v1.model` (minCount): has 0 value(s) of ont:release/contextLength, minCount is 1", + "ont:model/03b74727a860a56338e042c4420bb3f04b2fec5734175f4cb9fa853daf52b7e8 (rung qwen35-9b-q4km) violates shape `release-readiness-v1.model` (minCount): has 0 value(s) of ont:release/contextLength, minCount is 1", + "ont:model/6a1a2eb6d15622bf3c96857206351ba97e1af16c30d7a74ee38970e434e9407e (rung qwen2-1.5b-q4km) violates shape `release-readiness-v1.model` (minCount): has 0 value(s) of ont:release/contextLength, minCount is 1", + "ont:model/84b5f7f112156d63836a01a69dc3f11a6ba63b10a23b8ca7a7efaf52d5a2d806 (rung qwen35-27b-q4km) violates shape `release-readiness-v1.model` (minCount): has 0 value(s) of ont:release/contextLength, minCount is 1", + "ont:model/aaf42c8b7c3cab2bf3d69c355048d4a0ee9973d48f16c731c0520ee914699223 (rung qwen35-2b-q4km) violates shape `release-readiness-v1.model` (minCount): has 0 value(s) of ont:release/contextLength, minCount is 1", + "ont:model/b139949c5bd74937ad8ed8c8cf3d9ffb1e99c866c823204dc42c0d91fa181897 (rung qwen3-1.7b-q4km) violates shape `release-readiness-v1.model` (minCount): has 0 value(s) of ont:release/contextLength, minCount is 1", + "ont:model/bd258782e35f7f458f8aced1adc053e6e92e89bc735ba3be89d38a06121dc517 (rung qwen35-0.8b-q4km) violates shape `release-readiness-v1.model` (minCount): has 0 value(s) of ont:release/contextLength, minCount is 1", + "ont:model/d98cdcbd03e17ce47681435b5150e34c1417f50b5c0019dd560e4882c5745785 (rung qwen3-8b-q4km) violates shape `release-readiness-v1.model` (minCount): has 0 value(s) of ont:release/contextLength, minCount is 1", + "ont:release-tokenizer/0.69.0/Qwen3-1.7B-Q4_K_M.gguf violates shape `release-readiness-v1.tokenizer` (minCount): has 0 value(s) of ont:release/row, minCount is 1", + "ont:release-tokenizer/0.69.0/Qwen3-8B-Q4_K_M.gguf violates shape `release-readiness-v1.tokenizer` (minCount): has 0 value(s) of ont:release/row, minCount is 1", + "ont:release-tokenizer/0.69.0/Qwen3.5-0.8B-Q4_K_M.gguf violates shape `release-readiness-v1.tokenizer` (minCount): has 0 value(s) of ont:release/row, minCount is 1", + "ont:release-tokenizer/0.69.0/Qwen3.5-27B-Q4_K_M.gguf violates shape `release-readiness-v1.tokenizer` (minCount): has 0 value(s) of ont:release/row, minCount is 1", + "ont:release-tokenizer/0.69.0/Qwen3.5-2B-Q4_K_M.gguf violates shape `release-readiness-v1.tokenizer` (minCount): has 0 value(s) of ont:release/row, minCount is 1", + "ont:release-tokenizer/0.69.0/Qwen3.5-4B-Q4_K_M.gguf violates shape `release-readiness-v1.tokenizer` (minCount): has 0 value(s) of ont:release/row, minCount is 1", + "ont:release-tokenizer/0.69.0/Qwen3.5-9B-Q4_K_M.gguf violates shape `release-readiness-v1.tokenizer` (minCount): has 0 value(s) of ont:release/row, minCount is 1", + "ont:release-tokenizer/0.69.0/qwen2.5-1.5b-instruct-q4_k_m.gguf violates shape `release-readiness-v1.tokenizer` (minCount): has 0 value(s) of ont:release/row, minCount is 1" + ] +} \ No newline at end of file diff --git a/evidence/release/proof-0.69.0/models/gx10.json b/evidence/release/proof-0.69.0/models/gx10.json new file mode 100644 index 0000000000..f35409e9a5 --- /dev/null +++ b/evidence/release/proof-0.69.0/models/gx10.json @@ -0,0 +1,243 @@ +{ + "schema": "apr-model-ladder-receipt/v2", + "host": "gx10", + "version": "0.69.0", + "sha": "225b2a9ab", + "apr_sha": "225b2a9abbd171b3249794722f8da5fc8027e565", + "apr_version": "apr 0.69.0 (225b2a9ab)", + "cc": "12.1", + "gpu": "NVIDIA GB10", + "translated_from": [ + "rel-069-state/ladder/0.69.0/gx10.json", + "q4k-sweep-gx10.jsonl", + "inventory listing of ~/models, ~/.apr/models, ~/.cache/apr/models (2026-09-21T16:20:36Z)" + ], + "inventory": [ + { + "file": "Qwen3-1.7B-Q4_K_M.gguf", + "bytes": 1107409472, + "sha256": "b139949c5bd74937ad8ed8c8cf3d9ffb1e99c866c823204dc42c0d91fa181897", + "arch": "qwen3" + }, + { + "file": "Qwen3-8B-Q4_K_M.gguf", + "bytes": 5027783488, + "sha256": "d98cdcbd03e17ce47681435b5150e34c1417f50b5c0019dd560e4882c5745785", + "arch": "qwen3" + }, + { + "file": "Qwen3-Coder-30B-A3B-Instruct-Q4_K_M.gguf", + "bytes": 18556689568 + }, + { + "file": "Qwen3.5-0.8B-Q4_K_M.gguf", + "bytes": 532517120, + "sha256": "bd258782e35f7f458f8aced1adc053e6e92e89bc735ba3be89d38a06121dc517", + "arch": "qwen35" + }, + { + "file": "Qwen3.5-27B-Q4_K_M.gguf", + "bytes": 16740812704, + "sha256": "84b5f7f112156d63836a01a69dc3f11a6ba63b10a23b8ca7a7efaf52d5a2d806", + "arch": "qwen35" + }, + { + "file": "Qwen3.5-2B-Q4_K_M.gguf", + "bytes": 1280835840, + "sha256": "aaf42c8b7c3cab2bf3d69c355048d4a0ee9973d48f16c731c0520ee914699223", + "arch": "qwen35" + }, + { + "file": "Qwen3.5-4B-Q4_K_M.gguf", + "bytes": 2740937888, + "sha256": "00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4", + "arch": "qwen35" + }, + { + "file": "Qwen3.5-9B-Q4_K_M.gguf", + "bytes": 5680522464, + "sha256": "03b74727a860a56338e042c4420bb3f04b2fec5734175f4cb9fa853daf52b7e8", + "arch": "qwen35" + }, + { + "file": "qwen2.5-1.5b-instruct-q4_k_m.gguf", + "bytes": 1117320736, + "sha256": "6a1a2eb6d15622bf3c96857206351ba97e1af16c30d7a74ee38970e434e9407e", + "arch": "qwen2" + }, + { + "file": "qwen2.5-coder-1.5b-instruct-q4_k_m.apr", + "bytes": 1115528900 + }, + { + "file": "qwen2.5-coder-1.5b-instruct-q4_k_m.gguf", + "bytes": 1117320768 + }, + { + "file": "qwen2.5-coder-1.5b-instruct-q4k.apr", + "bytes": 1115198720 + }, + { + "file": "qwen2.5-coder-32b-instruct-q4_k_m.gguf", + "bytes": 19851335872 + }, + { + "file": "qwen2.5-coder-7b-instruct-q4_k_m.gguf", + "bytes": 4683073536 + } + ], + "cells": [ + { + "sha256": "b139949c5bd74937ad8ed8c8cf3d9ffb1e99c866c823204dc42c0d91fa181897", + "file": "Qwen3-1.7B-Q4_K_M.gguf", + "verb": "run", + "thinking": "off", + "context": "4k", + "verdict": "pass", + "backend": "cuda", + "fallback": false, + "rc": 0, + "reason": "0.69.0 ladder row (golden prompts; prompt_tokens and answer_chars not recorded)" + }, + { + "sha256": "d98cdcbd03e17ce47681435b5150e34c1417f50b5c0019dd560e4882c5745785", + "file": "Qwen3-8B-Q4_K_M.gguf", + "verb": "run", + "thinking": "off", + "context": "4k", + "verdict": "pass", + "backend": "cuda", + "fallback": false, + "rc": 0, + "reason": "0.69.0 ladder row (golden prompts; prompt_tokens and answer_chars not recorded)" + }, + { + "sha256": "bd258782e35f7f458f8aced1adc053e6e92e89bc735ba3be89d38a06121dc517", + "file": "Qwen3.5-0.8B-Q4_K_M.gguf", + "verb": "run", + "thinking": "off", + "context": "4k", + "verdict": "pass", + "backend": "cuda", + "fallback": false, + "rc": 0, + "reason": "0.69.0 ladder row (golden prompts; prompt_tokens and answer_chars not recorded)" + }, + { + "sha256": "84b5f7f112156d63836a01a69dc3f11a6ba63b10a23b8ca7a7efaf52d5a2d806", + "file": "Qwen3.5-27B-Q4_K_M.gguf", + "verb": "run", + "thinking": "off", + "context": "4k", + "verdict": "pass", + "backend": "cuda", + "fallback": false, + "rc": 0, + "reason": "0.69.0 ladder row (golden prompts; prompt_tokens and answer_chars not recorded)" + }, + { + "sha256": "aaf42c8b7c3cab2bf3d69c355048d4a0ee9973d48f16c731c0520ee914699223", + "file": "Qwen3.5-2B-Q4_K_M.gguf", + "verb": "run", + "thinking": "off", + "context": "4k", + "verdict": "pass", + "backend": "cuda", + "fallback": false, + "rc": 0, + "reason": "0.69.0 ladder row (golden prompts; prompt_tokens and answer_chars not recorded)" + }, + { + "sha256": "00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4", + "file": "Qwen3.5-4B-Q4_K_M.gguf", + "verb": "run", + "thinking": "off", + "context": "4k", + "verdict": "pass", + "backend": "cuda", + "fallback": false, + "rc": 0, + "reason": "0.69.0 ladder row (golden prompts; prompt_tokens and answer_chars not recorded)" + }, + { + "sha256": "03b74727a860a56338e042c4420bb3f04b2fec5734175f4cb9fa853daf52b7e8", + "file": "Qwen3.5-9B-Q4_K_M.gguf", + "verb": "run", + "thinking": "off", + "context": "4k", + "verdict": "pass", + "backend": "cuda", + "fallback": false, + "rc": 0, + "reason": "0.69.0 ladder row (golden prompts; prompt_tokens and answer_chars not recorded)" + }, + { + "sha256": "6a1a2eb6d15622bf3c96857206351ba97e1af16c30d7a74ee38970e434e9407e", + "file": "qwen2.5-1.5b-instruct-q4_k_m.gguf", + "verb": "run", + "thinking": "off", + "context": "4k", + "verdict": "pass", + "backend": "cuda", + "fallback": false, + "rc": 0, + "reason": "0.69.0 ladder row (golden prompts; prompt_tokens and answer_chars not recorded)" + }, + { + "file": "qwen2.5-coder-1.5b-instruct-q4k.apr", + "verb": "run", + "thinking": "off", + "context": "4k", + "verdict": "pass", + "backend": "cuda", + "fallback": false, + "rc": 0, + "reason": "0.69.0 Q4_K sweep row (no sha256 recorded): 3 golden test cases passed" + }, + { + "file": "qwen2.5-coder-1.5b-instruct-q4_k_m.apr", + "verb": "run", + "thinking": "off", + "context": "4k", + "verdict": "pass", + "backend": "cuda", + "fallback": false, + "rc": 0, + "reason": "0.69.0 Q4_K sweep row (no sha256 recorded): 3 golden test cases passed" + }, + { + "file": "qwen2.5-coder-1.5b-instruct-q4_k_m.gguf", + "verb": "run", + "thinking": "off", + "context": "4k", + "verdict": "pass", + "backend": "cuda", + "fallback": false, + "rc": 0, + "reason": "0.69.0 Q4_K sweep row (no sha256 recorded): 3 golden test cases passed" + }, + { + "file": "qwen2.5-coder-32b-instruct-q4_k_m.gguf", + "verb": "run", + "thinking": "off", + "context": "4k", + "verdict": "pass", + "backend": "cuda", + "fallback": false, + "rc": 0, + "reason": "0.69.0 Q4_K sweep row (no sha256 recorded): 3 golden test cases passed" + }, + { + "file": "qwen2.5-coder-7b-instruct-q4_k_m.gguf", + "verb": "run", + "thinking": "off", + "context": "4k", + "verdict": "pass", + "backend": "cuda", + "fallback": false, + "rc": 0, + "reason": "0.69.0 Q4_K sweep row (no sha256 recorded): 3 golden test cases passed" + } + ], + "rungs": [] +} \ No newline at end of file diff --git a/evidence/release/proof-0.69.0/models/lambda.json b/evidence/release/proof-0.69.0/models/lambda.json new file mode 100644 index 0000000000..e2680549b0 --- /dev/null +++ b/evidence/release/proof-0.69.0/models/lambda.json @@ -0,0 +1,259 @@ +{ + "schema": "apr-model-ladder-receipt/v2", + "host": "lambda", + "version": "0.69.0", + "sha": "225b2a9ab", + "apr_sha": "225b2a9abbd171b3249794722f8da5fc8027e565", + "apr_version": "apr 0.69.0 (225b2a9ab)", + "cc": "8.9", + "gpu": "NVIDIA GeForce RTX 4090", + "translated_from": [ + "rel-069-state/ladder/0.69.0/lambda.json", + "q4k-sweep-lambda.jsonl", + "inventory listing of ~/models, ~/.apr/models, ~/.cache/apr/models (2026-09-21T16:20:36Z)" + ], + "inventory": [ + { + "file": "Qwen3-1.7B-Q4_K_M.gguf", + "bytes": 1107409472, + "sha256": "b139949c5bd74937ad8ed8c8cf3d9ffb1e99c866c823204dc42c0d91fa181897", + "arch": "qwen3" + }, + { + "file": "Qwen3-30B-A3B-Instruct-2507-Q4_K_M.gguf", + "bytes": 18556686752 + }, + { + "file": "Qwen3-8B-Q4_K_M.gguf", + "bytes": 5027783488, + "sha256": "d98cdcbd03e17ce47681435b5150e34c1417f50b5c0019dd560e4882c5745785", + "arch": "qwen3" + }, + { + "file": "Qwen3-Coder-30B-A3B-Instruct-Q4_K_M.gguf", + "bytes": 18556689568 + }, + { + "file": "Qwen3.5-0.8B-Q4_K_M.gguf", + "bytes": 532517120, + "sha256": "bd258782e35f7f458f8aced1adc053e6e92e89bc735ba3be89d38a06121dc517", + "arch": "qwen35" + }, + { + "file": "Qwen3.5-27B-Q4_K_M.gguf", + "bytes": 16740812704, + "sha256": "84b5f7f112156d63836a01a69dc3f11a6ba63b10a23b8ca7a7efaf52d5a2d806", + "arch": "qwen35" + }, + { + "file": "Qwen3.5-2B-Q4_K_M.gguf", + "bytes": 1280835840, + "sha256": "aaf42c8b7c3cab2bf3d69c355048d4a0ee9973d48f16c731c0520ee914699223", + "arch": "qwen35" + }, + { + "file": "Qwen3.5-4B-Q4_K_M.gguf", + "bytes": 2740937888, + "sha256": "00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4", + "arch": "qwen35" + }, + { + "file": "Qwen3.5-4B-UD-Q4_K_XL.gguf", + "bytes": 2912109728 + }, + { + "file": "Qwen3.5-9B-Q4_K_M.gguf", + "bytes": 5680522464, + "sha256": "03b74727a860a56338e042c4420bb3f04b2fec5734175f4cb9fa853daf52b7e8", + "arch": "qwen35" + }, + { + "file": "qwen2.5-1.5b-instruct-q4_k_m.gguf", + "bytes": 1117320736, + "sha256": "6a1a2eb6d15622bf3c96857206351ba97e1af16c30d7a74ee38970e434e9407e", + "arch": "qwen2" + }, + { + "file": "qwen2.5-coder-0.5b-instruct-q4_k_m.gguf", + "bytes": 491400064 + }, + { + "file": "qwen2.5-coder-1.5b-instruct-q4_k_m.gguf", + "bytes": 1117320768 + }, + { + "file": "qwen2.5-coder-1.5b-instruct-q4k-v2.apr", + "bytes": 1115198720 + }, + { + "file": "qwen2.5-coder-1.5b-instruct-q4k.apr", + "bytes": 1115198720 + }, + { + "file": "qwen2.5-coder-7b-instruct-q4_k_m.gguf", + "bytes": 4683073536 + }, + { + "file": "qwen2.5-coder-1.5b-q4k.apr", + "bytes": 1115528900 + }, + { + "file": "tinyllama-1.1b-chat-v1.0.Q4_K_M.gguf", + "bytes": 668788096 + } + ], + "cells": [ + { + "sha256": "b139949c5bd74937ad8ed8c8cf3d9ffb1e99c866c823204dc42c0d91fa181897", + "file": "Qwen3-1.7B-Q4_K_M.gguf", + "verb": "run", + "thinking": "off", + "context": "4k", + "verdict": "pass", + "backend": "cuda", + "fallback": false, + "rc": 0, + "reason": "0.69.0 ladder row (golden prompts; prompt_tokens and answer_chars not recorded)" + }, + { + "sha256": "d98cdcbd03e17ce47681435b5150e34c1417f50b5c0019dd560e4882c5745785", + "file": "Qwen3-8B-Q4_K_M.gguf", + "verb": "run", + "thinking": "off", + "context": "4k", + "verdict": "fail", + "backend": "cuda", + "fallback": false, + "rc": 0, + "reason": "0.69.0 ladder row (golden prompts; prompt_tokens and answer_chars not recorded): golden_output: Empty output" + }, + { + "sha256": "bd258782e35f7f458f8aced1adc053e6e92e89bc735ba3be89d38a06121dc517", + "file": "Qwen3.5-0.8B-Q4_K_M.gguf", + "verb": "run", + "thinking": "off", + "context": "4k", + "verdict": "pass", + "backend": "cuda", + "fallback": false, + "rc": 0, + "reason": "0.69.0 ladder row (golden prompts; prompt_tokens and answer_chars not recorded)" + }, + { + "sha256": "84b5f7f112156d63836a01a69dc3f11a6ba63b10a23b8ca7a7efaf52d5a2d806", + "file": "Qwen3.5-27B-Q4_K_M.gguf", + "verb": "run", + "thinking": "off", + "context": "4k", + "verdict": "pass", + "backend": "cuda", + "fallback": false, + "rc": 0, + "reason": "0.69.0 ladder row (golden prompts; prompt_tokens and answer_chars not recorded)" + }, + { + "sha256": "aaf42c8b7c3cab2bf3d69c355048d4a0ee9973d48f16c731c0520ee914699223", + "file": "Qwen3.5-2B-Q4_K_M.gguf", + "verb": "run", + "thinking": "off", + "context": "4k", + "verdict": "pass", + "backend": "cuda", + "fallback": false, + "rc": 0, + "reason": "0.69.0 ladder row (golden prompts; prompt_tokens and answer_chars not recorded)" + }, + { + "sha256": "00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4", + "file": "Qwen3.5-4B-Q4_K_M.gguf", + "verb": "run", + "thinking": "off", + "context": "4k", + "verdict": "pass", + "backend": "cuda", + "fallback": false, + "rc": 0, + "reason": "0.69.0 ladder row (golden prompts; prompt_tokens and answer_chars not recorded)" + }, + { + "sha256": "03b74727a860a56338e042c4420bb3f04b2fec5734175f4cb9fa853daf52b7e8", + "file": "Qwen3.5-9B-Q4_K_M.gguf", + "verb": "run", + "thinking": "off", + "context": "4k", + "verdict": "pass", + "backend": "cuda", + "fallback": false, + "rc": 0, + "reason": "0.69.0 ladder row (golden prompts; prompt_tokens and answer_chars not recorded)" + }, + { + "sha256": "6a1a2eb6d15622bf3c96857206351ba97e1af16c30d7a74ee38970e434e9407e", + "file": "qwen2.5-1.5b-instruct-q4_k_m.gguf", + "verb": "run", + "thinking": "off", + "context": "4k", + "verdict": "pass", + "backend": "cuda", + "fallback": false, + "rc": 0, + "reason": "0.69.0 ladder row (golden prompts; prompt_tokens and answer_chars not recorded)" + }, + { + "file": "qwen2.5-coder-0.5b-instruct-q4_k_m.gguf", + "verb": "run", + "thinking": "off", + "context": "4k", + "verdict": "fail", + "backend": "cuda", + "fallback": true, + "rc": 14, + "reason": "0.69.0 Q4_K sweep row (no sha256 recorded): GPU output failed (CPU passed): golden_output_gpu: gibberish (fragment \"```\\n\" repeats 3+ times)" + }, + { + "file": "qwen2.5-coder-1.5b-instruct-q4k.apr", + "verb": "run", + "thinking": "off", + "context": "4k", + "verdict": "pass", + "backend": "cuda", + "fallback": false, + "rc": 0, + "reason": "0.69.0 Q4_K sweep row (no sha256 recorded): 3 golden test cases passed" + }, + { + "file": "qwen2.5-coder-1.5b-instruct-q4k-v2.apr", + "verb": "run", + "thinking": "off", + "context": "4k", + "verdict": "pass", + "backend": "cuda", + "fallback": false, + "rc": 0, + "reason": "0.69.0 Q4_K sweep row (no sha256 recorded): 3 golden test cases passed" + }, + { + "file": "qwen2.5-coder-7b-instruct-q4_k_m.gguf", + "verb": "run", + "thinking": "off", + "context": "4k", + "verdict": "fail", + "backend": "cuda", + "fallback": true, + "rc": 14, + "reason": "0.69.0 Q4_K sweep row (no sha256 recorded): 3 golden test cases passed" + }, + { + "file": "Qwen3-30B-A3B-Instruct-2507-Q4_K_M.gguf", + "verb": "run", + "thinking": "off", + "context": "4k", + "verdict": "fail", + "backend": "cuda", + "fallback": false, + "rc": 14, + "reason": "0.69.0 Q4_K sweep row (no sha256 recorded): gate absent" + } + ], + "rungs": [] +} \ No newline at end of file diff --git a/evidence/release/proof-0.69.0/translate.py b/evidence/release/proof-0.69.0/translate.py new file mode 100644 index 0000000000..2cfc21ab36 --- /dev/null +++ b/evidence/release/proof-0.69.0/translate.py @@ -0,0 +1,62 @@ +"""0.69.0's MEASURED evidence, carried into apr-model-ladder-receipt/v2 without inventing a measurement. + +Inputs (all measured at the 0.69.0 release, apr 0.69.0 (225b2a9ab)): + ladder rel-069-state/ladder/0.69.0/.json (v1; sha256 per rung, measured on the host) + sweep q4k-sweep-.jsonl (every Q4_K file the sweep reached; NO sha256) + holds inventory listing of the ladder contract's inventory dirs (file + bytes; NO sha256) +Rules: + * inventory = what the host holds. sha256 ONLY where the 0.69.0 ladder measured that exact file on that host; + otherwise absent (pv names it `unmeasuredModel`: the release never identified it, so never proved it). + * arch only from the ladder rung; context_length, thinking, memory arithmetic: absent (0.69.0 measured none). + * cells: the ladder row becomes the (run, think-off, 4k) row -- verdict pass iff the row was green, fallback + from backends.cuda -- with prompt_tokens and answer_chars ABSENT, because 0.69.0 ran golden prompts and + recorded neither. A sweep row for a file with no measured sha is carried UNKEYED (pv: `unkeyedRow`, verdict). + * chat / serve / code / think-on / every other rung: no row, because 0.69.0 never ran them. +""" +import json, sys +AP, SW, SP, OUT = sys.argv[1:5] +REL = "225b2a9abbd171b3249794722f8da5fc8027e565" +for host in ["lambda", "gx10"]: + lad = json.load(open(f"{AP}/ladder/0.69.0/{host}.json")) + ladder_contract = {"qwen2-1.5b-q4km": ("qwen2.5-1.5b-instruct-q4_k_m.gguf", "qwen2"), + "qwen3-1.7b-q4km": ("Qwen3-1.7B-Q4_K_M.gguf", "qwen3"), + "qwen3-8b-q4km": ("Qwen3-8B-Q4_K_M.gguf", "qwen3"), + "qwen35-0.8b-q4km": ("Qwen3.5-0.8B-Q4_K_M.gguf", "qwen35"), + "qwen35-2b-q4km": ("Qwen3.5-2B-Q4_K_M.gguf", "qwen35"), + "qwen35-4b-q4km": ("Qwen3.5-4B-Q4_K_M.gguf", "qwen35"), + "qwen35-9b-q4km": ("Qwen3.5-9B-Q4_K_M.gguf", "qwen35"), + "qwen35-27b-q4km": ("Qwen3.5-27B-Q4_K_M.gguf", "qwen35")} + by_file = {} + for r in lad["rungs"]: + f, arch = ladder_contract[r["id"]] + by_file[f] = (r, arch) + holds = json.load(open(f"{SP}/inv-{host}.json")) + inventory, cells = [], [] + for h in holds: + item = {"file": h["file"], "bytes": h["bytes"]} + if h["file"] in by_file and by_file[h["file"]][0].get("sha256"): + r, arch = by_file[h["file"]] + item["sha256"] = r["sha256"]; item["arch"] = arch + cuda = (r.get("backends") or {}).get("cuda") or {} + cells.append({"sha256": r["sha256"], "file": h["file"], "verb": "run", "thinking": "off", "context": "4k", + "verdict": "pass" if r.get("green") else "fail", "backend": "cuda", + "fallback": bool(cuda.get("fallback")), "rc": cuda.get("rc"), + "reason": "0.69.0 ladder row (golden prompts; prompt_tokens and answer_chars not recorded)" + + ("" if r.get("green") else ": " + str((r.get("golden_output") or {}).get("message", ""))[:120])}) + inventory.append(item) + for l in open(f"{SW}/q4k-sweep-{host}.jsonl"): + s = json.loads(l) + if s["model"] in by_file: + continue # the ladder row, with its measured sha, already carries it + cells.append({"file": s["model"], "verb": "run", "thinking": "off", "context": "4k", + "verdict": "pass" if s["green"] else "fail", "backend": "cuda", + "fallback": bool(s.get("gpu_fallback")), "rc": s.get("gpu_run_rc"), + "reason": "0.69.0 Q4_K sweep row (no sha256 recorded): " + + str((s.get("golden_output") or {}).get("message", ""))[:120]}) + out = {"schema": "apr-model-ladder-receipt/v2", "host": host, "version": "0.69.0", "sha": lad["sha"], + "apr_sha": REL, "apr_version": lad.get("apr_version"), "cc": lad["cc"], "gpu": lad.get("gpu"), + "translated_from": [f"rel-069-state/ladder/0.69.0/{host}.json", f"q4k-sweep-{host}.jsonl", + "inventory listing of ~/models, ~/.apr/models, ~/.cache/apr/models (2026-09-21T16:20:36Z)"], + "inventory": inventory, "cells": cells, "rungs": []} + json.dump(out, open(f"{OUT}/models/{host}.json", "w"), indent=1) + print(host, len(inventory), "held,", sum(1 for i in inventory if "sha256" in i), "hashed,", len(cells), "rows") diff --git a/tests/fixtures/ont/release-green/contracts/ladder.yaml b/tests/fixtures/ont/release-green/contracts/ladder.yaml new file mode 100644 index 0000000000..1d6ed226f1 --- /dev/null +++ b/tests/fixtures/ont/release-green/contracts/ladder.yaml @@ -0,0 +1,20 @@ +metadata: + version: "1.0.0" + kind: pattern + created: '2026-09-21' + last_modified: '2026-09-21' + author: fixture + description: fixture ladder — two required hosts, no rungs (the universe is the inventory), long rungs for qwen35 +name: ladder +version: "1.0.0" +scope: fixture +status: active +entity: + type: gguf +ladder: + hosts: + - {id: lambda, cc: sm_89, required: true} + - {id: gx10, cc: sm_121, required: true} + cells: + long_rungs_for: {families: [qwen35], representatives: {}} + rungs: [] diff --git a/tests/fixtures/ont/release-green/dogfood-receipt.json b/tests/fixtures/ont/release-green/dogfood-receipt.json new file mode 100644 index 0000000000..d3ebfa779e --- /dev/null +++ b/tests/fixtures/ont/release-green/dogfood-receipt.json @@ -0,0 +1,10 @@ +{ + "crate": "aprender", + "version": "0.69.1", + "timestamp": "2026-09-21T00:00:00Z", + "commit": "1111111111111111111111111111111111111111", + "gates": [], + "phase": "pre-publish", + "deferred": [], + "verdict": "GO" +} \ No newline at end of file diff --git a/tests/fixtures/ont/release-green/evidence/dogfood/kernels/0.69.1/gx10.json b/tests/fixtures/ont/release-green/evidence/dogfood/kernels/0.69.1/gx10.json new file mode 100644 index 0000000000..1d5b1c458f --- /dev/null +++ b/tests/fixtures/ont/release-green/evidence/dogfood/kernels/0.69.1/gx10.json @@ -0,0 +1,46 @@ +{ + "schema": "apr-kernel-diff-receipt/v1", + "host": "gx10", + "version": "0.69.1", + "apr_sha": "1111111111111111111111111111111111111111", + "cc": "12.1", + "sm": "sm_121", + "dispatch": [ + { + "sha256": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "file": "tiny-qwen35.gguf", + "kernels": [ + { + "kernel": "q4k_gemv", + "quant": "q4_k" + }, + { + "kernel": "rmsnorm", + "quant": "f32" + } + ] + } + ], + "rows": [ + { + "kernel": "q4k_gemv", + "quant": "q4_k", + "reference": "cpu", + "max_err": 0.001, + "bound": 0.004, + "bound_source": "fixture://q4k-pair", + "verdict": "pass", + "reason": "" + }, + { + "kernel": "rmsnorm", + "quant": "f32", + "reference": "cpu", + "max_err": 1e-06, + "bound": 1e-05, + "bound_source": "fixture://rmsnorm-pair", + "verdict": "pass", + "reason": "" + } + ] +} \ No newline at end of file diff --git a/tests/fixtures/ont/release-green/evidence/dogfood/kernels/0.69.1/lambda.json b/tests/fixtures/ont/release-green/evidence/dogfood/kernels/0.69.1/lambda.json new file mode 100644 index 0000000000..4964de4bce --- /dev/null +++ b/tests/fixtures/ont/release-green/evidence/dogfood/kernels/0.69.1/lambda.json @@ -0,0 +1,46 @@ +{ + "schema": "apr-kernel-diff-receipt/v1", + "host": "lambda", + "version": "0.69.1", + "apr_sha": "1111111111111111111111111111111111111111", + "cc": "8.9", + "sm": "sm_89", + "dispatch": [ + { + "sha256": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "file": "tiny-qwen35.gguf", + "kernels": [ + { + "kernel": "q4k_gemv", + "quant": "q4_k" + }, + { + "kernel": "rmsnorm", + "quant": "f32" + } + ] + } + ], + "rows": [ + { + "kernel": "q4k_gemv", + "quant": "q4_k", + "reference": "cpu", + "max_err": 0.001, + "bound": 0.004, + "bound_source": "fixture://q4k-pair", + "verdict": "pass", + "reason": "" + }, + { + "kernel": "rmsnorm", + "quant": "f32", + "reference": "cpu", + "max_err": 1e-06, + "bound": 1e-05, + "bound_source": "fixture://rmsnorm-pair", + "verdict": "pass", + "reason": "" + } + ] +} \ No newline at end of file diff --git a/tests/fixtures/ont/release-green/evidence/dogfood/models/0.69.1/gx10.json b/tests/fixtures/ont/release-green/evidence/dogfood/models/0.69.1/gx10.json new file mode 100644 index 0000000000..4e882a0b89 --- /dev/null +++ b/tests/fixtures/ont/release-green/evidence/dogfood/models/0.69.1/gx10.json @@ -0,0 +1,59 @@ +{ + "schema": "apr-model-ladder-receipt/v2", + "host": "gx10", + "version": "0.69.1", + "sha": "111111111", + "apr_sha": "1111111111111111111111111111111111111111", + "cc": "12.1", + "gpu": "fixture GPU", + "executed": 1, + "red": 0, + "inventory": [ + { + "file": "tiny-qwen35.gguf", + "sha256": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "bytes": 4, + "arch": "qwen35", + "quant": "q4_k_m", + "context_length": 10000, + "thinking_markers": [ + "", + "", + "enable_thinking" + ], + "chat_template_sha256": "cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc", + "owes_long_rungs": true, + "thinking_modes": [ + "on", + "off" + ] + } + ], + "rungs": [], + "cells": [ + {"sha256":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","file":"tiny-qwen35.gguf","verb":"run","thinking":"on","context":"4k","prompt_tokens":4200,"max_tokens":256,"answer_chars":42,"ttft_ms":120.0,"verdict":"pass","backend":"cuda","fallback":false,"rc":0,"reason":"","think_closed":true}, + {"sha256":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","file":"tiny-qwen35.gguf","verb":"run","thinking":"on","context":"consumer-max","prompt_tokens":8100,"max_tokens":256,"answer_chars":42,"ttft_ms":120.0,"verdict":"pass","backend":"cuda","fallback":false,"rc":0,"reason":"","think_closed":true}, + {"sha256":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","file":"tiny-qwen35.gguf","verb":"run","thinking":"on","context":"declared","prompt_tokens":9800,"max_tokens":256,"answer_chars":42,"ttft_ms":120.0,"verdict":"pass","backend":"cuda","fallback":false,"rc":0,"reason":"","think_closed":true}, + {"sha256":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","file":"tiny-qwen35.gguf","verb":"run","thinking":"off","context":"4k","prompt_tokens":4200,"max_tokens":256,"answer_chars":42,"ttft_ms":120.0,"verdict":"pass","backend":"cuda","fallback":false,"rc":0,"reason":""}, + {"sha256":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","file":"tiny-qwen35.gguf","verb":"run","thinking":"off","context":"consumer-max","prompt_tokens":8100,"max_tokens":256,"answer_chars":42,"ttft_ms":120.0,"verdict":"pass","backend":"cuda","fallback":false,"rc":0,"reason":""}, + {"sha256":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","file":"tiny-qwen35.gguf","verb":"run","thinking":"off","context":"declared","prompt_tokens":9800,"max_tokens":256,"answer_chars":42,"ttft_ms":120.0,"verdict":"pass","backend":"cuda","fallback":false,"rc":0,"reason":""}, + {"sha256":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","file":"tiny-qwen35.gguf","verb":"chat","thinking":"on","context":"4k","prompt_tokens":4200,"max_tokens":256,"answer_chars":42,"ttft_ms":120.0,"verdict":"pass","backend":"cuda","fallback":false,"rc":0,"reason":"","think_closed":true}, + {"sha256":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","file":"tiny-qwen35.gguf","verb":"chat","thinking":"on","context":"consumer-max","prompt_tokens":8100,"max_tokens":256,"answer_chars":42,"ttft_ms":120.0,"verdict":"pass","backend":"cuda","fallback":false,"rc":0,"reason":"","think_closed":true}, + {"sha256":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","file":"tiny-qwen35.gguf","verb":"chat","thinking":"on","context":"declared","prompt_tokens":9800,"max_tokens":256,"answer_chars":42,"ttft_ms":120.0,"verdict":"pass","backend":"cuda","fallback":false,"rc":0,"reason":"","think_closed":true}, + {"sha256":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","file":"tiny-qwen35.gguf","verb":"chat","thinking":"off","context":"4k","prompt_tokens":4200,"max_tokens":256,"answer_chars":42,"ttft_ms":120.0,"verdict":"pass","backend":"cuda","fallback":false,"rc":0,"reason":""}, + {"sha256":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","file":"tiny-qwen35.gguf","verb":"chat","thinking":"off","context":"consumer-max","prompt_tokens":8100,"max_tokens":256,"answer_chars":42,"ttft_ms":120.0,"verdict":"pass","backend":"cuda","fallback":false,"rc":0,"reason":""}, + {"sha256":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","file":"tiny-qwen35.gguf","verb":"chat","thinking":"off","context":"declared","prompt_tokens":9800,"max_tokens":256,"answer_chars":42,"ttft_ms":120.0,"verdict":"pass","backend":"cuda","fallback":false,"rc":0,"reason":""}, + {"sha256":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","file":"tiny-qwen35.gguf","verb":"serve","thinking":"on","context":"4k","prompt_tokens":4200,"max_tokens":256,"answer_chars":42,"ttft_ms":120.0,"verdict":"pass","backend":"cuda","fallback":false,"rc":0,"reason":"","think_closed":true}, + {"sha256":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","file":"tiny-qwen35.gguf","verb":"serve","thinking":"on","context":"consumer-max","prompt_tokens":8100,"max_tokens":256,"answer_chars":42,"ttft_ms":120.0,"verdict":"pass","backend":"cuda","fallback":false,"rc":0,"reason":"","think_closed":true}, + {"sha256":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","file":"tiny-qwen35.gguf","verb":"serve","thinking":"on","context":"declared","prompt_tokens":9800,"max_tokens":256,"answer_chars":42,"ttft_ms":120.0,"verdict":"pass","backend":"cuda","fallback":false,"rc":0,"reason":"","think_closed":true}, + {"sha256":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","file":"tiny-qwen35.gguf","verb":"serve","thinking":"off","context":"4k","prompt_tokens":4200,"max_tokens":256,"answer_chars":42,"ttft_ms":120.0,"verdict":"pass","backend":"cuda","fallback":false,"rc":0,"reason":""}, + {"sha256":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","file":"tiny-qwen35.gguf","verb":"serve","thinking":"off","context":"consumer-max","prompt_tokens":8100,"max_tokens":256,"answer_chars":42,"ttft_ms":120.0,"verdict":"pass","backend":"cuda","fallback":false,"rc":0,"reason":""}, + {"sha256":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","file":"tiny-qwen35.gguf","verb":"serve","thinking":"off","context":"declared","prompt_tokens":9800,"max_tokens":256,"answer_chars":42,"ttft_ms":120.0,"verdict":"pass","backend":"cuda","fallback":false,"rc":0,"reason":""}, + {"sha256":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","file":"tiny-qwen35.gguf","verb":"code","thinking":"on","context":"4k","prompt_tokens":4200,"max_tokens":256,"answer_chars":42,"ttft_ms":120.0,"verdict":"pass","backend":"cuda","fallback":false,"rc":0,"reason":"","think_closed":true}, + {"sha256":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","file":"tiny-qwen35.gguf","verb":"code","thinking":"on","context":"consumer-max","prompt_tokens":8100,"max_tokens":256,"answer_chars":42,"ttft_ms":120.0,"verdict":"pass","backend":"cuda","fallback":false,"rc":0,"reason":"","think_closed":true}, + {"sha256":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","file":"tiny-qwen35.gguf","verb":"code","thinking":"on","context":"declared","prompt_tokens":9800,"max_tokens":256,"answer_chars":42,"ttft_ms":120.0,"verdict":"pass","backend":"cuda","fallback":false,"rc":0,"reason":"","think_closed":true}, + {"sha256":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","file":"tiny-qwen35.gguf","verb":"code","thinking":"off","context":"4k","prompt_tokens":4200,"max_tokens":256,"answer_chars":42,"ttft_ms":120.0,"verdict":"pass","backend":"cuda","fallback":false,"rc":0,"reason":""}, + {"sha256":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","file":"tiny-qwen35.gguf","verb":"code","thinking":"off","context":"consumer-max","prompt_tokens":8100,"max_tokens":256,"answer_chars":42,"ttft_ms":120.0,"verdict":"pass","backend":"cuda","fallback":false,"rc":0,"reason":""}, + {"sha256":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","file":"tiny-qwen35.gguf","verb":"code","thinking":"off","context":"declared","prompt_tokens":9800,"max_tokens":256,"answer_chars":42,"ttft_ms":120.0,"verdict":"pass","backend":"cuda","fallback":false,"rc":0,"reason":""} + ] +} diff --git a/tests/fixtures/ont/release-green/evidence/dogfood/models/0.69.1/lambda.json b/tests/fixtures/ont/release-green/evidence/dogfood/models/0.69.1/lambda.json new file mode 100644 index 0000000000..3c8fd7eb72 --- /dev/null +++ b/tests/fixtures/ont/release-green/evidence/dogfood/models/0.69.1/lambda.json @@ -0,0 +1,59 @@ +{ + "schema": "apr-model-ladder-receipt/v2", + "host": "lambda", + "version": "0.69.1", + "sha": "111111111", + "apr_sha": "1111111111111111111111111111111111111111", + "cc": "8.9", + "gpu": "fixture GPU", + "executed": 1, + "red": 0, + "inventory": [ + { + "file": "tiny-qwen35.gguf", + "sha256": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "bytes": 4, + "arch": "qwen35", + "quant": "q4_k_m", + "context_length": 10000, + "thinking_markers": [ + "", + "", + "enable_thinking" + ], + "chat_template_sha256": "cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc", + "owes_long_rungs": true, + "thinking_modes": [ + "on", + "off" + ] + } + ], + "rungs": [], + "cells": [ + {"sha256":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","file":"tiny-qwen35.gguf","verb":"run","thinking":"on","context":"4k","prompt_tokens":4200,"max_tokens":256,"answer_chars":42,"ttft_ms":120.0,"verdict":"pass","backend":"cuda","fallback":false,"rc":0,"reason":"","think_closed":true}, + {"sha256":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","file":"tiny-qwen35.gguf","verb":"run","thinking":"on","context":"consumer-max","prompt_tokens":8100,"max_tokens":256,"answer_chars":42,"ttft_ms":120.0,"verdict":"pass","backend":"cuda","fallback":false,"rc":0,"reason":"","think_closed":true}, + {"sha256":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","file":"tiny-qwen35.gguf","verb":"run","thinking":"on","context":"declared","prompt_tokens":9800,"max_tokens":256,"answer_chars":42,"ttft_ms":120.0,"verdict":"pass","backend":"cuda","fallback":false,"rc":0,"reason":"","think_closed":true}, + {"sha256":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","file":"tiny-qwen35.gguf","verb":"run","thinking":"off","context":"4k","prompt_tokens":4200,"max_tokens":256,"answer_chars":42,"ttft_ms":120.0,"verdict":"pass","backend":"cuda","fallback":false,"rc":0,"reason":""}, + {"sha256":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","file":"tiny-qwen35.gguf","verb":"run","thinking":"off","context":"consumer-max","prompt_tokens":8100,"max_tokens":256,"answer_chars":42,"ttft_ms":120.0,"verdict":"pass","backend":"cuda","fallback":false,"rc":0,"reason":""}, + {"sha256":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","file":"tiny-qwen35.gguf","verb":"run","thinking":"off","context":"declared","prompt_tokens":9800,"max_tokens":256,"answer_chars":42,"ttft_ms":120.0,"verdict":"pass","backend":"cuda","fallback":false,"rc":0,"reason":""}, + {"sha256":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","file":"tiny-qwen35.gguf","verb":"chat","thinking":"on","context":"4k","prompt_tokens":4200,"max_tokens":256,"answer_chars":42,"ttft_ms":120.0,"verdict":"pass","backend":"cuda","fallback":false,"rc":0,"reason":"","think_closed":true}, + {"sha256":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","file":"tiny-qwen35.gguf","verb":"chat","thinking":"on","context":"consumer-max","prompt_tokens":8100,"max_tokens":256,"answer_chars":42,"ttft_ms":120.0,"verdict":"pass","backend":"cuda","fallback":false,"rc":0,"reason":"","think_closed":true}, + {"sha256":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","file":"tiny-qwen35.gguf","verb":"chat","thinking":"on","context":"declared","prompt_tokens":9800,"max_tokens":256,"answer_chars":42,"ttft_ms":120.0,"verdict":"pass","backend":"cuda","fallback":false,"rc":0,"reason":"","think_closed":true}, + {"sha256":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","file":"tiny-qwen35.gguf","verb":"chat","thinking":"off","context":"4k","prompt_tokens":4200,"max_tokens":256,"answer_chars":42,"ttft_ms":120.0,"verdict":"pass","backend":"cuda","fallback":false,"rc":0,"reason":""}, + {"sha256":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","file":"tiny-qwen35.gguf","verb":"chat","thinking":"off","context":"consumer-max","prompt_tokens":8100,"max_tokens":256,"answer_chars":42,"ttft_ms":120.0,"verdict":"pass","backend":"cuda","fallback":false,"rc":0,"reason":""}, + {"sha256":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","file":"tiny-qwen35.gguf","verb":"chat","thinking":"off","context":"declared","prompt_tokens":9800,"max_tokens":256,"answer_chars":42,"ttft_ms":120.0,"verdict":"pass","backend":"cuda","fallback":false,"rc":0,"reason":""}, + {"sha256":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","file":"tiny-qwen35.gguf","verb":"serve","thinking":"on","context":"4k","prompt_tokens":4200,"max_tokens":256,"answer_chars":42,"ttft_ms":120.0,"verdict":"pass","backend":"cuda","fallback":false,"rc":0,"reason":"","think_closed":true}, + {"sha256":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","file":"tiny-qwen35.gguf","verb":"serve","thinking":"on","context":"consumer-max","prompt_tokens":8100,"max_tokens":256,"answer_chars":42,"ttft_ms":120.0,"verdict":"pass","backend":"cuda","fallback":false,"rc":0,"reason":"","think_closed":true}, + {"sha256":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","file":"tiny-qwen35.gguf","verb":"serve","thinking":"on","context":"declared","prompt_tokens":9800,"max_tokens":256,"answer_chars":42,"ttft_ms":120.0,"verdict":"pass","backend":"cuda","fallback":false,"rc":0,"reason":"","think_closed":true}, + {"sha256":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","file":"tiny-qwen35.gguf","verb":"serve","thinking":"off","context":"4k","prompt_tokens":4200,"max_tokens":256,"answer_chars":42,"ttft_ms":120.0,"verdict":"pass","backend":"cuda","fallback":false,"rc":0,"reason":""}, + {"sha256":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","file":"tiny-qwen35.gguf","verb":"serve","thinking":"off","context":"consumer-max","prompt_tokens":8100,"max_tokens":256,"answer_chars":42,"ttft_ms":120.0,"verdict":"pass","backend":"cuda","fallback":false,"rc":0,"reason":""}, + {"sha256":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","file":"tiny-qwen35.gguf","verb":"serve","thinking":"off","context":"declared","prompt_tokens":9800,"max_tokens":256,"answer_chars":42,"ttft_ms":120.0,"verdict":"pass","backend":"cuda","fallback":false,"rc":0,"reason":""}, + {"sha256":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","file":"tiny-qwen35.gguf","verb":"code","thinking":"on","context":"4k","prompt_tokens":4200,"max_tokens":256,"answer_chars":42,"ttft_ms":120.0,"verdict":"pass","backend":"cuda","fallback":false,"rc":0,"reason":"","think_closed":true}, + {"sha256":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","file":"tiny-qwen35.gguf","verb":"code","thinking":"on","context":"consumer-max","prompt_tokens":8100,"max_tokens":256,"answer_chars":42,"ttft_ms":120.0,"verdict":"pass","backend":"cuda","fallback":false,"rc":0,"reason":"","think_closed":true}, + {"sha256":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","file":"tiny-qwen35.gguf","verb":"code","thinking":"on","context":"declared","prompt_tokens":9800,"max_tokens":256,"answer_chars":42,"ttft_ms":120.0,"verdict":"pass","backend":"cuda","fallback":false,"rc":0,"reason":"","think_closed":true}, + {"sha256":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","file":"tiny-qwen35.gguf","verb":"code","thinking":"off","context":"4k","prompt_tokens":4200,"max_tokens":256,"answer_chars":42,"ttft_ms":120.0,"verdict":"pass","backend":"cuda","fallback":false,"rc":0,"reason":""}, + {"sha256":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","file":"tiny-qwen35.gguf","verb":"code","thinking":"off","context":"consumer-max","prompt_tokens":8100,"max_tokens":256,"answer_chars":42,"ttft_ms":120.0,"verdict":"pass","backend":"cuda","fallback":false,"rc":0,"reason":""}, + {"sha256":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","file":"tiny-qwen35.gguf","verb":"code","thinking":"off","context":"declared","prompt_tokens":9800,"max_tokens":256,"answer_chars":42,"ttft_ms":120.0,"verdict":"pass","backend":"cuda","fallback":false,"rc":0,"reason":""} + ] +} diff --git a/tests/fixtures/ont/release-green/evidence/dogfood/tokenizer/0.69.1/parity.json b/tests/fixtures/ont/release-green/evidence/dogfood/tokenizer/0.69.1/parity.json new file mode 100644 index 0000000000..6d2c474e18 --- /dev/null +++ b/tests/fixtures/ont/release-green/evidence/dogfood/tokenizer/0.69.1/parity.json @@ -0,0 +1,21 @@ +{ + "schema": "apr-tokenizer-parity-receipt/v1", + "version": "0.69.1", + "apr_sha": "1111111111111111111111111111111111111111", + "host": "lambda", + "rows": [ + { + "sha256": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "file": "tiny-qwen35.gguf", + "family": "qwen35", + "corpus_sha256": "eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee", + "comparator": "llama.cpp", + "comparator_sha": "60b06ab9a", + "tokens_compared": 5000, + "mismatches": 0, + "roundtrip_ok": true, + "verdict": "pass", + "reason": "" + } + ] +} \ No newline at end of file diff --git a/tests/fixtures/ont/release-green/evidence/release/context-rungs.json b/tests/fixtures/ont/release-green/evidence/release/context-rungs.json new file mode 100644 index 0000000000..6837b5a667 --- /dev/null +++ b/tests/fixtures/ont/release-green/evidence/release/context-rungs.json @@ -0,0 +1,24 @@ +{ + "schema": "apr-release-context-rungs/v1", + "consumers": [ + { + "consumer": "fixture-consumer", + "max_prompt_tokens": 8000, + "basis": "measured", + "source": "fixture://consumer" + } + ], + "rungs": [ + { + "id": "4k", + "tokens": 4096, + "derived_from": [ + "fixture://rung-4k" + ] + }, + { + "id": "consumer-max", + "long": true + } + ] +} \ No newline at end of file