diff --git a/CHANGELOG.md b/CHANGELOG.md index 0b84d0e271..3d0e1b7a9d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,17 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] ### Known issues +- **Qwen3 MoE (Qwen3-30B-A3B, Qwen3-Coder-30B-A3B) on GB10 is not claimed for 0.70 (#3715); planned for 0.70.1.** + On GB10 (sm_121) every verb (run, chat, serve, code) times out at 600 s on the 20k-token rung. The 0.69.1 + pre-load refusal is gone in 0.70.0, so these models now load and run, but too slowly to be claimed. The + release gate does not owe the qwen3moe x GB10 cells (`cells.declaimed` in + `contracts/model-capability-ladder-v1.yaml`), and the ladder gate is RED if any rung or long-rung + representative still claims them. Withdrawn, not waived: lambda holds both files and still owes every + qwen3moe cell. +- **GB10 dense models: prompt processing unbatched (11.4 tok/s); planned for 0.70.1 (#4590).** Widens the Qwen3 entry below to every dense model on GB10 (operator ruling 2026-09-28 + 16:15Z): the serial prefill is the sm_12x default for every non-hybrid arch, so dense Qwen2.5 on GB10 + is de-claimed too (`cells.declaimed`: qwen2 and qwen3 on gx10). Output is correct; only prefill speed + is affected. Withdrawn, not waived: lambda still owes every dense cell, gx10 still owes every Qwen3.5 cell. - **Qwen3 on GB10 (sm_121, and the sm_12x family) is correct but very slow, and is not claimed for 0.70 (#4590).** Dense Qwen3 on sm_12x prefills one token at a time, the sm_12x default, because the generic batched prefill corrupts the KV cache there (FALSIFY-CPU-GPU-009). Measured on 0.70.0 diff --git a/README.md b/README.md index 1500157cb4..3d46653b21 100644 --- a/README.md +++ b/README.md @@ -115,6 +115,15 @@ A non-`pass` cell names the gates the receipt records, which is not always the c +> **Known issue — Qwen3 MoE (30B-A3B) on GB10: not claimed for 0.70; planned for 0.70.1 (#3715).** Every verb +> times out at 600 s on the 20k-token rung on GB10 (sm_121), so qwen3moe on GB10 is withdrawn from this +> release's claims (`cells.declaimed`). + +> **Known issue — GB10 dense models: prompt processing unbatched (11.4 tok/s); planned for 0.70.1 (#4590).** Output is correct; only prefill speed is affected. On GB10 (sm_121) every dense +> model (Qwen2.5, Qwen3) is withdrawn from this release's claims (`cells.declaimed` in +> [`contracts/model-capability-ladder-v1.yaml`](contracts/model-capability-ladder-v1.yaml)); Qwen3.5 on +> GB10 and every model on other hosts are still claimed and gated. + ### Command surface | Stage | Commands | diff --git a/contracts/model-capability-ladder-v1.yaml b/contracts/model-capability-ladder-v1.yaml index d6a5e24664..1852abcb79 100644 --- a/contracts/model-capability-ladder-v1.yaml +++ b/contracts/model-capability-ladder-v1.yaml @@ -236,11 +236,32 @@ ladder: correct but 27-34x slower than Qwen3.5 on GB10 sm_121 (8k think-on: Qwen3-1.7B 967 s, Qwen3-8B 2163 s vs Qwen3.5-9B 63-80 s; serial prefill, the sm12x default). 0.70.1 P0 ships native sm_121 SASS and re-tests batched prefill on GB10. lambda still owes every Qwen3 cell. + # G2 (operator ruling 2026-09-28 16:15Z): R1 was Qwen3 only; this widens it to EVERY dense arch gx10 holds. + # Serial prefill is the default for every arch at cc>=120 (select_prefill_path), not only Qwen3. + - arch: qwen2 + host: gx10 + issue: 4590 + until: "0.70.1" + why: >- + GB10 dense models: prompt processing unbatched (11.4 tok/s); fixed in 0.70.1 (#4590). Correct output, + prefill speed only: sm_12x prefills every dense arch one token at a time (select_prefill_path). Withdrawn, + not waived: lambda still owes every Qwen2 cell, and gx10 still owes every Qwen3.5 (hybrid) cell. + # #3715 H4 (cop 2026-09-28 20:26Z, scope = qwen3moe on gx10 only; STAGED, branch-only, operator rules 07:30Z): + # arch-level DATA is clean here -- no named rung and no long-rung representative is qwen3moe. + - arch: qwen3moe + host: gx10 + issue: 3715 + until: "0.70.1" + why: >- + Qwen3-30B-A3B (and Coder-30B-A3B) on GB10 sm_121: every verb hits the 600 s cell timeout at the + 20k rung (0d H4 19:51Z, 12+12 cells). The 0.69.1 pre-load refusal is gone in 0.70.0-dev, so the + cells run and time out. Withdrawn, not waived: lambda holds both files and still owes every + qwen3moe cell; gx10 still owes every Qwen3.5 (hybrid) cell. Operator D1 APPROVED 2026-09-28 20:10Z. long_rungs_for: families: [qwen35] # every consumer's family: the full rung set, through declared representatives: # one file per other arch the fleet holds - qwen2: qwen2.5-coder-7b-instruct-q4_k_m.gguf # lambda + gx10 - qwen3: Qwen3-8B-Q4_K_M.gguf # lambda + gx10 + qwen2: qwen2.5-coder-7b-instruct-q4_k_m.gguf # lambda (gx10 de-claimed, #4590) + qwen3: Qwen3-8B-Q4_K_M.gguf # lambda (gx10 de-claimed, #4590) # qwen3moe: REMOVED for 0.69.1 (#3817). Not a fit question — this build has no CUDA # forward for the architecture at all (#3714, lands 0.70.0), so `apr run --gpu` refuses # pre-load by name and there is no GPU cell to owe. The CPU path is unchanged and works. @@ -251,6 +272,7 @@ ladder: arch: qwen2 gguf: qwen2.5-1.5b-instruct-q4_k_m.gguf sha256: 6a1a2eb6d15622bf3c96857206351ba97e1af16c30d7a74ee38970e434e9407e + hosts: [lambda] # gx10 DE-CLAIMED for 0.70 (#4590, cells.declaimed, G2); restore to every host in 0.70.1 backends: [cpu, cuda] required: true note: "control rung — the architecture every other gate already exercises; if this is red the host is broken, not the model" diff --git a/crates/aprender-contracts-cli/tests/ont_release_readiness.rs b/crates/aprender-contracts-cli/tests/ont_release_readiness.rs index 94d2755ccb..e513fc834d 100644 --- a/crates/aprender-contracts-cli/tests/ont_release_readiness.rs +++ b/crates/aprender-contracts-cli/tests/ont_release_readiness.rs @@ -1157,3 +1157,116 @@ fn names(v: &serde_json::Value) -> Vec<&str> { .map(|a| a.iter().filter_map(serde_json::Value::as_str).collect()) .unwrap_or_default() } + +// ── G2 (#4590, operator ruling 2026-09-28 16:15Z): a GB10 dense model is WITHDRAWN, not waived ─────────── +// +// scripts/model_ladder.sh moves an arch `cells.declaimed` names for the host into the receipt's +// `declaimed_inventory[]`; pv reads `inventory[]` only, so the withdrawn (host, model) owes nothing. Withdrawn +// is not waived: every other host that holds the model still owes every cell of it. And a claim of it on GB10 +// (the item back in gx10's `inventory[]`, unmeasured) is RED at the enforce-mode gate. + +const DENSE_SHA: &str = "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd"; + +fn dense_item() -> Value { + serde_json::json!({ + "file": "tiny-qwen2-dense.gguf", "sha256": DENSE_SHA, "bytes": 4, "arch": "qwen2", + "quant": "q4_k_m", "context_length": 10000, "owes_long_rungs": false, "thinking_modes": ["off"], + "chat_template_sha256": "cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc" + }) +} + +fn dense_cells(t: &Path, host: &str) -> Vec { + read(&t.join("cells.json"))["cells"] + .as_array() + .expect("cells") + .iter() + .filter(|c| c["host"] == host && c["model_sha256"] == DENSE_SHA) + .filter_map(|c| c["id"].as_str().map(str::to_string)) + .collect() +} + +/// The green base plus a dense model lambda HOLDS and MEASURES (inventory, kernel dispatch, tokenizer parity, +/// one passing row per derived cell) and gx10 holds WITHDRAWN (`declaimed_inventory[]`, as model_ladder.sh writes it). +fn withdrawn_on_gx10() -> tempfile::TempDir { + let t = green(); + edit(&models(t.path(), "lambda"), |v| { + v["inventory"] + .as_array_mut() + .expect("inventory") + .push(dense_item()) + }); + edit(&kernels(t.path(), "lambda"), |v| { + let mut d = v["dispatch"][0].clone(); + d["sha256"] = DENSE_SHA.into(); + d["file"] = "tiny-qwen2-dense.gguf".into(); + v["dispatch"].as_array_mut().expect("dispatch").push(d); + }); + edit(&tokenizer(t.path()), |v| { + let mut r = v["rows"][0].clone(); + r["sha256"] = DENSE_SHA.into(); + r["file"] = "tiny-qwen2-dense.gguf".into(); + r["family"] = "qwen2".into(); + v["rows"].as_array_mut().expect("rows").push(r); + }); + edit(&models(t.path(), "gx10"), |v| { + v["declaimed_inventory"] = Value::Array(vec![dense_item()]) + }); + synthesize(t.path()); + t +} + +#[test] +fn a_withdrawn_gb10_dense_model_owes_nothing_there_a_planted_claim_is_red_and_lambda_still_owes_it() +{ + let t = withdrawn_on_gx10(); + let r = gate(t.path(), &[]); + assert_eq!( + r.code, + 0, + "withdrawn on gx10, measured on lambda: GREEN\n{:#}", + json_of(&r)["findings"] + ); + assert!( + !dense_cells(t.path(), "lambda").is_empty(), + "withdrawn is not waived: lambda owes the dense model's cells" + ); + assert!( + dense_cells(t.path(), "gx10").is_empty(), + "gx10 owes no cell of a withdrawn model" + ); + + // PLANTED CLAIM: gx10's receipt claims the dense model (back in inventory[]) with no row measured -> RED. + edit(&models(t.path(), "gx10"), |v| { + let item = v["declaimed_inventory"] + .as_array_mut() + .expect("declaimed_inventory") + .remove(0); + v["inventory"].as_array_mut().expect("inventory").push(item); + }); + let red = gate(t.path(), &[]); + assert_eq!( + red.code, + 1, + "a planted GB10 dense claim is RED\n{}", + show(&red) + ); + let v = json_of(&red); + let msgs: Vec<&str> = v["findings"] + .as_array() + .expect("findings") + .iter() + .filter_map(|f| f["message"].as_str()) + .collect(); + assert!( + msgs.iter() + .any(|m| m.contains("/gx10") && m.contains("tiny-qwen2-dense")), + "the planted GB10 dense claim is what went red:\n{}", + msgs.iter().take(40).copied().collect::>().join("\n") + ); + + // WITHDRAWN != WAIVED: lambda dropping one dense row is RED naming that cell. + let t = withdrawn_on_gx10(); + let id = dense_cells(t.path(), "lambda").remove(0); + drop_row(t.path(), "lambda", &id); + assert_red_naming(&gate(t.path(), &[]), &named(&id)); +} diff --git a/docs/audits/impl-PMAT-3715-receipt.md b/docs/audits/impl-PMAT-3715-receipt.md index 5358ffc9fb..d7b84ea512 100644 --- a/docs/audits/impl-PMAT-3715-receipt.md +++ b/docs/audits/impl-PMAT-3715-receipt.md @@ -145,3 +145,34 @@ Measured on this branch's head: Known and not fixed here, because it is apr's gap and not the producer's: `apr code --output-format json` carries no backend field (`crates/aprender-orchestrate/src/agent/code_envelope.rs` `envelope()`). So every `code` cell is red and T-1 STOPs until that field lands. This is why the branch is on the FINAL path, not rc.1 (cop, 2026-09-28). + +## 8. PR #4603, `0d/3715-b1-main` (base `split/ont10-rest-0.70` @ 9a29a7d19f): B1 enforces at T-1/T-4 + +Sections 4, 6 and 7 describe earlier branches. Section 7 in particular is the `0d/3715-cells-at-t1` diff, which +touched no Rust and no contract. **This** diff (`git diff origin/split/ont10-rest-0.70...HEAD`, 25 files) does touch +Rust (`crates/aprender-contracts-cli/tests/ont_release_readiness.rs`, a test) and one contract +(`contracts/model-capability-ladder-v1.yaml`, data only: `cells.declaimed`). It: + +- deletes the report-only arm of `release_readiness.sh` (R10); `report` is refused as env value and as a committed default; +- makes `autopilot.sh` `cut_tag()` and `check_publish_preflight.sh` R8 refuse a `WARN R8` row and any rc 0 that lacks + `#3715 ENFORCE PASS version=V commit=MC` for this version and HEAD; +- adds the ladder RED for a de-claimed (host, arch) the ladder still claims (`check_model_ladder.sh`, planted-RED cases); +- withdraws the GB10 dense claims through data (`cells.declaimed`: qwen3moe, qwen2 on gx10), with the README and CHANGELOG warning. + +**De-claim is not a waiver.** Ruling, operator 2026-09-28 16:15Z (via the cop, `cop-inbox/handoff/ruling-gx10-1615.md`): +"every GB10 dense claim is withdrawn via DATA (gx10 inventory), with a warning in the notes and README, and the +enforce-mode gate turns RED on any GB10-dense claim ('withdrawn ≠ waived')". The gate has no waiver input; a claim +that returns needs receipts at the 0.70.1 pin (G5). The control rung `qwen2.5-1.5b` is `hosts: [lambda]` for the same +reason: it is a dense GB10 claim on the withdrawn set. Wording in README and CHANGELOG is "planned for 0.70.1", not "fixed". + +Measured at head `9efceaa024` (intel for cargo; scripts on lambda, load 8): + +| command | result | +|---------|--------| +| `bash scripts/release/release_readiness.sh --selftest` | 30/30 rows, incl. `mode_has_no_report_arm` | +| `bash scripts/check_release_models_t1.sh --self-test` | PASS 41 rows | +| `bash scripts/check_model_ladder.sh --self-test` | 153 cases, 0 bad | +| `bash scripts/check_publish_preflight.sh --selftest` | 57/57 rows | +| `cargo test -p aprender-contracts-cli --test ont_release_readiness` | see the PR thread (re-run at the final head) | + +Blocked on the gate pv: 0.69.2 has no `--surface`; the chain is pv 0.69.4, pin, B1, rc.1. diff --git a/docs/audits/quorum-PMAT-3715.json b/docs/audits/quorum-PMAT-3715.json index a916791f9d..678195ee49 100644 --- a/docs/audits/quorum-PMAT-3715.json +++ b/docs/audits/quorum-PMAT-3715.json @@ -1,16 +1,17 @@ { "ticket": "PMAT-3715", - "base": "origin/main", - "base_resolved": "origin/main", - "base_note": "no origin/origin/main exists; judged against the local ref", - "head": "84289cb7ba38adf96c0a1c5f836eb84ccb8d0646", - "diff_sha256": "9df958d287befbf36a6b50be55984468291262bf8a3cf2cf9507b584e2f5676b", + "base": "origin/split/ont10-rest-0.70", + "base_resolved": "origin/split/ont10-rest-0.70", + "base_note": "no origin/origin/split/ont10-rest-0.70 exists; judged against the local ref", + "head": "33b032506af6cb0300815e499ba9109dbda76a2c", + "diff_sha256": "d69e5fbd0a41da0b3f37e0f695a547ee58cf4e3ae8a4c4fed92015668801d74a", "width": 3, "executor": "agy", "prompt_mode": "file", - "prompt_bytes": 291117, + "prompt_bytes": 103770, + "prompt_sha256": "8fbdb9ff2c4381dca7e69a1b7fc21d58116ff831bb58f14d08dcc66f5284a975", "author": { - "model": "claude-opus-5", + "model": "claude-sonnet-5-5", "family": "claude", "source": "flag" }, @@ -20,25 +21,48 @@ "lane": 1, "status": "SUCCESS", "verdict": "PASS", - "summary": "The PR accurately and completely implements the PMAT-3715 release-readiness shape and its requested scope additions. I verified the graph extraction logic, the generation of KernelDiffReceipt and TokenizerParityReceipt nodes, the parsing of the GGUF `context_length` for declared rungs, and the correct verification of thinking mode consistency against `enable_thinking` and `thinking_markers`. The release-readiness-v1 shape rigorously checks for missing, skipped, or stale receipts and requires `fallback=false` without providing any waiver path. Furthermore, the `positive_control()` has been successfully added to `extract_controls()` to ensure the gate cannot ship without its control, perfectly fulfilling the ticket requirements without introducing any forbidden behavior.", - "findings": [], - "raw_bytes": 2809, - "err_bytes": 100, + "summary": "Lane 1 verdict: PASS. I read the whole brief in slices (the ticket, the receipt with its §8 for this diff, and the full diff) and nothing in it refutes the change.\n\nWhat the diff does:\n- **Report-only mode is gone.** `release_readiness.sh` no longer has a report arm. `resolve_mode` refuses `report` whether it comes from the environment or from the committed default. A grep of the script finds no leftover REPORT-ONLY path.\n- **Both decision points refuse a WARN row.** At T-4, `check_publish_preflight.sh` R8 refuses a `WARN R8` row. It also refuses an exit code of 0 that comes without the `#3715 ENFORCE PASS version=V commit=HEAD` line. At T-1, `autopilot.sh` now dies on a WARN row. The receipt says `cut_tag()` requires the ENFORCE PASS line. That check is already in `autopilot.sh` (lines 205-207); this diff does not add it.\n- **Self-tests now expect refusals.** The rows that expected a WARN to pass or continue now expect a refusal: `r8_report_mode_warn_refuses`, the new `r8_rc0_without_enforce_pass_refuses`, and `readiness-warn-stops`. None of them asserts the opposite of the ticket.\n- **The GB10 dense withdrawal is data, not a waiver.** `cells.declaimed` gains entries for qwen2 and qwen3moe on gx10 (qwen3 was already there). The new `declaim_still_claimed` check turns the ladder RED when a rung or long-rung representative still claims a withdrawn (host, arch). It has a planted-RED case and a mutant (`declaim-claimed`). The committed evidence shows rc=0 on the real contract and rc=1 on the planted cases.\n- **The new Rust test proves \"withdrawn is not waived\".** A withdrawn gx10 model is green and owes no gx10 cells, while lambda still owes its cells. Putting the model back in gx10's inventory unmeasured turns it RED naming `/gx10`. Dropping one lambda row turns it RED naming that cell.\n\nThe ticket's \"every model × {lambda, gx10}\" universe is narrowed only through the operator ruling the receipt quotes. There is still no waiver input.\n\nI found two minor issues, neither of which refutes the diff. I did not find the lane-witness token: `paiml-lane-witness` does not exist in this worktree's git dir, so `tree_witness` is left out.", + "findings": [ + { + "file": "contracts/model-capability-ladder-v1.yaml", + "line": 256, + "claim": "The new qwen2/gx10 de-claim `why` text says \"fixed in 0.70.1 (#4590)\". Receipt §8 says the wording is \"planned for 0.70.1\", not \"fixed\". That claim is only true of README and CHANGELOG, not of the contract data.", + "grounding": "cited", + "fix": "Change \"fixed in 0.70.1\" to \"planned for 0.70.1\" in the declaimed[].why for qwen2/gx10." + }, + { + "file": "docs/audits/impl-PMAT-3715-receipt.md", + "line": 445, + "claim": "The receipt gives no measured result for the one Rust change, the new test in `ont_release_readiness.rs` (\"see the PR thread (re-run at the final head)\"). The test's green status is therefore asserted, not recorded; CI's required checks must supply it.", + "grounding": "cited", + "fix": "Record the `cargo test -p aprender-contracts-cli --test ont_release_readiness` pass count at the final head in §8." + } + ], + "raw_bytes": 7103, + "err_bytes": 0, "envelope_status": "SUCCESS", "verdict_source": "structured_output", + "executor": "claude-code", "grounding_check": "parity", - "model": "gemini-3.1-pro-high", - "model_measured": "gemini-3.1-pro-high", - "model_source": "measured", - "family": "gemini", - "role": "independent", + "role": "counted", + "model": "claude-opus-5-5", + "model_measured": "claude-opus-5-5", + "model_source": "flag", + "family": "claude", + "brief_sha256": "8fbdb9ff2c4381dca7e69a1b7fc21d58116ff831bb58f14d08dcc66f5284a975", + "trace": { + "input_sha256": "0e7fd102e6601d03555c0dd0dd479ae66e33f7d73865e94027506da22bb8e039", + "output_sha256": "eed9e065d60dd14fe4e730e43a18bda087f9b47c0febffa1ce9518446e6064fe", + "store": null, + "store_why": "almacen not provisioned yet (infra-27): the blobs stay in the round's gitignored .lanes dir" + }, "fallback": { - "judged_by": "gemini-3.1-pro-high", + "judged_by": "claude-opus-5-5", "exhausted": false, "attempts": [ { - "model": "gemini-3.1-pro-high", - "family": "gemini", + "model": "claude-opus-5-5", + "family": "claude", "outcome": "answered" } ], @@ -49,25 +73,33 @@ "lane": 2, "status": "SUCCESS", "verdict": "PASS", - "summary": "The diff perfectly implements the requirements of PMAT-3715 and the receipt claims. The SHACL shape enforces the required matrix of cells, and all edge cases (missing, skipped, fallback, stale receipts, kernel architecture mismatches, tokenizer parity) are verified in the test suite to turn the gate red and name the cell. There are no doctrinal violations, weakened gates, or unauthorized scopes. The gate works exactly as requested.", + "summary": "The diff implements all ticketed requirements and receipt claims. Report-only mode is deleted from release_readiness.sh; R8 correctly refuses WARN rows and rc=0 without the enforced-pass marker; the ladder gate detects de-claimed models still being claimed; and GB10 dense withdrawals are documented and data-gated. Test coverage is comprehensive, including the critical withdrawn-model scenario.", "findings": [], - "raw_bytes": 3819, - "err_bytes": 100, + "raw_bytes": 1701, + "err_bytes": 0, "envelope_status": "SUCCESS", "verdict_source": "structured_output", + "executor": "claude-code", "grounding_check": "parity", - "model": "gemini-3.1-pro-low", - "model_measured": "gemini-3.1-pro-low", - "model_source": "measured", - "family": "gemini", - "role": "independent", + "role": "counted", + "model": "claude-haiku-4-5", + "model_measured": "claude-haiku-4-5", + "model_source": "flag", + "family": "claude", + "brief_sha256": "8fbdb9ff2c4381dca7e69a1b7fc21d58116ff831bb58f14d08dcc66f5284a975", + "trace": { + "input_sha256": "8dbcedc3d329baf75e449ad348da55f205b0310f5e6da7a911731175e0e36b9f", + "output_sha256": "c3f4c48b7882720bac05ac60e82a15f697e10bf2e8f95e7ae172ed93a62284b4", + "store": null, + "store_why": "almacen not provisioned yet (infra-27): the blobs stay in the round's gitignored .lanes dir" + }, "fallback": { - "judged_by": "gemini-3.1-pro-low", + "judged_by": "claude-haiku-4-5", "exhausted": false, "attempts": [ { - "model": "gemini-3.1-pro-low", - "family": "gemini", + "model": "claude-haiku-4-5", + "family": "claude", "outcome": "answered" } ], @@ -78,30 +110,33 @@ "lane": 3, "status": "SUCCESS", "verdict": "PASS", - "summary": "The audit prompt file was fully read, the repository builds, and all tests pass. No issues found.", + "summary": "The diff accurately implements the receipt claims. It strictly strengthens the gating logic by enforcing release readiness without waiver or report mode, verifying the enforcement string, properly recording withdrawn cells in data, and adding strict verification to ensure a declaimed arch/host combination is not contradictorily claimed elsewhere. No claims were unbacked, no scope exceeded, and no gates weakened.", "findings": [], - "raw_bytes": 1862, - "err_bytes": 100, + "raw_bytes": 3816, + "err_bytes": 62, "envelope_status": "SUCCESS", "verdict_source": "structured_output", - "model": "gpt-oss-120b-medium", - "model_measured": "gpt-oss-120b-medium", - "model_source": "measured", - "family": "openai", + "grounding_check": "parity", "role": "independent", + "model": "gemini-3.1-pro-high", + "model_measured": "gemini-3.1-pro-high", + "model_source": "measured", + "family": "gemini", + "executor": "agy", + "brief_sha256": "8fbdb9ff2c4381dca7e69a1b7fc21d58116ff831bb58f14d08dcc66f5284a975", + "trace": { + "input_sha256": "8f0c8a19747d3ab36e8922570d4b4ffad8e8752e98101af16c1431c32fa71289", + "output_sha256": "3725796da773be5d66b50f7bc4b631e3858458a0797382e0ffb5067df099cb6a", + "store": null, + "store_why": "almacen not provisioned yet (infra-27): the blobs stay in the round's gitignored .lanes dir" + }, "fallback": { - "judged_by": "gpt-oss-120b-medium", + "judged_by": "gemini-3.1-pro-high", "exhausted": false, "attempts": [ { - "model": "gpt-oss-120b-medium", - "family": "openai", - "outcome": "capacity", - "reason": "Our servers are experiencing high traffic right now, please try again in a minute. (UNAVAILABLE (code 503): No capacity available for model gpt-oss-120b-medium on the server)" - }, - { - "model": "gpt-oss-120b-medium", - "family": "openai", + "model": "gemini-3.1-pro-high", + "family": "gemini", "outcome": "answered" } ], @@ -110,30 +145,50 @@ } ], "dissent": [], - "dedup": [], + "dedup": [ + { + "file": "contracts/model-capability-ladder-v1.yaml", + "line": 256, + "lanes_agreeing": [ + 1 + ], + "claims": [ + "The new qwen2/gx10 de-claim `why` text says \"fixed in 0.70.1 (#4590)\". Receipt §8 says the wording is \"planned for 0.70.1\", not \"fixed\". That claim is only true of README and CHANGELOG, not of the contract data." + ] + }, + { + "file": "docs/audits/impl-PMAT-3715-receipt.md", + "line": 445, + "lanes_agreeing": [ + 1 + ], + "claims": [ + "The receipt gives no measured result for the one Rust change, the new test in `ont_release_readiness.rs` (\"see the PR thread (re-run at the final head)\"). The test's green status is therefore asserted, not recorded; CI's required checks must supply it." + ] + } + ], "uncovered": [], "coverage_source": "lanes", "partial": false, - "partial_reasons": [ - "lane 3: grounding NOT COMPARED — structured_output holds a verdict but no single verdict object could be read out of .response (1862 raw bytes), so agy's labels stand unchecked for this lane (PMAT-082)", - "lane 3: judged by gpt-oss-120b-medium after falling through gpt-oss-120b-medium (capacity: Our servers are experiencing high traffic right now, please try again in a minute. (UNAVAILABLE (code 503): No capacity available for model gpt-oss-120b-medium on the server)) (PMAT-321)" - ], + "partial_reasons": [], "fallback": { - "same_family_width": 1, + "same_family_width": 2, "chain": [ { - "model": "gemini-3.1-pro-high", - "family": "gemini", - "disposition": "configured" + "model": "claude-opus-5-5", + "family": "claude", + "disposition": "claude-code", + "why": "run by Claude Code on its own budget: a configured seat runs every round, a fallback step only when every non-Claude family is measured out (PMAT-360, operator standing rule)" }, { - "model": "gemini-3.1-pro-low", - "family": "gemini", - "disposition": "configured" + "model": "claude-haiku-4-5", + "family": "claude", + "disposition": "claude-code", + "why": "run by Claude Code on its own budget: a configured seat runs every round, a fallback step only when every non-Claude family is measured out (PMAT-360, operator standing rule)" }, { - "model": "gpt-oss-120b-medium", - "family": "openai", + "model": "gemini-3.1-pro-high", + "family": "gemini", "disposition": "configured" }, { @@ -141,18 +196,6 @@ "family": "qwen", "disposition": "not-run", "why": "no quorum.local_lane in the config — the aprender lane has no model to load" - }, - { - "model": "claude-opus-4-6-thinking", - "family": "claude", - "disposition": "width", - "why": "same family as the author: at most 1 lane, recorded role width, counted toward no floor (R-15a)" - }, - { - "model": "claude-sonnet-4-6", - "family": "claude", - "disposition": "width", - "why": "same family as the author: at most 1 lane, recorded role width, counted toward no floor (R-15a)" } ], "precheck": [ @@ -161,17 +204,50 @@ "model": "gemini-3.1-pro-high", "probe": 1, "outcome": "live" - }, - { - "family": "openai", - "model": "gpt-oss-120b-medium", - "probe": 1, - "outcome": "live" } ], + "degraded": null, "prah": { "source": "install-receipt", "path": "/home/noah/.claude/skills/paiml-implement/bin/prah" + }, + "tier1": { + "agy_tier1_only": true, + "tier1": true, + "matched": [ + "scripts/release/release_readiness.sh" + ], + "paths": 26, + "patterns": [ + "^\\.github/workflows/", + "(^|/)release[^/]*\\.(ya?ml|sh|rs|toml)$", + "cuda|kernel|\\.cu$|\\.ptx$", + "(^|/)[^/]*(gate|guard)[^/]*\\.(sh|rs|py)$|^hooks/", + "security|secret|credential|(^|/)deny\\.toml$", + "^skills/quorum-review/|(^|/)(receipt-lint|roadmap-lint|release-lint|kind-gate|model-gate)[^/]*$|^crates/prah-lint/" + ], + "builtin": "^skills/quorum-review/|(^|/)skills/paiml-implement/config\\.json$|(^|/)(modellib|lane-reduce|lane-fallback|lane-group|agy-lane|cc-lane|receipt-lint|route|quota)\\.sh$|^crates/prah-lint/" + }, + "bucket": { + "ledger": "/home/noah/.local/state/paiml-implement/agy-bucket.jsonl", + "window_s": 18000, + "pace": "off", + "buckets": { + "gemini": { + "bucket": "gemini", + "budget": 86, + "basis": "[U] default 30, raised to the 86 calls a window has made without a 429; no window reached one", + "window_start": 1790702669, + "window_calls": 16, + "window_429": 0, + "hour_calls": 6, + "share": 17, + "want": 2, + "state": "open", + "reason": "" + } + }, + "closed": [] } }, "auto_merge": { @@ -180,8 +256,69 @@ "disarmed": false, "note": "auto-merge not armed" }, + "cheap_seat": "claude-haiku-4-5", + "advisory_lane": { + "state": "answered", + "counts": false, + "row": { + "Verdict": { + "verdict": "PASS", + "cell": "gx10-cuda", + "backend": "cuda" + } + }, + "verdict": "PASS", + "why": null, + "served_by": "gx10-cuda", + "gpu_proof": { + "used_gpu_probe": true, + "server_holds_gpu": true, + "used_gpu_source": "completions probe (chat omits used_gpu, aprender#4146)", + "trace_lines": [ + "1083089, 309 MiB" + ] + }, + "apr": "0.70.0", + "apr_binary_sha256": "6b2a7dc66c38adcca20b315beac70a1a31e7bd0af2a1cc5c2a7730ca6086d417", + "apr_tag": "v0.70.0-dev.3990584f0", + "apr_commit": "3990584f0bf5af6bca34ea9298df92177f3ef79a", + "apr_build_why": null, + "model": "/home/noah/data/models/Qwen3.5-4B-Q4_K_M.gguf", + "model_sha256": "00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4", + "rc": 0, + "wall_s": 50, + "collected_s": 856, + "budget_s": 164, + "budget_basis": "ledger: 2 x p95 82 s over 103 gx10-cuda Verdict rows", + "brief": { + "bytes": 103770, + "sent_bytes": 103939, + "max_bytes": 196608 + }, + "trace": { + "input_sha256": "33fd5dbbbda8f82260fc4419cf55f44cebaaa6f28de772dcc4a432281008a1bc", + "raw": "advisory.json" + }, + "attempts": [ + { + "host": "gx10", + "ok": true, + "wall_s": 50, + "used_gpu": true, + "ts": "2026-09-29T18:58:48.387Z", + "wall_ms": 50419, + "request_id": "01a0ee88-a7c0-70d0-93a5-91affb567af0", + "load1": 58.64, + "cold": true + } + ], + "ledger": "/home/noah/.local/state/paiml-implement/advisory-ledger.jsonl", + "raw": "advisory.json", + "agrees_with_counted": true, + "counted": "PASS" + }, "lint": { "ok": true, - "output": "receipt complete: kind=artifact lanes=3 author=claude-opus-5/claude" + "output": "receipt complete: kind=artifact lanes=3 author=claude-sonnet-5-5/claude same_family=2/2" } } diff --git a/evidence/3715-declaim-planted-red/declaim_real.out b/evidence/3715-declaim-planted-red/declaim_real.out new file mode 100644 index 0000000000..3be3cac9b7 --- /dev/null +++ b/evidence/3715-declaim-planted-red/declaim_real.out @@ -0,0 +1,6 @@ +real contract rc=0 declaimed=[('gx10', 'qwen2'), ('gx10', 'qwen3'), ('gx10', 'qwen3moe')] +PLANTED qwen3moe rung (all hosts) rc=1 declaimed=[('gx10', 'qwen2'), ('gx10', 'qwen3'), ('gx10', 'qwen3moe')] + FAIL rung qwen3moe-30b-a3b-q4km still claims arch qwen3moe on gx10, which cells.declaimed withdraws (#3715) -- narrow its hosts: or drop the de-claim +planted rung narrowed to lambda rc=0 declaimed=[('gx10', 'qwen2'), ('gx10', 'qwen3'), ('gx10', 'qwen3moe')] +PLANTED rep, de-claimed on all hosts rc=1 declaimed=[('gx10', 'qwen2'), ('gx10', 'qwen3'), ('gx10', 'qwen3moe'), ('lambda', 'qwen3moe')] + FAIL long_rungs_for.representatives.qwen3moe still claims long rungs for arch qwen3moe, de-claimed on every required host diff --git a/evidence/3715-declaim-planted-red/declaim_real.py b/evidence/3715-declaim-planted-red/declaim_real.py new file mode 100644 index 0000000000..a7b45a9991 --- /dev/null +++ b/evidence/3715-declaim-planted-red/declaim_real.py @@ -0,0 +1,20 @@ +import sys, copy, yaml +sys.path.insert(0, "scripts/lib"); import model_ladder_cells as J +L = yaml.safe_load(open("contracts/model-capability-ladder-v1.yaml"))["ladder"] +def run(L, tag): + lines = [] + dec, drc = J.declaimed(L.get("cells") or {}, {h["id"] for h in L["hosts"]}, lines.append) + rc = drc or J.declaim_still_claimed(L, dec, lines.append) + print(f"{tag:34} rc={rc} declaimed={sorted(dec)}"); [print(" ", l) for l in lines] + return rc +a = run(L, "real contract") +P = copy.deepcopy(L) # planted: a qwen3moe rung with no hosts: key claims it on every host, gx10 included +P["rungs"].append({"id": "qwen3moe-30b-a3b-q4km", "family": "qwen3moe", "arch": "qwen3moe", "required": True, + "gguf": "Qwen3-30B-A3B-Instruct-2507-Q4_K_M.gguf", "backends": ["cpu", "cuda"]}) +b = run(P, "PLANTED qwen3moe rung (all hosts)") +Q = copy.deepcopy(P); Q["rungs"][-1]["hosts"] = ["lambda"] +c = run(Q, "planted rung narrowed to lambda") +R = copy.deepcopy(L); R["cells"]["long_rungs_for"]["representatives"]["qwen3moe"] = "Qwen3-30B-A3B-Instruct-2507-Q4_K_M.gguf" +R["cells"]["declaimed"].append(dict(R["cells"]["declaimed"][-1], host="lambda")) +d = run(R, "PLANTED rep, de-claimed on all hosts") +sys.exit(0 if (a, b, c, d) == (0, 1, 0, 1) else 1) diff --git a/scripts/check_model_ladder.sh b/scripts/check_model_ladder.sh index cba67f70fd..e5df3ad560 100755 --- a/scripts/check_model_ladder.sh +++ b/scripts/check_model_ladder.sh @@ -890,6 +890,7 @@ if [ "$SELF_TEST" = 1 ]; then cmutant rungs-floor red-cells-rung-dropped-vs-main 's/ if gone:/ if False:/' cmutant declaim-skip green-cells-declaimed 's/ if d: # printed on every run/ if False: # printed on every run/' cmutant declaim-bare red-cells-declaimed-bare 's/ if bad: # a bare de-claim/ if False: # a bare de-claim/' + cmutant declaim-claimed red-cells-declaimed-still-claimed 's/ if d: # a claim the de-claim withdraws/ if False: # a claim the de-claim withdraws/' mutant declaim-host declaimed-host-on-rung 's/ gone = {g for g in gone if (g, r.get("arch")) not in dcl}/ gone = set(gone)/' # #3957 F4/F8: the CRUX join (scripts/lib/model_ladder_crux.py), each rule deleted in a copy # imported through MODEL_LADDER_CRUX_LIB; the case that names the rule must go RED. diff --git a/scripts/check_publish_preflight.sh b/scripts/check_publish_preflight.sh index 1ad02733aa..2ae0bd330d 100755 --- a/scripts/check_publish_preflight.sh +++ b/scripts/check_publish_preflight.sh @@ -311,9 +311,21 @@ rule_r8() { receipt="$(newest_receipt "${PUBLISH_PREFLIGHT_RECEIPT_DIR:-$root/.dogfood}")" out="$(bash "$wrapper" --root "$root" --version "$version" --commit "$head" ${receipt:+--dogfood-receipt "$receipt"} 2>&1)"; rc=$? printf '%s\n' "$out" - [ "$rc" -eq 0 ] && return 0 - echo "FAIL R8 the release-readiness wrapper exited $rc (1 Fail under enforce, 2 could not judge, 3 caller error): none is a pass" - return 1 + if [ "$rc" -ne 0 ]; then + echo "FAIL R8 the release-readiness wrapper exited $rc (1 Fail, 2 could not judge, 3 caller error): none is a pass" + return 1 + fi + # R10/L19 (operator 2026-09-28): report-only is a waiver and a waiver is a stop. rc 0 alone is not + # a pass: the wrapper must have printed the enforced Pass for exactly this version and HEAD. + if printf '%s\n' "$out" | grep -qE '^WARN +R8 '; then + echo "FAIL R8 the release-readiness wrapper printed a WARN R8 row: report-only is a waiver, not a pass" + return 1 + fi + if ! printf '%s\n' "$out" | grep -qF "ok R8 #3715 ENFORCE PASS version=$version commit=$head pv="; then + echo "FAIL R8 the release-readiness wrapper exited 0 without '#3715 ENFORCE PASS' for $version at $head" + return 1 + fi + return 0 } gate() { @@ -506,10 +518,9 @@ fi exit "${FX_LADDER_RC:-0}" FXJUDGE # R8's wrapper: it must be asked about THIS root, version 1.2.3, HEAD and the newest dogfood - # receipt, and it answers FX_READINESS_RC (default 0; FX_READINESS_WARN=1 prints the - # report-mode WARN row a Fail verdict yields under DEFAULT_MODE=report; the committed mode is - # enforce since #3715 B1, so that row is only reachable by a hand-edited wrapper, and R8 still - # refuses on any non-zero exit whatever the mode). + # receipt, and it answers FX_READINESS_RC (default 0, with the enforced-Pass line). FX_READINESS_WARN=1 + # prints a WARN R8 row and exits 0: only a hand-edited wrapper can, and R8 refuses it (R10). + # FX_READINESS_NO_ENFORCE=1 exits 0 without the enforced-Pass line: refused too. mkdir -p "$1/scripts/release" cat > "$1/scripts/release/release_readiness.sh" <<'FXREADY' #!/usr/bin/env bash @@ -519,6 +530,8 @@ want="--version 1.2.3 --commit $(git -C "$me" rev-parse HEAD) --dogfood-receipt" case "${*:3}" in "$want "*.dogfood/receipt-*.json) : ;; *) echo "FAIL R8 wrapper asked: $*"; exit 3 ;; esac [ "${FX_READINESS_WARN:-0}" = 1 ] && echo "WARN R8 REPORT-ONLY release-readiness-v1 for 1.2.3: Fail, 3 violation(s): cell=3" [ "${FX_READINESS_RC:-0}" = 0 ] && [ "${FX_READINESS_WARN:-0}" = 0 ] && echo "ok R8 release-readiness-v1 for 1.2.3: Pass" +[ "${FX_READINESS_RC:-0}" = 0 ] && [ "${FX_READINESS_WARN:-0}" = 0 ] && [ "${FX_READINESS_NO_ENFORCE:-0}" = 0 ] \ + && echo "ok R8 #3715 ENFORCE PASS version=1.2.3 commit=$(git -C "$me" rev-parse HEAD) pv=pv-fixture out_sha256=0" exit "${FX_READINESS_RC:-0}" FXREADY } @@ -698,7 +711,8 @@ FXREADY # green row (the stub exits 3 unless it is asked about this root, 1.2.3, HEAD and the dogfood receipt). d="$tmp/r8"; build_repo "$d" row r8_pass_is_named 0 "ok R8 release-readiness-v1 for 1.2.3: Pass" "$d" - FX_READINESS_WARN=1 row r8_report_mode_warn_passes 0 "WARN R8 REPORT-ONLY" "$d" + FX_READINESS_WARN=1 row r8_report_mode_warn_refuses 1 "report-only is a waiver" "$d" + FX_READINESS_NO_ENFORCE=1 row r8_rc0_without_enforce_pass_refuses 1 "without '#3715 ENFORCE PASS'" "$d" FX_READINESS_RC=1 row r8_enforced_fail_refuses 1 "FAIL R8 the release-readiness wrapper exited 1" "$d" FX_READINESS_RC=2 row r8_could_not_judge_refuses 1 "FAIL R8 the release-readiness wrapper exited 2" "$d" FX_READINESS_RC=3 row r8_caller_error_refuses 1 "FAIL R8 the release-readiness wrapper exited 3" "$d" diff --git a/scripts/check_release_models_t1.sh b/scripts/check_release_models_t1.sh index 2b39ba00ac..6b19770fad 100755 --- a/scripts/check_release_models_t1.sh +++ b/scripts/check_release_models_t1.sh @@ -363,7 +363,7 @@ r_stops() { # TAG AUTOPILOT MODELS_T1 NEEDLE ENV... -> 0 when the step STOPped n return 0 } r_pass() { r_goes "rpass-$1" "$2" "$3" ok "ok R8 release-readiness-v1 Pass"; } -r_warn() { r_goes "rwarn-$1" "$2" "$3" WARN "WARN R8 REPORT-ONLY" 'FX_RR_LINE=WARN R8 REPORT-ONLY fixture Fail'; } +r_warn() { r_stops "rwarn-$1" "$2" "$3" "T-1 release-readiness-v1 printed a WARN R8 row" 'FX_RR_LINE=WARN R8 REPORT-ONLY fixture Fail'; } r_fail() { r_stops "rfail-$1" "$2" "$3" "T-1 release-readiness-v1 rc=1" FX_RR_RC=1 'FX_RR_LINE=FAIL R8 fixture Fail'; } r_decline() { r_stops "rdecl-$1" "$2" "$3" "T-1 release-readiness-v1 rc=2" FX_RR_RC=2 'FX_RR_LINE=FAIL R8 fixture decline'; } r_no_dr() { r_stops "rnodr-$1" "$2" "$3" "T-1 readiness: no dogfood receipt" FX_NO_DR=1; } @@ -379,7 +379,7 @@ for spec in "green-pair green_pair" "gx10-unreachable unreachable" "build-fails set -- $spec msg=$($2 real "$AUTOPILOT" "$MODELS"); row "$1" "$?" "$msg" done -for spec in "readiness-pass r_pass" "readiness-warn-continues r_warn" "readiness-fail-stops r_fail" \ +for spec in "readiness-pass r_pass" "readiness-warn-stops r_warn" "readiness-fail-stops r_fail" \ "readiness-decline-stops r_decline" "readiness-no-dogfood-receipt r_no_dr" \ "readiness-stale-apr-stops r_stale"; do set -- $spec diff --git a/scripts/lib/model_ladder_cases/green-cells-declaimed/ladder.yaml b/scripts/lib/model_ladder_cases/green-cells-declaimed/ladder.yaml index 0e8792c81f..03eb5e0579 100644 --- a/scripts/lib/model_ladder_cases/green-cells-declaimed/ladder.yaml +++ b/scripts/lib/model_ladder_cases/green-cells-declaimed/ladder.yaml @@ -38,6 +38,8 @@ ladder: - cpu - cuda gguf: c.gguf + hosts: + - lambda id: ctrl-q4km required: true sha256: '0000000000000000000000000000000000000000000000000000000000000000' diff --git a/scripts/lib/model_ladder_cases/red-cells-declaimed-still-claimed/certification.json b/scripts/lib/model_ladder_cases/red-cells-declaimed-still-claimed/certification.json new file mode 100644 index 0000000000..7c65c27b9e --- /dev/null +++ b/scripts/lib/model_ladder_cases/red-cells-declaimed-still-claimed/certification.json @@ -0,0 +1,11 @@ +{ + "schema": "crux-prompt-certification/v1", + "admitted_by_sha": { + "0000000000000000000000000000000000000000000000000000000000000000": [ + "fixture-control" + ], + "1111111111111111111111111111111111111111111111111111111111111111": [ + "fixture-control" + ] + } +} \ No newline at end of file diff --git a/scripts/lib/model_ladder_cases/red-cells-declaimed-still-claimed/context-rungs.json b/scripts/lib/model_ladder_cases/red-cells-declaimed-still-claimed/context-rungs.json new file mode 100644 index 0000000000..9b2cfacc39 --- /dev/null +++ b/scripts/lib/model_ladder_cases/red-cells-declaimed-still-claimed/context-rungs.json @@ -0,0 +1,38 @@ +{ + "schema": "apr-release-context-rungs/v1", + "rungs": [ + { + "id": "4k", + "tokens": 4096 + }, + { + "id": "60k", + "tokens": 61440, + "long": true + }, + { + "id": "consumer-max", + "long": true + }, + { + "id": "declared", + "long": true + } + ], + "consumers": [ + { + "name": "rah", + "max_prompt_tokens": 148000, + "derived_from": [ + "https://github.com/paiml/paiml-implement/issues/322" + ] + }, + { + "name": "arbiter", + "max_prompt_tokens": 60000, + "derived_from": [ + "https://github.com/paiml/aprender/issues/3716" + ] + } + ] +} \ No newline at end of file diff --git a/scripts/lib/model_ladder_cases/red-cells-declaimed-still-claimed/crux/gx10-cpu.json b/scripts/lib/model_ladder_cases/red-cells-declaimed-still-claimed/crux/gx10-cpu.json new file mode 100644 index 0000000000..ee0134492a --- /dev/null +++ b/scripts/lib/model_ladder_cases/red-cells-declaimed-still-claimed/crux/gx10-cpu.json @@ -0,0 +1,102 @@ +{ + "schema": "crux-inference-receipt/v1", + "host": "gx10", + "backend": "cpu", + "apr": { + "sha": "ca5eca5eca5eca5eca5eca5eca5eca5eca5eca5e" + }, + "cells": [ + { + "key": { + "model_sha256": "0000000000000000000000000000000000000000000000000000000000000000", + "host": "gx10", + "verb": "run", + "thinking": "off", + "rung": "golden", + "prompt_id": "fixture-control" + }, + "verdict": "GREEN" + }, + { + "key": { + "model_sha256": "0000000000000000000000000000000000000000000000000000000000000000", + "host": "gx10", + "verb": "chat", + "thinking": "off", + "rung": "golden", + "prompt_id": "fixture-control" + }, + "verdict": "GREEN" + }, + { + "key": { + "model_sha256": "0000000000000000000000000000000000000000000000000000000000000000", + "host": "gx10", + "verb": "serve run", + "thinking": "off", + "rung": "golden", + "prompt_id": "fixture-control" + }, + "verdict": "GREEN" + }, + { + "key": { + "model_sha256": "0000000000000000000000000000000000000000000000000000000000000000", + "host": "gx10", + "verb": "code", + "thinking": "off", + "rung": "golden", + "prompt_id": "fixture-control" + }, + "verdict": "GREEN" + }, + { + "key": { + "model_sha256": "1111111111111111111111111111111111111111111111111111111111111111", + "host": "gx10", + "verb": "run", + "thinking": "off", + "rung": "golden", + "prompt_id": "fixture-control" + }, + "verdict": "GREEN" + }, + { + "key": { + "model_sha256": "1111111111111111111111111111111111111111111111111111111111111111", + "host": "gx10", + "verb": "chat", + "thinking": "off", + "rung": "golden", + "prompt_id": "fixture-control" + }, + "verdict": "GREEN" + }, + { + "key": { + "model_sha256": "1111111111111111111111111111111111111111111111111111111111111111", + "host": "gx10", + "verb": "serve run", + "thinking": "off", + "rung": "golden", + "prompt_id": "fixture-control" + }, + "verdict": "GREEN" + }, + { + "key": { + "model_sha256": "1111111111111111111111111111111111111111111111111111111111111111", + "host": "gx10", + "verb": "code", + "thinking": "off", + "rung": "golden", + "prompt_id": "fixture-control" + }, + "verdict": "GREEN" + } + ], + "summary": { + "verdict": "PASS", + "cells": 8 + } +} \ No newline at end of file diff --git a/scripts/lib/model_ladder_cases/red-cells-declaimed-still-claimed/crux/gx10-gpu.json b/scripts/lib/model_ladder_cases/red-cells-declaimed-still-claimed/crux/gx10-gpu.json new file mode 100644 index 0000000000..8e39351285 --- /dev/null +++ b/scripts/lib/model_ladder_cases/red-cells-declaimed-still-claimed/crux/gx10-gpu.json @@ -0,0 +1,102 @@ +{ + "schema": "crux-inference-receipt/v1", + "host": "gx10", + "backend": "gpu", + "apr": { + "sha": "ca5eca5eca5eca5eca5eca5eca5eca5eca5eca5e" + }, + "cells": [ + { + "key": { + "model_sha256": "0000000000000000000000000000000000000000000000000000000000000000", + "host": "gx10", + "verb": "run", + "thinking": "off", + "rung": "golden", + "prompt_id": "fixture-control" + }, + "verdict": "GREEN" + }, + { + "key": { + "model_sha256": "0000000000000000000000000000000000000000000000000000000000000000", + "host": "gx10", + "verb": "chat", + "thinking": "off", + "rung": "golden", + "prompt_id": "fixture-control" + }, + "verdict": "GREEN" + }, + { + "key": { + "model_sha256": "0000000000000000000000000000000000000000000000000000000000000000", + "host": "gx10", + "verb": "serve run", + "thinking": "off", + "rung": "golden", + "prompt_id": "fixture-control" + }, + "verdict": "GREEN" + }, + { + "key": { + "model_sha256": "0000000000000000000000000000000000000000000000000000000000000000", + "host": "gx10", + "verb": "code", + "thinking": "off", + "rung": "golden", + "prompt_id": "fixture-control" + }, + "verdict": "GREEN" + }, + { + "key": { + "model_sha256": "1111111111111111111111111111111111111111111111111111111111111111", + "host": "gx10", + "verb": "run", + "thinking": "off", + "rung": "golden", + "prompt_id": "fixture-control" + }, + "verdict": "GREEN" + }, + { + "key": { + "model_sha256": "1111111111111111111111111111111111111111111111111111111111111111", + "host": "gx10", + "verb": "chat", + "thinking": "off", + "rung": "golden", + "prompt_id": "fixture-control" + }, + "verdict": "GREEN" + }, + { + "key": { + "model_sha256": "1111111111111111111111111111111111111111111111111111111111111111", + "host": "gx10", + "verb": "serve run", + "thinking": "off", + "rung": "golden", + "prompt_id": "fixture-control" + }, + "verdict": "GREEN" + }, + { + "key": { + "model_sha256": "1111111111111111111111111111111111111111111111111111111111111111", + "host": "gx10", + "verb": "code", + "thinking": "off", + "rung": "golden", + "prompt_id": "fixture-control" + }, + "verdict": "GREEN" + } + ], + "summary": { + "verdict": "PASS", + "cells": 8 + } +} \ No newline at end of file diff --git a/scripts/lib/model_ladder_cases/red-cells-declaimed-still-claimed/crux/lambda-cpu.json b/scripts/lib/model_ladder_cases/red-cells-declaimed-still-claimed/crux/lambda-cpu.json new file mode 100644 index 0000000000..f10933edc6 --- /dev/null +++ b/scripts/lib/model_ladder_cases/red-cells-declaimed-still-claimed/crux/lambda-cpu.json @@ -0,0 +1,102 @@ +{ + "schema": "crux-inference-receipt/v1", + "host": "lambda", + "backend": "cpu", + "apr": { + "sha": "ca5eca5eca5eca5eca5eca5eca5eca5eca5eca5e" + }, + "cells": [ + { + "key": { + "model_sha256": "0000000000000000000000000000000000000000000000000000000000000000", + "host": "lambda", + "verb": "run", + "thinking": "off", + "rung": "golden", + "prompt_id": "fixture-control" + }, + "verdict": "GREEN" + }, + { + "key": { + "model_sha256": "0000000000000000000000000000000000000000000000000000000000000000", + "host": "lambda", + "verb": "chat", + "thinking": "off", + "rung": "golden", + "prompt_id": "fixture-control" + }, + "verdict": "GREEN" + }, + { + "key": { + "model_sha256": "0000000000000000000000000000000000000000000000000000000000000000", + "host": "lambda", + "verb": "serve run", + "thinking": "off", + "rung": "golden", + "prompt_id": "fixture-control" + }, + "verdict": "GREEN" + }, + { + "key": { + "model_sha256": "0000000000000000000000000000000000000000000000000000000000000000", + "host": "lambda", + "verb": "code", + "thinking": "off", + "rung": "golden", + "prompt_id": "fixture-control" + }, + "verdict": "GREEN" + }, + { + "key": { + "model_sha256": "1111111111111111111111111111111111111111111111111111111111111111", + "host": "lambda", + "verb": "run", + "thinking": "off", + "rung": "golden", + "prompt_id": "fixture-control" + }, + "verdict": "GREEN" + }, + { + "key": { + "model_sha256": "1111111111111111111111111111111111111111111111111111111111111111", + "host": "lambda", + "verb": "chat", + "thinking": "off", + "rung": "golden", + "prompt_id": "fixture-control" + }, + "verdict": "GREEN" + }, + { + "key": { + "model_sha256": "1111111111111111111111111111111111111111111111111111111111111111", + "host": "lambda", + "verb": "serve run", + "thinking": "off", + "rung": "golden", + "prompt_id": "fixture-control" + }, + "verdict": "GREEN" + }, + { + "key": { + "model_sha256": "1111111111111111111111111111111111111111111111111111111111111111", + "host": "lambda", + "verb": "code", + "thinking": "off", + "rung": "golden", + "prompt_id": "fixture-control" + }, + "verdict": "GREEN" + } + ], + "summary": { + "verdict": "PASS", + "cells": 8 + } +} \ No newline at end of file diff --git a/scripts/lib/model_ladder_cases/red-cells-declaimed-still-claimed/crux/lambda-gpu.json b/scripts/lib/model_ladder_cases/red-cells-declaimed-still-claimed/crux/lambda-gpu.json new file mode 100644 index 0000000000..5512fda79c --- /dev/null +++ b/scripts/lib/model_ladder_cases/red-cells-declaimed-still-claimed/crux/lambda-gpu.json @@ -0,0 +1,102 @@ +{ + "schema": "crux-inference-receipt/v1", + "host": "lambda", + "backend": "gpu", + "apr": { + "sha": "ca5eca5eca5eca5eca5eca5eca5eca5eca5eca5e" + }, + "cells": [ + { + "key": { + "model_sha256": "0000000000000000000000000000000000000000000000000000000000000000", + "host": "lambda", + "verb": "run", + "thinking": "off", + "rung": "golden", + "prompt_id": "fixture-control" + }, + "verdict": "GREEN" + }, + { + "key": { + "model_sha256": "0000000000000000000000000000000000000000000000000000000000000000", + "host": "lambda", + "verb": "chat", + "thinking": "off", + "rung": "golden", + "prompt_id": "fixture-control" + }, + "verdict": "GREEN" + }, + { + "key": { + "model_sha256": "0000000000000000000000000000000000000000000000000000000000000000", + "host": "lambda", + "verb": "serve run", + "thinking": "off", + "rung": "golden", + "prompt_id": "fixture-control" + }, + "verdict": "GREEN" + }, + { + "key": { + "model_sha256": "0000000000000000000000000000000000000000000000000000000000000000", + "host": "lambda", + "verb": "code", + "thinking": "off", + "rung": "golden", + "prompt_id": "fixture-control" + }, + "verdict": "GREEN" + }, + { + "key": { + "model_sha256": "1111111111111111111111111111111111111111111111111111111111111111", + "host": "lambda", + "verb": "run", + "thinking": "off", + "rung": "golden", + "prompt_id": "fixture-control" + }, + "verdict": "GREEN" + }, + { + "key": { + "model_sha256": "1111111111111111111111111111111111111111111111111111111111111111", + "host": "lambda", + "verb": "chat", + "thinking": "off", + "rung": "golden", + "prompt_id": "fixture-control" + }, + "verdict": "GREEN" + }, + { + "key": { + "model_sha256": "1111111111111111111111111111111111111111111111111111111111111111", + "host": "lambda", + "verb": "serve run", + "thinking": "off", + "rung": "golden", + "prompt_id": "fixture-control" + }, + "verdict": "GREEN" + }, + { + "key": { + "model_sha256": "1111111111111111111111111111111111111111111111111111111111111111", + "host": "lambda", + "verb": "code", + "thinking": "off", + "rung": "golden", + "prompt_id": "fixture-control" + }, + "verdict": "GREEN" + } + ], + "summary": { + "verdict": "PASS", + "cells": 8 + } +} \ No newline at end of file diff --git a/scripts/lib/model_ladder_cases/red-cells-declaimed-still-claimed/expected_rc b/scripts/lib/model_ladder_cases/red-cells-declaimed-still-claimed/expected_rc new file mode 100644 index 0000000000..d00491fd7e --- /dev/null +++ b/scripts/lib/model_ladder_cases/red-cells-declaimed-still-claimed/expected_rc @@ -0,0 +1 @@ +1 diff --git a/scripts/lib/model_ladder_cases/red-cells-declaimed-still-claimed/ladder.yaml b/scripts/lib/model_ladder_cases/red-cells-declaimed-still-claimed/ladder.yaml new file mode 100644 index 0000000000..0e8792c81f --- /dev/null +++ b/scripts/lib/model_ladder_cases/red-cells-declaimed-still-claimed/ladder.yaml @@ -0,0 +1,51 @@ +ladder: + cells: + declaimed: + - arch: qwen2 + host: gx10 + issue: 4590 + until: 0.70.1 + why: 'fixture: correct but slow on this host' + long_rungs_for: + families: + - qwen35 + representatives: + qwen2: c.gguf + verbs: + - run + - chat + hosts: + - cc: sm_89 + gpu: RTX 4090 + id: lambda + required: true + - cc: sm_121 + gpu: GB10 + id: gx10 + required: true + inventory: + backends: + - cuda + dirs: + - ~/models + patterns: + - '*q4_k*.gguf' + - '*q4k*.apr' + - '*q4_k*.apr' + rungs: + - arch: qwen2 + backends: + - cpu + - cuda + gguf: c.gguf + id: ctrl-q4km + required: true + sha256: '0000000000000000000000000000000000000000000000000000000000000000' + - arch: qwen3 + backends: + - cpu + - cuda + gguf: q.gguf + id: q3-q4km + required: true + sha256: '1111111111111111111111111111111111111111111111111111111111111111' diff --git a/scripts/lib/model_ladder_cases/red-cells-declaimed-still-claimed/must_match b/scripts/lib/model_ladder_cases/red-cells-declaimed-still-claimed/must_match new file mode 100644 index 0000000000..66281b73fc --- /dev/null +++ b/scripts/lib/model_ladder_cases/red-cells-declaimed-still-claimed/must_match @@ -0,0 +1 @@ +FAIL rung ctrl-q4km still claims arch qwen2 on gx10, which cells.declaimed withdraws \(#4590\) diff --git a/scripts/lib/model_ladder_cases/red-cells-declaimed-still-claimed/receipts/gx10.json b/scripts/lib/model_ladder_cases/red-cells-declaimed-still-claimed/receipts/gx10.json new file mode 100644 index 0000000000..0264247e5a --- /dev/null +++ b/scripts/lib/model_ladder_cases/red-cells-declaimed-still-claimed/receipts/gx10.json @@ -0,0 +1,592 @@ +{ + "schema": "apr-model-ladder-receipt/v2", + "host": "gx10", + "version": "1.2.3", + "sha": "abc1234", + "gpu": "x", + "cc": "9.0", + "executed": 3, + "red": 0, + "rungs": [ + { + "id": "ctrl-q4km", + "present": true, + "sha_ok": true, + "required": true, + "capability_match": { + "passed": true, + "skipped": false, + "message": "ok" + }, + "golden_output": { + "passed": true, + "skipped": false, + "message": "ok" + }, + "backends": { + "cpu": { + "ran": true, + "fallback": false, + "rc": 0, + "verbs": { + "run": { + "ran": true, + "rc": 0 + }, + "chat": { + "ran": true, + "rc": 0 + }, + "code": { + "ran": true, + "rc": 0 + }, + "serve": { + "probed": true, + "routes": { + "/api/chat|stream=false": { + "http": 200, + "ok": true + }, + "/api/chat|stream=true": { + "http": 200, + "ok": true + }, + "/v1/chat/completions|stream=false": { + "http": 200, + "ok": true + }, + "/v1/chat/completions|stream=true": { + "http": 200, + "ok": true + }, + "/v1/completions|stream=false": { + "http": 200, + "ok": true + }, + "/v1/completions|stream=true": { + "http": 200, + "ok": true + } + }, + "teardown": "clean" + } + } + }, + "cuda": { + "ran": true, + "fallback": false, + "rc": 0, + "verbs": { + "run": { + "ran": true, + "rc": 0 + }, + "chat": { + "ran": true, + "rc": 0 + }, + "code": { + "ran": true, + "rc": 0 + }, + "serve": { + "probed": true, + "routes": { + "/api/chat|stream=false": { + "http": 200, + "ok": true + }, + "/api/chat|stream=true": { + "http": 200, + "ok": true + }, + "/v1/chat/completions|stream=false": { + "http": 200, + "ok": true + }, + "/v1/chat/completions|stream=true": { + "http": 200, + "ok": true + }, + "/v1/completions|stream=false": { + "http": 200, + "ok": true + }, + "/v1/completions|stream=true": { + "http": 200, + "ok": true + } + }, + "teardown": "clean" + } + } + } + }, + "green": true, + "file": "c.gguf", + "qa_rc": 0, + "fit": { + "tool": "llama-fit-params", + "pin": "d1d3c3396", + "llama_cpp": "d1d3c3396", + "free_mib": 23332, + "ctx": 32768, + "ngl": -1, + "rc": 0, + "raw": "-c 32768 -ngl -1", + "verdict": "fits", + "reason": "full GPU at ctx 32768" + } + }, + { + "id": "q3-q4km", + "present": true, + "sha_ok": true, + "required": true, + "capability_match": { + "passed": true, + "skipped": false, + "message": "ok" + }, + "golden_output": { + "passed": true, + "skipped": false, + "message": "ok" + }, + "backends": { + "cpu": { + "ran": true, + "fallback": false, + "rc": 0, + "verbs": { + "run": { + "ran": true, + "rc": 0 + }, + "chat": { + "ran": true, + "rc": 0 + }, + "code": { + "ran": true, + "rc": 0 + }, + "serve": { + "probed": true, + "routes": { + "/api/chat|stream=false": { + "http": 200, + "ok": true + }, + "/api/chat|stream=true": { + "http": 200, + "ok": true + }, + "/v1/chat/completions|stream=false": { + "http": 200, + "ok": true + }, + "/v1/chat/completions|stream=true": { + "http": 200, + "ok": true + }, + "/v1/completions|stream=false": { + "http": 200, + "ok": true + }, + "/v1/completions|stream=true": { + "http": 200, + "ok": true + } + }, + "teardown": "clean" + } + } + }, + "cuda": { + "ran": true, + "fallback": false, + "rc": 0, + "verbs": { + "run": { + "ran": true, + "rc": 0 + }, + "chat": { + "ran": true, + "rc": 0 + }, + "code": { + "ran": true, + "rc": 0 + }, + "serve": { + "probed": true, + "routes": { + "/api/chat|stream=false": { + "http": 200, + "ok": true + }, + "/api/chat|stream=true": { + "http": 200, + "ok": true + }, + "/v1/chat/completions|stream=false": { + "http": 200, + "ok": true + }, + "/v1/chat/completions|stream=true": { + "http": 200, + "ok": true + }, + "/v1/completions|stream=false": { + "http": 200, + "ok": true + }, + "/v1/completions|stream=true": { + "http": 200, + "ok": true + } + }, + "teardown": "clean" + } + } + } + }, + "green": true, + "file": "q.gguf", + "qa_rc": 0, + "fit": { + "tool": "llama-fit-params", + "pin": "d1d3c3396", + "llama_cpp": "d1d3c3396", + "free_mib": 23332, + "ctx": 32768, + "ngl": -1, + "rc": 0, + "raw": "-c 32768 -ngl -1", + "verdict": "fits", + "reason": "full GPU at ctx 32768" + } + } + ], + "inventory": [ + { + "file": "c.gguf", + "sha256": "0000000000000000000000000000000000000000000000000000000000000000", + "bytes": 1, + "arch": "qwen2", + "context_length": 32768, + "thinking_markers": [], + "generation_opens_think": false, + "thinking_modes": [ + "off" + ], + "weights_bytes": 1000000000, + "kv_bytes_per_token": 28672, + "kv_dtype": "f32", + "workspace_bytes": 500000000, + "chat_template_sha256": "cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc" + }, + { + "file": "q.gguf", + "sha256": "1111111111111111111111111111111111111111111111111111111111111111", + "bytes": 1, + "arch": "qwen35", + "context_length": 262144, + "thinking_markers": [ + "", + "", + "enable_thinking" + ], + "generation_opens_think": false, + "thinking_modes": [ + "off", + "on" + ], + "weights_bytes": 16000000000, + "kv_bytes_per_token": 65536, + "kv_dtype": "f32", + "workspace_bytes": 1000000000, + "chat_template_sha256": "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd" + } + ], + "gpu_mem_total_bytes": 128000000000, + "gpu_mem_free_bytes": 127000000000, + "cells": [ + { + "sha256": "1111111111111111111111111111111111111111111111111111111111111111", + "file": "q.gguf", + "verb": "run", + "thinking": "off", + "context": "4k", + "prompt_tokens": 4096, + "max_tokens": 64, + "backend": "cuda", + "fallback": false, + "rc": 0, + "verdict": "pass", + "answer_chars": 40, + "think_closed": null, + "ttft_ms": 100, + "reason": "ok" + }, + { + "sha256": "1111111111111111111111111111111111111111111111111111111111111111", + "file": "q.gguf", + "verb": "run", + "thinking": "on", + "context": "4k", + "prompt_tokens": 4096, + "max_tokens": 64, + "backend": "cuda", + "fallback": false, + "rc": 0, + "verdict": "pass", + "answer_chars": 40, + "think_closed": true, + "ttft_ms": 100, + "reason": "ok" + }, + { + "sha256": "1111111111111111111111111111111111111111111111111111111111111111", + "file": "q.gguf", + "verb": "chat", + "thinking": "off", + "context": "4k", + "prompt_tokens": 4096, + "max_tokens": 64, + "backend": "cuda", + "fallback": false, + "rc": 0, + "verdict": "pass", + "answer_chars": 40, + "think_closed": null, + "ttft_ms": 100, + "reason": "ok" + }, + { + "sha256": "1111111111111111111111111111111111111111111111111111111111111111", + "file": "q.gguf", + "verb": "chat", + "thinking": "on", + "context": "4k", + "prompt_tokens": 4096, + "max_tokens": 64, + "backend": "cuda", + "fallback": false, + "rc": 0, + "verdict": "pass", + "answer_chars": 40, + "think_closed": true, + "ttft_ms": 100, + "reason": "ok" + }, + { + "sha256": "1111111111111111111111111111111111111111111111111111111111111111", + "file": "q.gguf", + "verb": "run", + "thinking": "off", + "context": "60k", + "prompt_tokens": 61440, + "max_tokens": 64, + "backend": "cuda", + "fallback": false, + "rc": 0, + "verdict": "pass", + "answer_chars": 40, + "think_closed": null, + "ttft_ms": 100, + "reason": "ok" + }, + { + "sha256": "1111111111111111111111111111111111111111111111111111111111111111", + "file": "q.gguf", + "verb": "run", + "thinking": "on", + "context": "60k", + "prompt_tokens": 61440, + "max_tokens": 64, + "backend": "cuda", + "fallback": false, + "rc": 0, + "verdict": "pass", + "answer_chars": 40, + "think_closed": true, + "ttft_ms": 100, + "reason": "ok" + }, + { + "sha256": "1111111111111111111111111111111111111111111111111111111111111111", + "file": "q.gguf", + "verb": "chat", + "thinking": "off", + "context": "60k", + "prompt_tokens": 61440, + "max_tokens": 64, + "backend": "cuda", + "fallback": false, + "rc": 0, + "verdict": "pass", + "answer_chars": 40, + "think_closed": null, + "ttft_ms": 100, + "reason": "ok" + }, + { + "sha256": "1111111111111111111111111111111111111111111111111111111111111111", + "file": "q.gguf", + "verb": "chat", + "thinking": "on", + "context": "60k", + "prompt_tokens": 61440, + "max_tokens": 64, + "backend": "cuda", + "fallback": false, + "rc": 0, + "verdict": "pass", + "answer_chars": 40, + "think_closed": true, + "ttft_ms": 100, + "reason": "ok" + }, + { + "sha256": "1111111111111111111111111111111111111111111111111111111111111111", + "file": "q.gguf", + "verb": "run", + "thinking": "off", + "context": "consumer-max", + "prompt_tokens": 148000, + "max_tokens": 64, + "backend": "cuda", + "fallback": false, + "rc": 0, + "verdict": "pass", + "answer_chars": 40, + "think_closed": null, + "ttft_ms": 100, + "reason": "ok" + }, + { + "sha256": "1111111111111111111111111111111111111111111111111111111111111111", + "file": "q.gguf", + "verb": "run", + "thinking": "on", + "context": "consumer-max", + "prompt_tokens": 148000, + "max_tokens": 64, + "backend": "cuda", + "fallback": false, + "rc": 0, + "verdict": "pass", + "answer_chars": 40, + "think_closed": true, + "ttft_ms": 100, + "reason": "ok" + }, + { + "sha256": "1111111111111111111111111111111111111111111111111111111111111111", + "file": "q.gguf", + "verb": "chat", + "thinking": "off", + "context": "consumer-max", + "prompt_tokens": 148000, + "max_tokens": 64, + "backend": "cuda", + "fallback": false, + "rc": 0, + "verdict": "pass", + "answer_chars": 40, + "think_closed": null, + "ttft_ms": 100, + "reason": "ok" + }, + { + "sha256": "1111111111111111111111111111111111111111111111111111111111111111", + "file": "q.gguf", + "verb": "chat", + "thinking": "on", + "context": "consumer-max", + "prompt_tokens": 148000, + "max_tokens": 64, + "backend": "cuda", + "fallback": false, + "rc": 0, + "verdict": "pass", + "answer_chars": 40, + "think_closed": true, + "ttft_ms": 100, + "reason": "ok" + }, + { + "sha256": "1111111111111111111111111111111111111111111111111111111111111111", + "file": "q.gguf", + "verb": "run", + "thinking": "off", + "context": "declared", + "prompt_tokens": 262144, + "max_tokens": 64, + "backend": "cuda", + "fallback": false, + "rc": 0, + "verdict": "pass", + "answer_chars": 40, + "think_closed": null, + "ttft_ms": 100, + "reason": "ok" + }, + { + "sha256": "1111111111111111111111111111111111111111111111111111111111111111", + "file": "q.gguf", + "verb": "run", + "thinking": "on", + "context": "declared", + "prompt_tokens": 262144, + "max_tokens": 64, + "backend": "cuda", + "fallback": false, + "rc": 0, + "verdict": "pass", + "answer_chars": 40, + "think_closed": true, + "ttft_ms": 100, + "reason": "ok" + }, + { + "sha256": "1111111111111111111111111111111111111111111111111111111111111111", + "file": "q.gguf", + "verb": "chat", + "thinking": "off", + "context": "declared", + "prompt_tokens": 262144, + "max_tokens": 64, + "backend": "cuda", + "fallback": false, + "rc": 0, + "verdict": "pass", + "answer_chars": 40, + "think_closed": null, + "ttft_ms": 100, + "reason": "ok" + }, + { + "sha256": "1111111111111111111111111111111111111111111111111111111111111111", + "file": "q.gguf", + "verb": "chat", + "thinking": "on", + "context": "declared", + "prompt_tokens": 262144, + "max_tokens": 64, + "backend": "cuda", + "fallback": false, + "rc": 0, + "verdict": "pass", + "answer_chars": 40, + "think_closed": true, + "ttft_ms": 100, + "reason": "ok" + } + ], + "apr_sha": "ca5eca5eca5eca5eca5eca5eca5eca5eca5eca5e" +} diff --git a/scripts/lib/model_ladder_cases/red-cells-declaimed-still-claimed/receipts/lambda.json b/scripts/lib/model_ladder_cases/red-cells-declaimed-still-claimed/receipts/lambda.json new file mode 100644 index 0000000000..b7064ff018 --- /dev/null +++ b/scripts/lib/model_ladder_cases/red-cells-declaimed-still-claimed/receipts/lambda.json @@ -0,0 +1,652 @@ +{ + "schema": "apr-model-ladder-receipt/v2", + "host": "lambda", + "version": "1.2.3", + "sha": "abc1234", + "gpu": "x", + "cc": "9.0", + "executed": 3, + "red": 0, + "rungs": [ + { + "id": "ctrl-q4km", + "present": true, + "sha_ok": true, + "required": true, + "capability_match": { + "passed": true, + "skipped": false, + "message": "ok" + }, + "golden_output": { + "passed": true, + "skipped": false, + "message": "ok" + }, + "backends": { + "cpu": { + "ran": true, + "fallback": false, + "rc": 0, + "verbs": { + "run": { + "ran": true, + "rc": 0 + }, + "chat": { + "ran": true, + "rc": 0 + }, + "code": { + "ran": true, + "rc": 0 + }, + "serve": { + "probed": true, + "routes": { + "/api/chat|stream=false": { + "http": 200, + "ok": true + }, + "/api/chat|stream=true": { + "http": 200, + "ok": true + }, + "/v1/chat/completions|stream=false": { + "http": 200, + "ok": true + }, + "/v1/chat/completions|stream=true": { + "http": 200, + "ok": true + }, + "/v1/completions|stream=false": { + "http": 200, + "ok": true + }, + "/v1/completions|stream=true": { + "http": 200, + "ok": true + } + }, + "teardown": "clean" + } + } + }, + "cuda": { + "ran": true, + "fallback": false, + "rc": 0, + "verbs": { + "run": { + "ran": true, + "rc": 0 + }, + "chat": { + "ran": true, + "rc": 0 + }, + "code": { + "ran": true, + "rc": 0 + }, + "serve": { + "probed": true, + "routes": { + "/api/chat|stream=false": { + "http": 200, + "ok": true + }, + "/api/chat|stream=true": { + "http": 200, + "ok": true + }, + "/v1/chat/completions|stream=false": { + "http": 200, + "ok": true + }, + "/v1/chat/completions|stream=true": { + "http": 200, + "ok": true + }, + "/v1/completions|stream=false": { + "http": 200, + "ok": true + }, + "/v1/completions|stream=true": { + "http": 200, + "ok": true + } + }, + "teardown": "clean" + } + } + } + }, + "green": true, + "file": "c.gguf", + "qa_rc": 0, + "fit": { + "tool": "llama-fit-params", + "pin": "d1d3c3396", + "llama_cpp": "d1d3c3396", + "free_mib": 23332, + "ctx": 32768, + "ngl": -1, + "rc": 0, + "raw": "-c 32768 -ngl -1", + "verdict": "fits", + "reason": "full GPU at ctx 32768" + } + }, + { + "id": "q3-q4km", + "present": true, + "sha_ok": true, + "required": true, + "capability_match": { + "passed": true, + "skipped": false, + "message": "ok" + }, + "golden_output": { + "passed": true, + "skipped": false, + "message": "ok" + }, + "backends": { + "cpu": { + "ran": true, + "fallback": false, + "rc": 0, + "verbs": { + "run": { + "ran": true, + "rc": 0 + }, + "chat": { + "ran": true, + "rc": 0 + }, + "code": { + "ran": true, + "rc": 0 + }, + "serve": { + "probed": true, + "routes": { + "/api/chat|stream=false": { + "http": 200, + "ok": true + }, + "/api/chat|stream=true": { + "http": 200, + "ok": true + }, + "/v1/chat/completions|stream=false": { + "http": 200, + "ok": true + }, + "/v1/chat/completions|stream=true": { + "http": 200, + "ok": true + }, + "/v1/completions|stream=false": { + "http": 200, + "ok": true + }, + "/v1/completions|stream=true": { + "http": 200, + "ok": true + } + }, + "teardown": "clean" + } + } + }, + "cuda": { + "ran": true, + "fallback": false, + "rc": 0, + "verbs": { + "run": { + "ran": true, + "rc": 0 + }, + "chat": { + "ran": true, + "rc": 0 + }, + "code": { + "ran": true, + "rc": 0 + }, + "serve": { + "probed": true, + "routes": { + "/api/chat|stream=false": { + "http": 200, + "ok": true + }, + "/api/chat|stream=true": { + "http": 200, + "ok": true + }, + "/v1/chat/completions|stream=false": { + "http": 200, + "ok": true + }, + "/v1/chat/completions|stream=true": { + "http": 200, + "ok": true + }, + "/v1/completions|stream=false": { + "http": 200, + "ok": true + }, + "/v1/completions|stream=true": { + "http": 200, + "ok": true + } + }, + "teardown": "clean" + } + } + } + }, + "green": true, + "file": "q.gguf", + "qa_rc": 0, + "fit": { + "tool": "llama-fit-params", + "pin": "d1d3c3396", + "llama_cpp": "d1d3c3396", + "free_mib": 23332, + "ctx": 32768, + "ngl": -1, + "rc": 0, + "raw": "-c 32768 -ngl -1", + "verdict": "fits", + "reason": "full GPU at ctx 32768" + } + } + ], + "inventory": [ + { + "file": "c.gguf", + "sha256": "0000000000000000000000000000000000000000000000000000000000000000", + "bytes": 1, + "arch": "qwen2", + "context_length": 32768, + "thinking_markers": [], + "generation_opens_think": false, + "thinking_modes": [ + "off" + ], + "weights_bytes": 1000000000, + "kv_bytes_per_token": 28672, + "kv_dtype": "f32", + "workspace_bytes": 500000000, + "chat_template_sha256": "cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc" + }, + { + "file": "q.gguf", + "sha256": "1111111111111111111111111111111111111111111111111111111111111111", + "bytes": 1, + "arch": "qwen35", + "context_length": 262144, + "thinking_markers": [ + "", + "", + "enable_thinking" + ], + "generation_opens_think": false, + "thinking_modes": [ + "off", + "on" + ], + "weights_bytes": 16000000000, + "kv_bytes_per_token": 65536, + "kv_dtype": "f32", + "workspace_bytes": 1000000000, + "chat_template_sha256": "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd" + } + ], + "gpu_mem_total_bytes": 24000000000, + "gpu_mem_free_bytes": 23000000000, + "cells": [ + { + "sha256": "0000000000000000000000000000000000000000000000000000000000000000", + "file": "c.gguf", + "verb": "run", + "thinking": "off", + "context": "4k", + "prompt_tokens": 4096, + "max_tokens": 64, + "backend": "cuda", + "fallback": false, + "rc": 0, + "verdict": "pass", + "answer_chars": 40, + "think_closed": null, + "ttft_ms": 100, + "reason": "ok" + }, + { + "sha256": "0000000000000000000000000000000000000000000000000000000000000000", + "file": "c.gguf", + "verb": "chat", + "thinking": "off", + "context": "4k", + "prompt_tokens": 4096, + "max_tokens": 64, + "backend": "cuda", + "fallback": false, + "rc": 0, + "verdict": "pass", + "answer_chars": 40, + "think_closed": null, + "ttft_ms": 100, + "reason": "ok" + }, + { + "sha256": "0000000000000000000000000000000000000000000000000000000000000000", + "file": "c.gguf", + "verb": "run", + "thinking": "off", + "context": "declared", + "prompt_tokens": 32768, + "max_tokens": 64, + "backend": "cuda", + "fallback": false, + "rc": 0, + "verdict": "pass", + "answer_chars": 40, + "think_closed": null, + "ttft_ms": 100, + "reason": "ok" + }, + { + "sha256": "0000000000000000000000000000000000000000000000000000000000000000", + "file": "c.gguf", + "verb": "chat", + "thinking": "off", + "context": "declared", + "prompt_tokens": 32768, + "max_tokens": 64, + "backend": "cuda", + "fallback": false, + "rc": 0, + "verdict": "pass", + "answer_chars": 40, + "think_closed": null, + "ttft_ms": 100, + "reason": "ok" + }, + { + "sha256": "1111111111111111111111111111111111111111111111111111111111111111", + "file": "q.gguf", + "verb": "run", + "thinking": "off", + "context": "4k", + "prompt_tokens": 4096, + "max_tokens": 64, + "backend": "cuda", + "fallback": false, + "rc": 0, + "verdict": "pass", + "answer_chars": 40, + "think_closed": null, + "ttft_ms": 100, + "reason": "ok" + }, + { + "sha256": "1111111111111111111111111111111111111111111111111111111111111111", + "file": "q.gguf", + "verb": "run", + "thinking": "on", + "context": "4k", + "prompt_tokens": 4096, + "max_tokens": 64, + "backend": "cuda", + "fallback": false, + "rc": 0, + "verdict": "pass", + "answer_chars": 40, + "think_closed": true, + "ttft_ms": 100, + "reason": "ok" + }, + { + "sha256": "1111111111111111111111111111111111111111111111111111111111111111", + "file": "q.gguf", + "verb": "chat", + "thinking": "off", + "context": "4k", + "prompt_tokens": 4096, + "max_tokens": 64, + "backend": "cuda", + "fallback": false, + "rc": 0, + "verdict": "pass", + "answer_chars": 40, + "think_closed": null, + "ttft_ms": 100, + "reason": "ok" + }, + { + "sha256": "1111111111111111111111111111111111111111111111111111111111111111", + "file": "q.gguf", + "verb": "chat", + "thinking": "on", + "context": "4k", + "prompt_tokens": 4096, + "max_tokens": 64, + "backend": "cuda", + "fallback": false, + "rc": 0, + "verdict": "pass", + "answer_chars": 40, + "think_closed": true, + "ttft_ms": 100, + "reason": "ok" + }, + { + "sha256": "1111111111111111111111111111111111111111111111111111111111111111", + "file": "q.gguf", + "verb": "run", + "thinking": "off", + "context": "60k", + "prompt_tokens": 61440, + "max_tokens": 64, + "backend": "cuda", + "fallback": false, + "rc": 0, + "verdict": "pass", + "answer_chars": 40, + "think_closed": null, + "ttft_ms": 100, + "reason": "ok" + }, + { + "sha256": "1111111111111111111111111111111111111111111111111111111111111111", + "file": "q.gguf", + "verb": "run", + "thinking": "on", + "context": "60k", + "prompt_tokens": 61440, + "max_tokens": 64, + "backend": "cuda", + "fallback": false, + "rc": 0, + "verdict": "pass", + "answer_chars": 40, + "think_closed": true, + "ttft_ms": 100, + "reason": "ok" + }, + { + "sha256": "1111111111111111111111111111111111111111111111111111111111111111", + "file": "q.gguf", + "verb": "chat", + "thinking": "off", + "context": "60k", + "prompt_tokens": 61440, + "max_tokens": 64, + "backend": "cuda", + "fallback": false, + "rc": 0, + "verdict": "pass", + "answer_chars": 40, + "think_closed": null, + "ttft_ms": 100, + "reason": "ok" + }, + { + "sha256": "1111111111111111111111111111111111111111111111111111111111111111", + "file": "q.gguf", + "verb": "chat", + "thinking": "on", + "context": "60k", + "prompt_tokens": 61440, + "max_tokens": 64, + "backend": "cuda", + "fallback": false, + "rc": 0, + "verdict": "pass", + "answer_chars": 40, + "think_closed": true, + "ttft_ms": 100, + "reason": "ok" + }, + { + "sha256": "1111111111111111111111111111111111111111111111111111111111111111", + "file": "q.gguf", + "verb": "run", + "thinking": "off", + "context": "consumer-max", + "prompt_tokens": 148000, + "max_tokens": 64, + "backend": "cuda", + "fallback": false, + "rc": 3, + "verdict": "refused", + "required_bytes": 26699328000, + "available_bytes": 24000000000, + "reason": "needs 26699328000 B > 24000000000 B total" + }, + { + "sha256": "1111111111111111111111111111111111111111111111111111111111111111", + "file": "q.gguf", + "verb": "run", + "thinking": "on", + "context": "consumer-max", + "prompt_tokens": 148000, + "max_tokens": 64, + "backend": "cuda", + "fallback": false, + "rc": 3, + "verdict": "refused", + "required_bytes": 26699328000, + "available_bytes": 24000000000, + "reason": "needs 26699328000 B > 24000000000 B total" + }, + { + "sha256": "1111111111111111111111111111111111111111111111111111111111111111", + "file": "q.gguf", + "verb": "chat", + "thinking": "off", + "context": "consumer-max", + "prompt_tokens": 148000, + "max_tokens": 64, + "backend": "cuda", + "fallback": false, + "rc": 3, + "verdict": "refused", + "required_bytes": 26699328000, + "available_bytes": 24000000000, + "reason": "needs 26699328000 B > 24000000000 B total" + }, + { + "sha256": "1111111111111111111111111111111111111111111111111111111111111111", + "file": "q.gguf", + "verb": "chat", + "thinking": "on", + "context": "consumer-max", + "prompt_tokens": 148000, + "max_tokens": 64, + "backend": "cuda", + "fallback": false, + "rc": 3, + "verdict": "refused", + "required_bytes": 26699328000, + "available_bytes": 24000000000, + "reason": "needs 26699328000 B > 24000000000 B total" + }, + { + "sha256": "1111111111111111111111111111111111111111111111111111111111111111", + "file": "q.gguf", + "verb": "run", + "thinking": "off", + "context": "declared", + "prompt_tokens": 262144, + "max_tokens": 64, + "backend": "cuda", + "fallback": false, + "rc": 3, + "verdict": "refused", + "required_bytes": 34179869184, + "available_bytes": 24000000000, + "reason": "needs 34179869184 B > 24000000000 B total" + }, + { + "sha256": "1111111111111111111111111111111111111111111111111111111111111111", + "file": "q.gguf", + "verb": "run", + "thinking": "on", + "context": "declared", + "prompt_tokens": 262144, + "max_tokens": 64, + "backend": "cuda", + "fallback": false, + "rc": 3, + "verdict": "refused", + "required_bytes": 34179869184, + "available_bytes": 24000000000, + "reason": "needs 34179869184 B > 24000000000 B total" + }, + { + "sha256": "1111111111111111111111111111111111111111111111111111111111111111", + "file": "q.gguf", + "verb": "chat", + "thinking": "off", + "context": "declared", + "prompt_tokens": 262144, + "max_tokens": 64, + "backend": "cuda", + "fallback": false, + "rc": 3, + "verdict": "refused", + "required_bytes": 34179869184, + "available_bytes": 24000000000, + "reason": "needs 34179869184 B > 24000000000 B total" + }, + { + "sha256": "1111111111111111111111111111111111111111111111111111111111111111", + "file": "q.gguf", + "verb": "chat", + "thinking": "on", + "context": "declared", + "prompt_tokens": 262144, + "max_tokens": 64, + "backend": "cuda", + "fallback": false, + "rc": 3, + "verdict": "refused", + "required_bytes": 34179869184, + "available_bytes": 24000000000, + "reason": "needs 34179869184 B > 24000000000 B total" + } + ], + "apr_sha": "ca5eca5eca5eca5eca5eca5eca5eca5eca5eca5e" +} diff --git a/scripts/lib/model_ladder_cases/red-cells-declaimed-still-claimed/version b/scripts/lib/model_ladder_cases/red-cells-declaimed-still-claimed/version new file mode 100644 index 0000000000..0495c4a88c --- /dev/null +++ b/scripts/lib/model_ladder_cases/red-cells-declaimed-still-claimed/version @@ -0,0 +1 @@ +1.2.3 diff --git a/scripts/lib/model_ladder_cells.py b/scripts/lib/model_ladder_cells.py index 75357dd411..4154180423 100644 --- a/scripts/lib/model_ladder_cells.py +++ b/scripts/lib/model_ladder_cells.py @@ -296,6 +296,27 @@ def declaimed(C, hosts, out): return got, rc +def declaim_still_claimed(L, dec, out): + """-> rc. A (host, arch) that cells.declaimed withdraws and the ladder still claims is a contradiction (#3715, + cop 2026-09-28 20:00Z). Released pv reads rung hosts and not declaimed, so a rung whose host set still names + the de-claimed host keeps the claim alive where the release decision is made; a long-rung representative for + an arch de-claimed on every required host claims long rungs nobody owes. Either one is RED, never a note.""" + req = {h.get("id") for h in L.get("hosts") or [] if h.get("required")} + rc = 0 + for r in L.get("rungs") or []: + for h in sorted(set(r.get("hosts") or req)): + d = dec.get((h, r.get("arch"))) + if d: # a claim the de-claim withdraws + out(f"FAIL rung {r.get('id')} still claims arch {r.get('arch')} on {h}, which cells.declaimed withdraws (#{d['issue']}) -- narrow its hosts: or drop the de-claim") + rc = 1 + reps = ((L.get("cells") or {}).get("long_rungs_for") or {}).get("representatives") or {} + for arch in sorted(reps): + if req and all((h, arch) in dec for h in req): # every required host de-claims it + out(f"FAIL long_rungs_for.representatives.{arch} still claims long rungs for arch {arch}, de-claimed on every required host") + rc = 1 + return rc + + def judge(L, receipts, rungs_doc, out, rungs_main=None): C = L.get("cells") if not C: @@ -339,6 +360,8 @@ def judge(L, receipts, rungs_doc, out, rungs_main=None): rungs, consumer_max, rc = load_rungs(rungs_doc, out) dec, drc = declaimed(C, {h.get("id") for h in L.get("hosts") or []}, out) rc = rc or drc + if declaim_still_claimed(L, dec, out): + rc = 1 if rungs_main is not None and _rungs_floor(rungs_main, rungs_doc, out): rc = 1 if not verbs: diff --git a/scripts/release/autopilot.sh b/scripts/release/autopilot.sh index 91217e8c8d..e0e80d5cbd 100755 --- a/scripts/release/autopilot.sh +++ b/scripts/release/autopilot.sh @@ -172,7 +172,11 @@ if run_step readiness; then --dogfood-receipt "$DR" --surface "$AP/surface-t1.json" --out "$AP/readiness-t1.json" > "$AP/readiness-t1.log" 2>&1; rc=$? grep -E '^(ok|WARN|FAIL) +R8 ' "$AP/readiness-t1.log" >> "$STATUS" [ $rc -eq 0 ] || die "T-1 release-readiness-v1 rc=$rc: nothing is tagged ($AP/readiness-t1.log)" - say "READINESS $(grep -oE '^(ok|WARN)' "$AP/readiness-t1.log" | tail -n 1) at $MC" + # R10 (operator 2026-09-28): report-only is not an option. A wrapper that prints a WARN R8 row and + # exits 0 (a hand-edited or stale copy) is a waiver, and a waiver is a stop. + ! grep -qE '^WARN +R8 ' "$AP/readiness-t1.log" \ + || die "T-1 release-readiness-v1 printed a WARN R8 row (report-only = waiver = stop): nothing is tagged ($AP/readiness-t1.log)" + say "READINESS ok at $MC" fi # 3. tag + release (binary-release.yml fires on release: published, from the TAG's workflow file) # cut_tag -- PMAT-3459. The milestone gate lives INSIDE the diff --git a/scripts/release/release_readiness.sh b/scripts/release/release_readiness.sh index 700d7778a6..905704de4e 100755 --- a/scripts/release/release_readiness.sh +++ b/scripts/release/release_readiness.sh @@ -8,19 +8,16 @@ # missing edge `minCount 1` rejects). Both call sites must ask it the same question with the same subject, # the same receipts-commit rule and the same exit mapping — two hand-copied `pv lint` lines would drift. # -# MODE. `enforce` (the committed default below, flipped 2026-09-28 for 0.70, #3715 B1) or `report`. +# MODE. `enforce`, and nothing else (flipped 2026-09-28 for 0.70, #3715 B1; the report arm was deleted +# for R10: "report-only is not an option", L19: a silent report-only check = hard stop). # Measured 2026-09-25 with pv 0.69.3 on 0.69.1 @d8a6df53a: rc 1, 1101 violations, 1008 of them cells with # no `ont:release/row`, because models_t1 measured rungs only. models_t1 now passes --cells on both hosts, # and the operator's de-claim of Qwen3 on gx10 (#4590) is DATA the released pv reads (rung hosts + the -# receipt's inventory), so a Fail STOPs the train before the tag. Going back to `report` is a reviewed -# commit to DEFAULT_MODE, never an environment variable. -# RELEASE_READINESS_MODE may only STRENGTHEN the committed mode (`enforce`); asking for `report` over an -# `enforce` default, or any other value, is a caller error (3). A gate an env var can weaken is theater. -# Could-not-judge under `report` (cop ruling 2026-09-25 09:15Z: "record the rc and receipt, don't -# stop"): a decline (pv 2), an unknown exit, a missing pv, or a "Fail" exit carrying no Fail verdict is -# printed as its FAIL row PLUS a `WARN R8 REPORT-ONLY could not judge (rc 2)` row, and exits 0. The -# rc is on the page, not hidden. Under `enforce` each is exit 2. A caller error (3) stops in either -# mode: it means this script was called wrong, and that is a wiring defect, not a verdict. +# receipt's inventory), so a Fail STOPs the train before the tag. +# RELEASE_READINESS_MODE may only say `enforce`; `report`, as the env value OR as a committed DEFAULT_MODE, +# or any other value, is a caller error (3). A gate an env var or a one-word edit can weaken is theater. +# Could-not-judge (a decline (pv 2), an unknown exit, a missing pv, or a "Fail" exit carrying no Fail +# verdict) is exit 2, a stop. A caller error (3) means this script was called wrong: a wiring defect. # # RECEIPTS-COMMIT (T-4). The committed receipts were measured at the binary's commit (their `apr_sha`), # which is never the tagged HEAD: the receipts are committed on top. pv takes `--receipts-commit` on trust @@ -28,8 +25,8 @@ # `git diff X ` is empty outside evidence/. Otherwise the flag is withheld, pv grades the # receipts against the release commit, and a stale receipt is a violation — the row says which. # -# EXIT 0 Pass, or a Fail verdict under `report` (WARN row) · 1 Fail under `enforce` · -# 2 pv declined / could not judge / is missing, under `enforce` (a WARN + 0 under `report`) · +# EXIT 0 Pass · 1 Fail · +# 2 pv declined / could not judge / is missing · # 3 caller error. Every non-zero STOPs the caller. # # SEAMS (the selftest drives every row through them; production sets neither): @@ -47,12 +44,12 @@ SHAPE=release-readiness-v1 caller_error() { printf 'FAIL R8 %s: caller error: %s\n' "$PROG" "$*"; exit 3; } -# resolve_mode committed-default env-value -> prints the mode; 1 on a value that would weaken or is unknown +# resolve_mode committed-default env-value -> prints `enforce`; 1 on anything else. There is no report mode +# (operator R10, 2026-09-28: "report-only is not an option"; L19: a silent report-only check = hard stop). resolve_mode() { + [ "$1" = enforce ] || return 1 case "${2:-}" in - '') printf '%s\n' "$1" ;; - enforce) printf 'enforce\n' ;; - report) [ "$1" = report ] || return 1; printf 'report\n' ;; + ''|enforce) printf 'enforce\n' ;; *) return 1 ;; esac } @@ -158,7 +155,7 @@ judge() { # judge mode root version commit receipts dogfood out case "$rc" in 0) echo "ok R8 $label: Pass" # #3715 B1 (operator 2026-09-28): the ONE line autopilot cut_tag() requires before `git tag`. - # Printed only for an enforced Pass, so a report-mode run, a skipped step or an absent log + # Printed only for an enforced Pass, so a weakened run, a skipped step or an absent log # all leave the tag refused. out_sha256 is pv's verdict output, for the receipt. if [ "$mode" = enforce ]; then echo "ok R8 #3715 ENFORCE PASS version=$version commit=$commit pv=${pvv:-unknown} out_sha256=$(sha256sum < "$out" | cut -c1-64)" @@ -168,11 +165,7 @@ judge() { # judge mode root version commit receipts dogfood out if ! sum="$(summarize "$out")"; then echo "FAIL R8 $label: pv exited 1 with no Fail verdict in its output, so it did not judge"; tail -n 3 "$out" | sed 's/^/ /'; return 2 fi - if [ "$mode" = enforce ]; then - echo "FAIL R8 $label: Fail, $sum"; return 1 - fi - echo "WARN R8 REPORT-ONLY $label: Fail, $sum (not a stop until #3712 lands; DEFAULT_MODE in $PROG)" - return 0 ;; + echo "FAIL R8 $label: Fail, $sum"; return 1 ;; 2) echo "FAIL R8 $label: pv DECLINED (rc 2), and a decline is not a pass: $(tail -n 1 "$out")"; return 2 ;; 3) echo "FAIL R8 $label: pv refused the call (rc 3, caller error): $(tail -n 1 "$out")"; return 3 ;; *) echo "FAIL R8 $label: pv exited $rc, which is outside its 0/1/2/3 contract: $(tail -n 1 "$out")"; return 2 ;; @@ -200,7 +193,7 @@ main() { [ -n "$version" ] || caller_error "--version is required" [ -n "$commit" ] || caller_error "--commit is required" mode="$(resolve_mode "$DEFAULT_MODE" "${RELEASE_READINESS_MODE:-}")" \ - || caller_error "RELEASE_READINESS_MODE='${RELEASE_READINESS_MODE:-}' may only strengthen the committed mode ($DEFAULT_MODE) to enforce" + || caller_error "RELEASE_READINESS_MODE='${RELEASE_READINESS_MODE:-}' / DEFAULT_MODE='$DEFAULT_MODE': the only mode is enforce; there is no report mode (may only strengthen)" [ -n "$root" ] || root="$(cd -- "$(dirname -- "$SCRIPT_PATH")/../.." && pwd)" git -C "$root" rev-parse --verify --quiet HEAD >/dev/null || { echo "FAIL R8 $root is not a git repository"; exit 2; } [ -z "$dogfood" ] || [ -f "$dogfood" ] || caller_error "--dogfood-receipt $dogfood does not exist" @@ -211,10 +204,6 @@ main() { [ -f "$surface" ] || surface="" if [ -n "$out" ]; then tmpout="$out"; else tmpout="$(mktemp)"; fi judge "$mode" "$root" "$version" "$commit" "$receipts" "$dogfood" "$tmpout" "$surface"; rc=$? - if [ "$rc" = 2 ] && [ "$mode" = report ]; then - echo "WARN R8 REPORT-ONLY could not judge (rc 2), recorded and not a stop until #3712 lands (DEFAULT_MODE in $PROG)" - rc=0 - fi [ -n "$out" ] || rm -f -- "$tmpout" exit "$rc" } @@ -315,11 +304,11 @@ STUB else printf ' BROKE %-44s DEFAULT_MODE=%s\n' committed_default_is_enforce "$DEFAULT_MODE"; fail=$((fail + 1)) fi - if [ "$(resolve_mode report '')" = report ] && [ "$(resolve_mode enforce '')" = enforce ] \ - && ! resolve_mode enforce report >/dev/null && [ "$(resolve_mode report enforce)" = enforce ]; then - printf ' ok %-44s report over enforce refused, enforce over report taken\n' mode_env_only_strengthens; pass=$((pass + 1)) + if [ "$(resolve_mode enforce '')" = enforce ] && [ "$(resolve_mode enforce enforce)" = enforce ] \ + && ! resolve_mode enforce report >/dev/null && ! resolve_mode report '' >/dev/null && ! resolve_mode report enforce >/dev/null; then + printf ' ok %-44s report is refused as env AND as a committed default\n' mode_has_no_report_arm; pass=$((pass + 1)) else - printf ' BROKE %-44s\n' mode_env_only_strengthens; fail=$((fail + 1)) + printf ' BROKE %-44s\n' mode_has_no_report_arm; fail=$((fail + 1)) fi printf -- '--- %s/%s rows ---\n' "$pass" "$((pass + fail))" [ "$fail" -eq 0 ]