From 7d7a338f301eb938632f567c3a94f5cbe9266f76 Mon Sep 17 00:00:00 2001 From: Noah Gift Date: Mon, 21 Sep 2026 18:40:10 +0200 Subject: [PATCH 1/4] feat(ladder): the model gate judges cells (verb x thinking x context rung), derived from the measured inventory Operator 2026-09-21: "lets ensure this release has MORE than enough tokens for our needs (overdo it), and our verbs are dogfooded, i.e chat with and without thinking, ditto run, ditto code, etc,". #3712 row B1, the judge side; stacked on row A. - scripts/lib/model_ladder_cells.py (called from check_model_ladder.sh): the owed set is short rungs up to each model's context_length plus long rungs for qwen35 and one representative per other arch, x cells.verbs x the thinking modes its TEMPLATE evidence derives (#3723). fit := weights + kv_bytes_per_token*tokens + workspace <= the host's TOTAL GPU memory (operator "a", #3710). A fitting cell must pass (cuda, no fallback, rc 0, prompt >= rung, an answer, thinking closed); a non-fitting one must be an honest refusal; every owed cell passes on >= 1 required host; every arch has a held representative. Rungs come from evidence/release/context-rungs.json (#3715), one declaration shared with pv; missing or unsized is a named FAIL. - contract: ladder.cells {verbs, long_rungs_for}; MCL-INV-008; FALSIFY-MCL-016. The cells block is an anti-shrink floor vs origin/main. - case table 26 -> 45; 8 cells-module mutants, each killed by its case. Refs #3712 Co-Authored-By: Claude Opus 5 (1M context) --- contracts/model-capability-ladder-v1.yaml | 28 ++ docs/roadmaps/entries/PMAT-3712.yaml | 2 +- docs/roadmaps/roadmap.yaml | 2 +- scripts/check_model_ladder.sh | 49 +- .../green-cells/context-rungs.json | 38 ++ .../green-cells/expected_rc | 1 + .../green-cells/ladder.yaml | 45 ++ .../model_ladder_cases/green-cells/must_match | 1 + .../green-cells/receipts/gx10.json | 457 +++++++++++++++++ .../green-cells/receipts/lambda.json | 449 +++++++++++++++++ .../model_ladder_cases/green-cells/version | 1 + .../context-rungs.json | 38 ++ .../expected_rc | 1 + .../ladder.yaml | 44 ++ .../must_match | 1 + .../receipts/gx10.json | 457 +++++++++++++++++ .../receipts/lambda.json | 449 +++++++++++++++++ .../version | 1 + .../context-rungs.json | 38 ++ .../red-cells-block-dropped/expected_rc | 1 + .../red-cells-block-dropped/ladder.yaml | 36 ++ .../red-cells-block-dropped/ladder_main.yaml | 45 ++ .../red-cells-block-dropped/must_match | 1 + .../receipts/gx10.json | 457 +++++++++++++++++ .../receipts/lambda.json | 449 +++++++++++++++++ .../red-cells-block-dropped/version | 1 + .../context-rungs.json | 37 ++ .../expected_rc | 1 + .../ladder.yaml | 45 ++ .../must_match | 1 + .../receipts/gx10.json | 457 +++++++++++++++++ .../receipts/lambda.json | 449 +++++++++++++++++ .../version | 1 + .../context-rungs.json | 38 ++ .../red-cells-cotenant-refusal/expected_rc | 1 + .../red-cells-cotenant-refusal/ladder.yaml | 45 ++ .../red-cells-cotenant-refusal/must_match | 1 + .../receipts/gx10.json | 457 +++++++++++++++++ .../receipts/lambda.json | 451 +++++++++++++++++ .../red-cells-cotenant-refusal/version | 1 + .../context-rungs.json | 38 ++ .../expected_rc | 1 + .../ladder.yaml | 45 ++ .../must_match | 1 + .../receipts/gx10.json | 465 +++++++++++++++++ .../receipts/lambda.json | 449 +++++++++++++++++ .../red-cells-declared-passes-nowhere/version | 1 + .../context-rungs.json | 38 ++ .../red-cells-duplicate-row/expected_rc | 1 + .../red-cells-duplicate-row/ladder.yaml | 45 ++ .../red-cells-duplicate-row/must_match | 1 + .../receipts/gx10.json | 457 +++++++++++++++++ .../receipts/lambda.json | 466 ++++++++++++++++++ .../red-cells-duplicate-row/version | 1 + .../red-cells-missing-cell/context-rungs.json | 38 ++ .../red-cells-missing-cell/expected_rc | 1 + .../red-cells-missing-cell/ladder.yaml | 45 ++ .../red-cells-missing-cell/must_match | 1 + .../red-cells-missing-cell/receipts/gx10.json | 440 +++++++++++++++++ .../receipts/lambda.json | 449 +++++++++++++++++ .../red-cells-missing-cell/version | 1 + .../red-cells-no-answer/context-rungs.json | 38 ++ .../red-cells-no-answer/expected_rc | 1 + .../red-cells-no-answer/ladder.yaml | 45 ++ .../red-cells-no-answer/must_match | 1 + .../red-cells-no-answer/receipts/gx10.json | 457 +++++++++++++++++ .../red-cells-no-answer/receipts/lambda.json | 449 +++++++++++++++++ .../red-cells-no-answer/version | 1 + .../red-cells-no-cells/context-rungs.json | 38 ++ .../red-cells-no-cells/expected_rc | 1 + .../red-cells-no-cells/ladder.yaml | 45 ++ .../red-cells-no-cells/must_match | 1 + .../red-cells-no-cells/receipts/gx10.json | 457 +++++++++++++++++ .../red-cells-no-cells/receipts/lambda.json | 116 +++++ .../red-cells-no-cells/version | 1 + .../context-rungs.json | 38 ++ .../red-cells-no-context-length/expected_rc | 1 + .../red-cells-no-context-length/ladder.yaml | 45 ++ .../red-cells-no-context-length/must_match | 1 + .../receipts/gx10.json | 456 +++++++++++++++++ .../receipts/lambda.json | 449 +++++++++++++++++ .../red-cells-no-context-length/version | 1 + .../context-rungs.json | 38 ++ .../red-cells-oom-not-refusal/expected_rc | 1 + .../red-cells-oom-not-refusal/ladder.yaml | 45 ++ .../red-cells-oom-not-refusal/must_match | 1 + .../receipts/gx10.json | 457 +++++++++++++++++ .../receipts/lambda.json | 449 +++++++++++++++++ .../red-cells-oom-not-refusal/version | 1 + .../context-rungs.json | 38 ++ .../red-cells-pass-fell-back/expected_rc | 1 + .../red-cells-pass-fell-back/ladder.yaml | 45 ++ .../red-cells-pass-fell-back/must_match | 1 + .../receipts/gx10.json | 457 +++++++++++++++++ .../receipts/lambda.json | 449 +++++++++++++++++ .../red-cells-pass-fell-back/version | 1 + .../context-rungs.json | 38 ++ .../red-cells-prompt-under-rung/expected_rc | 1 + .../red-cells-prompt-under-rung/ladder.yaml | 45 ++ .../red-cells-prompt-under-rung/must_match | 1 + .../receipts/gx10.json | 457 +++++++++++++++++ .../receipts/lambda.json | 449 +++++++++++++++++ .../red-cells-prompt-under-rung/version | 1 + .../context-rungs.json | 38 ++ .../expected_rc | 1 + .../ladder.yaml | 45 ++ .../must_match | 1 + .../receipts/gx10.json | 457 +++++++++++++++++ .../receipts/lambda.json | 449 +++++++++++++++++ .../version | 1 + .../red-cells-rungs-file-missing/expected_rc | 1 + .../red-cells-rungs-file-missing/ladder.yaml | 45 ++ .../red-cells-rungs-file-missing/must_match | 1 + .../receipts/gx10.json | 457 +++++++++++++++++ .../receipts/lambda.json | 449 +++++++++++++++++ .../red-cells-rungs-file-missing/version | 1 + .../context-rungs.json | 38 ++ .../expected_rc | 1 + .../ladder.yaml | 45 ++ .../must_match | 1 + .../receipts/gx10.json | 457 +++++++++++++++++ .../receipts/lambda.json | 316 ++++++++++++ .../version | 1 + .../context-rungs.json | 38 ++ .../expected_rc | 1 + .../ladder.yaml | 45 ++ .../must_match | 1 + .../receipts/gx10.json | 457 +++++++++++++++++ .../receipts/lambda.json | 449 +++++++++++++++++ .../red-cells-thinking-never-closed/version | 1 + .../red-cells-verb-dropped/context-rungs.json | 38 ++ .../red-cells-verb-dropped/expected_rc | 1 + .../red-cells-verb-dropped/ladder.yaml | 44 ++ .../red-cells-verb-dropped/ladder_main.yaml | 45 ++ .../red-cells-verb-dropped/must_match | 1 + .../red-cells-verb-dropped/receipts/gx10.json | 457 +++++++++++++++++ .../receipts/lambda.json | 449 +++++++++++++++++ .../red-cells-verb-dropped/version | 1 + scripts/lib/model_ladder_cells.py | 214 ++++++++ 139 files changed, 18719 insertions(+), 7 deletions(-) create mode 100644 scripts/lib/model_ladder_cases/green-cells/context-rungs.json create mode 100644 scripts/lib/model_ladder_cases/green-cells/expected_rc create mode 100644 scripts/lib/model_ladder_cases/green-cells/ladder.yaml create mode 100644 scripts/lib/model_ladder_cases/green-cells/must_match create mode 100644 scripts/lib/model_ladder_cases/green-cells/receipts/gx10.json create mode 100644 scripts/lib/model_ladder_cases/green-cells/receipts/lambda.json create mode 100644 scripts/lib/model_ladder_cases/green-cells/version create mode 100644 scripts/lib/model_ladder_cases/red-cells-arch-without-representative/context-rungs.json create mode 100644 scripts/lib/model_ladder_cases/red-cells-arch-without-representative/expected_rc create mode 100644 scripts/lib/model_ladder_cases/red-cells-arch-without-representative/ladder.yaml create mode 100644 scripts/lib/model_ladder_cases/red-cells-arch-without-representative/must_match create mode 100644 scripts/lib/model_ladder_cases/red-cells-arch-without-representative/receipts/gx10.json create mode 100644 scripts/lib/model_ladder_cases/red-cells-arch-without-representative/receipts/lambda.json create mode 100644 scripts/lib/model_ladder_cases/red-cells-arch-without-representative/version create mode 100644 scripts/lib/model_ladder_cases/red-cells-block-dropped/context-rungs.json create mode 100644 scripts/lib/model_ladder_cases/red-cells-block-dropped/expected_rc create mode 100644 scripts/lib/model_ladder_cases/red-cells-block-dropped/ladder.yaml create mode 100644 scripts/lib/model_ladder_cases/red-cells-block-dropped/ladder_main.yaml create mode 100644 scripts/lib/model_ladder_cases/red-cells-block-dropped/must_match create mode 100644 scripts/lib/model_ladder_cases/red-cells-block-dropped/receipts/gx10.json create mode 100644 scripts/lib/model_ladder_cases/red-cells-block-dropped/receipts/lambda.json create mode 100644 scripts/lib/model_ladder_cases/red-cells-block-dropped/version create mode 100644 scripts/lib/model_ladder_cases/red-cells-consumer-without-a-number/context-rungs.json create mode 100644 scripts/lib/model_ladder_cases/red-cells-consumer-without-a-number/expected_rc create mode 100644 scripts/lib/model_ladder_cases/red-cells-consumer-without-a-number/ladder.yaml create mode 100644 scripts/lib/model_ladder_cases/red-cells-consumer-without-a-number/must_match create mode 100644 scripts/lib/model_ladder_cases/red-cells-consumer-without-a-number/receipts/gx10.json create mode 100644 scripts/lib/model_ladder_cases/red-cells-consumer-without-a-number/receipts/lambda.json create mode 100644 scripts/lib/model_ladder_cases/red-cells-consumer-without-a-number/version create mode 100644 scripts/lib/model_ladder_cases/red-cells-cotenant-refusal/context-rungs.json create mode 100644 scripts/lib/model_ladder_cases/red-cells-cotenant-refusal/expected_rc create mode 100644 scripts/lib/model_ladder_cases/red-cells-cotenant-refusal/ladder.yaml create mode 100644 scripts/lib/model_ladder_cases/red-cells-cotenant-refusal/must_match create mode 100644 scripts/lib/model_ladder_cases/red-cells-cotenant-refusal/receipts/gx10.json create mode 100644 scripts/lib/model_ladder_cases/red-cells-cotenant-refusal/receipts/lambda.json create mode 100644 scripts/lib/model_ladder_cases/red-cells-cotenant-refusal/version create mode 100644 scripts/lib/model_ladder_cases/red-cells-declared-passes-nowhere/context-rungs.json create mode 100644 scripts/lib/model_ladder_cases/red-cells-declared-passes-nowhere/expected_rc create mode 100644 scripts/lib/model_ladder_cases/red-cells-declared-passes-nowhere/ladder.yaml create mode 100644 scripts/lib/model_ladder_cases/red-cells-declared-passes-nowhere/must_match create mode 100644 scripts/lib/model_ladder_cases/red-cells-declared-passes-nowhere/receipts/gx10.json create mode 100644 scripts/lib/model_ladder_cases/red-cells-declared-passes-nowhere/receipts/lambda.json create mode 100644 scripts/lib/model_ladder_cases/red-cells-declared-passes-nowhere/version create mode 100644 scripts/lib/model_ladder_cases/red-cells-duplicate-row/context-rungs.json create mode 100644 scripts/lib/model_ladder_cases/red-cells-duplicate-row/expected_rc create mode 100644 scripts/lib/model_ladder_cases/red-cells-duplicate-row/ladder.yaml create mode 100644 scripts/lib/model_ladder_cases/red-cells-duplicate-row/must_match create mode 100644 scripts/lib/model_ladder_cases/red-cells-duplicate-row/receipts/gx10.json create mode 100644 scripts/lib/model_ladder_cases/red-cells-duplicate-row/receipts/lambda.json create mode 100644 scripts/lib/model_ladder_cases/red-cells-duplicate-row/version create mode 100644 scripts/lib/model_ladder_cases/red-cells-missing-cell/context-rungs.json create mode 100644 scripts/lib/model_ladder_cases/red-cells-missing-cell/expected_rc create mode 100644 scripts/lib/model_ladder_cases/red-cells-missing-cell/ladder.yaml create mode 100644 scripts/lib/model_ladder_cases/red-cells-missing-cell/must_match create mode 100644 scripts/lib/model_ladder_cases/red-cells-missing-cell/receipts/gx10.json create mode 100644 scripts/lib/model_ladder_cases/red-cells-missing-cell/receipts/lambda.json create mode 100644 scripts/lib/model_ladder_cases/red-cells-missing-cell/version create mode 100644 scripts/lib/model_ladder_cases/red-cells-no-answer/context-rungs.json create mode 100644 scripts/lib/model_ladder_cases/red-cells-no-answer/expected_rc create mode 100644 scripts/lib/model_ladder_cases/red-cells-no-answer/ladder.yaml create mode 100644 scripts/lib/model_ladder_cases/red-cells-no-answer/must_match create mode 100644 scripts/lib/model_ladder_cases/red-cells-no-answer/receipts/gx10.json create mode 100644 scripts/lib/model_ladder_cases/red-cells-no-answer/receipts/lambda.json create mode 100644 scripts/lib/model_ladder_cases/red-cells-no-answer/version create mode 100644 scripts/lib/model_ladder_cases/red-cells-no-cells/context-rungs.json create mode 100644 scripts/lib/model_ladder_cases/red-cells-no-cells/expected_rc create mode 100644 scripts/lib/model_ladder_cases/red-cells-no-cells/ladder.yaml create mode 100644 scripts/lib/model_ladder_cases/red-cells-no-cells/must_match create mode 100644 scripts/lib/model_ladder_cases/red-cells-no-cells/receipts/gx10.json create mode 100644 scripts/lib/model_ladder_cases/red-cells-no-cells/receipts/lambda.json create mode 100644 scripts/lib/model_ladder_cases/red-cells-no-cells/version create mode 100644 scripts/lib/model_ladder_cases/red-cells-no-context-length/context-rungs.json create mode 100644 scripts/lib/model_ladder_cases/red-cells-no-context-length/expected_rc create mode 100644 scripts/lib/model_ladder_cases/red-cells-no-context-length/ladder.yaml create mode 100644 scripts/lib/model_ladder_cases/red-cells-no-context-length/must_match create mode 100644 scripts/lib/model_ladder_cases/red-cells-no-context-length/receipts/gx10.json create mode 100644 scripts/lib/model_ladder_cases/red-cells-no-context-length/receipts/lambda.json create mode 100644 scripts/lib/model_ladder_cases/red-cells-no-context-length/version create mode 100644 scripts/lib/model_ladder_cases/red-cells-oom-not-refusal/context-rungs.json create mode 100644 scripts/lib/model_ladder_cases/red-cells-oom-not-refusal/expected_rc create mode 100644 scripts/lib/model_ladder_cases/red-cells-oom-not-refusal/ladder.yaml create mode 100644 scripts/lib/model_ladder_cases/red-cells-oom-not-refusal/must_match create mode 100644 scripts/lib/model_ladder_cases/red-cells-oom-not-refusal/receipts/gx10.json create mode 100644 scripts/lib/model_ladder_cases/red-cells-oom-not-refusal/receipts/lambda.json create mode 100644 scripts/lib/model_ladder_cases/red-cells-oom-not-refusal/version create mode 100644 scripts/lib/model_ladder_cases/red-cells-pass-fell-back/context-rungs.json create mode 100644 scripts/lib/model_ladder_cases/red-cells-pass-fell-back/expected_rc create mode 100644 scripts/lib/model_ladder_cases/red-cells-pass-fell-back/ladder.yaml create mode 100644 scripts/lib/model_ladder_cases/red-cells-pass-fell-back/must_match create mode 100644 scripts/lib/model_ladder_cases/red-cells-pass-fell-back/receipts/gx10.json create mode 100644 scripts/lib/model_ladder_cases/red-cells-pass-fell-back/receipts/lambda.json create mode 100644 scripts/lib/model_ladder_cases/red-cells-pass-fell-back/version create mode 100644 scripts/lib/model_ladder_cases/red-cells-prompt-under-rung/context-rungs.json create mode 100644 scripts/lib/model_ladder_cases/red-cells-prompt-under-rung/expected_rc create mode 100644 scripts/lib/model_ladder_cases/red-cells-prompt-under-rung/ladder.yaml create mode 100644 scripts/lib/model_ladder_cases/red-cells-prompt-under-rung/must_match create mode 100644 scripts/lib/model_ladder_cases/red-cells-prompt-under-rung/receipts/gx10.json create mode 100644 scripts/lib/model_ladder_cases/red-cells-prompt-under-rung/receipts/lambda.json create mode 100644 scripts/lib/model_ladder_cases/red-cells-prompt-under-rung/version create mode 100644 scripts/lib/model_ladder_cases/red-cells-representative-held-nowhere/context-rungs.json create mode 100644 scripts/lib/model_ladder_cases/red-cells-representative-held-nowhere/expected_rc create mode 100644 scripts/lib/model_ladder_cases/red-cells-representative-held-nowhere/ladder.yaml create mode 100644 scripts/lib/model_ladder_cases/red-cells-representative-held-nowhere/must_match create mode 100644 scripts/lib/model_ladder_cases/red-cells-representative-held-nowhere/receipts/gx10.json create mode 100644 scripts/lib/model_ladder_cases/red-cells-representative-held-nowhere/receipts/lambda.json create mode 100644 scripts/lib/model_ladder_cases/red-cells-representative-held-nowhere/version create mode 100644 scripts/lib/model_ladder_cases/red-cells-rungs-file-missing/expected_rc create mode 100644 scripts/lib/model_ladder_cases/red-cells-rungs-file-missing/ladder.yaml create mode 100644 scripts/lib/model_ladder_cases/red-cells-rungs-file-missing/must_match create mode 100644 scripts/lib/model_ladder_cases/red-cells-rungs-file-missing/receipts/gx10.json create mode 100644 scripts/lib/model_ladder_cases/red-cells-rungs-file-missing/receipts/lambda.json create mode 100644 scripts/lib/model_ladder_cases/red-cells-rungs-file-missing/version create mode 100644 scripts/lib/model_ladder_cases/red-cells-thinking-modes-disagree-with-template/context-rungs.json create mode 100644 scripts/lib/model_ladder_cases/red-cells-thinking-modes-disagree-with-template/expected_rc create mode 100644 scripts/lib/model_ladder_cases/red-cells-thinking-modes-disagree-with-template/ladder.yaml create mode 100644 scripts/lib/model_ladder_cases/red-cells-thinking-modes-disagree-with-template/must_match create mode 100644 scripts/lib/model_ladder_cases/red-cells-thinking-modes-disagree-with-template/receipts/gx10.json create mode 100644 scripts/lib/model_ladder_cases/red-cells-thinking-modes-disagree-with-template/receipts/lambda.json create mode 100644 scripts/lib/model_ladder_cases/red-cells-thinking-modes-disagree-with-template/version create mode 100644 scripts/lib/model_ladder_cases/red-cells-thinking-never-closed/context-rungs.json create mode 100644 scripts/lib/model_ladder_cases/red-cells-thinking-never-closed/expected_rc create mode 100644 scripts/lib/model_ladder_cases/red-cells-thinking-never-closed/ladder.yaml create mode 100644 scripts/lib/model_ladder_cases/red-cells-thinking-never-closed/must_match create mode 100644 scripts/lib/model_ladder_cases/red-cells-thinking-never-closed/receipts/gx10.json create mode 100644 scripts/lib/model_ladder_cases/red-cells-thinking-never-closed/receipts/lambda.json create mode 100644 scripts/lib/model_ladder_cases/red-cells-thinking-never-closed/version create mode 100644 scripts/lib/model_ladder_cases/red-cells-verb-dropped/context-rungs.json create mode 100644 scripts/lib/model_ladder_cases/red-cells-verb-dropped/expected_rc create mode 100644 scripts/lib/model_ladder_cases/red-cells-verb-dropped/ladder.yaml create mode 100644 scripts/lib/model_ladder_cases/red-cells-verb-dropped/ladder_main.yaml create mode 100644 scripts/lib/model_ladder_cases/red-cells-verb-dropped/must_match create mode 100644 scripts/lib/model_ladder_cases/red-cells-verb-dropped/receipts/gx10.json create mode 100644 scripts/lib/model_ladder_cases/red-cells-verb-dropped/receipts/lambda.json create mode 100644 scripts/lib/model_ladder_cases/red-cells-verb-dropped/version create mode 100644 scripts/lib/model_ladder_cells.py diff --git a/contracts/model-capability-ladder-v1.yaml b/contracts/model-capability-ladder-v1.yaml index a006a7526e..d091e9049e 100644 --- a/contracts/model-capability-ladder-v1.yaml +++ b/contracts/model-capability-ladder-v1.yaml @@ -62,6 +62,23 @@ ladder: dirs: ["~/models", "~/.apr/models", "~/.cache/apr/models"] # depth 1, the fleet's model dirs patterns: ["*q4_k*.gguf", "*q4k*.gguf", "*q4_k*.apr", "*q4k*.apr"] # case-insensitive backends: [cuda] + # #3712 row B (operator 2026-09-21: "MORE than enough tokens for our needs (overdo it), and our verbs are + # dogfooded, i.e chat with and without thinking, ditto run, ditto code"). A cell is (model, host, verb, + # thinking, context rung). The RUNGS are not declared here: ids and token counts live in ONE file, + # evidence/release/context-rungs.json (apr-release-context-rungs/v1, #3715), which pv reads too; + # consumer-max is the max over its consumers[].max_prompt_tokens. Thinking modes are derived per model + # from its chat template (#3723). A cell fits a host iff weights + kv_bytes_per_token * tokens + workspace + # <= the host's TOTAL GPU memory (operator "a", #3710): a fitting cell must pass, a non-fitting one must be + # an honest pre-load refusal, and every owed cell must pass on >= 1 required host. Long rungs are owed by + # the families below plus ONE representative per other arch, named here by file (cop 2026-09-21). + cells: + verbs: [run, chat, serve, code] + long_rungs_for: + families: [qwen35] # every consumer's family: the full rung set, through declared + representatives: # one file per other arch the fleet holds + qwen2: qwen2.5-coder-7b-instruct-q4_k_m.gguf # lambda + gx10 + qwen3: Qwen3-8B-Q4_K_M.gguf # lambda + gx10 + qwen3moe: Qwen3-Coder-30B-A3B-Instruct-Q4_K_M.gguf # gx10 (128 GB unified); fits no 24 GB card long rungs: - id: qwen2-1.5b-q4km family: qwen2.5 @@ -190,6 +207,11 @@ proof_obligations: property: "No Q4_K rung is optional, and every one claims CUDA" formal: "∀ rung ∈ ladder.rungs: q4k(rung) ⟹ rung.required = true ∧ cuda ∈ rung.backends" applies_to: gate_verdict + - id: MCL-INV-008 + type: invariant + property: "Every owed (model, verb, thinking, rung) cell is proved: pass where it fits, an honest refusal where it does not, and a pass on at least one required host" + formal: "∀ host, m ∈ inventory(host), v ∈ cells.verbs, t ∈ modes(template(m)), r ∈ rungs(m): |rows(host, m, v, t, r)| = 1 ∧ (fits(m, r, total(host)) ⟹ pass(row) ∧ backend = cuda ∧ ¬fallback ∧ prompt_tokens ≥ tokens(r) ∧ answer_chars > 0 ∧ (t = on ⟹ think_closed)) ∧ (¬fits ⟹ refused(row) ∧ required_bytes(row) > total(host)) ∧ ∃ host': pass(host', m, v, t, r)" + applies_to: gate_verdict - id: MCL-INV-007 type: invariant property: "The universe is the host's measured inventory — every Q4_K model it holds is in the run and green on CUDA" @@ -273,6 +295,12 @@ falsification_tests: test: "bash scripts/check_model_ladder.sh --self-test" if_fails: "a ladder run on gx10's unified memory OOM-kills the CI pool (15:56:08Z, 18 kills in 10 s) or hangs a release on a stuck lock" + - id: FALSIFY-MCL-016 + rule: "Cells: verbs x thinking x context rung, derived and proved" + prediction: "check_model_ladder.sh --self-test: green-cells passes (20 owed on lambda, 12 pass, 8 honest refusals); a missing cell, a duplicate row, a fallback, a thinking block that never closed, a prompt under its rung, an empty answer, a co-tenant refusal, an OOM instead of a refusal, a rung that passes nowhere, thinking_modes that disagree with the template, a missing context_length (#3733), no cells, a consumer without a number, a missing context-rungs file, a dropped cells block or verb, and an arch with no held representative are each RED naming it; 8 module mutants are each killed by their case" + test: "bash scripts/check_model_ladder.sh --self-test" + if_fails: "a release dogfoods one verb at one short prompt with thinking off and calls the model family working" + qa_gate: id: F-MCL-001 name: Model Capability Ladder diff --git a/docs/roadmaps/entries/PMAT-3712.yaml b/docs/roadmaps/entries/PMAT-3712.yaml index 69f9f9c31b..7306f6900d 100644 --- a/docs/roadmaps/entries/PMAT-3712.yaml +++ b/docs/roadmaps/entries/PMAT-3712.yaml @@ -14,4 +14,4 @@ estimated_effort: null labels: - kind:code - notes: 'ACCEPTANCE (hand-entered from gh#3712; this row = done_when 1 + 2, the GATE side; cop assignment 2026-09-21: worker B). Issue done_when 1, verbatim: "Universe = measured inventory, not a list. The release gate derives the model set from what''s ON each required host (every `*Q4_K*` GGUF/APR under the declared models dir on lambda AND gx10), unioned with the ladder. A model present on a host but missing from the gate''s run is a FAIL naming it. The ladder has no `required: false` for any Q4_K rung; a guard refuses the key on a Q4_K rung (case row + mutant)." Issue done_when 2, verbatim: "Every (model, host) cell is GREEN on CUDA: capability_match passed (not skipped), golden_output passed (not skipped; #3711), `apr run --gpu` rc 0 with no fallback line. One red cell = NO-GO. There is no known, optional or pre-existing exemption, and no threshold like >= N%." WHAT THIS ROW DOES: (a) contracts/model-capability-ladder-v1.yaml gains `ladder.inventory` {dirs, patterns (case-insensitive *q4_k*/*q4k* .gguf/.apr), backends [cuda]}; (b) scripts/model_ladder.sh measures ladder UNION the host''s inventory (every match, depth 1), judges each inventory model on cuda with the SAME measure() as a rung, and writes receipt schema apr-model-ladder-receipt/v2 carrying `inventory` [{file, sha256, bytes}], `file` per row, and `apr_sha` (full 40-hex, for #3715); (c) scripts/check_model_ladder.sh FAILs: `required` not true on a Q4_K rung; a Q4_K rung that does not claim cuda; a ladder with no inventory; a receipt that is not v2 or has no inventory; an EMPTY inventory; an inventory file with no present row (MISSING from the run, named); an inventory-only model not green on cuda (skip, fallback, rc != 0 are RED); (d) case table grows 18 -> 26 (every case red for exactly its own reason, one FAIL line each except the two-host cases), plus 3 self-mutants each killed by its case (q4k-required-false, q4k-without-cuda, inventory-missing); --case with no such case is now RED, and the root is derived from the script path, not git rev-parse (#3581). WHAT IT DOES NOT DO (cop ruling 2026-09-21): qwen3-8b-q4km stays `required: false` in the contract, and the gate refuses that key at T-2, so the release cannot ship on it. Flipping it here would turn pv lint red on EVERY PR: the armed ladder-green shape reads the committed 0.68.2 lambda receipt (golden_output: Empty output), measured rc 0 -> 1 with only that flip. The flip folds into the 0.69.1 batch with the qwen3-8b fix and a committed green lambda receipt. So done_when 1''s `no required: false` clause is NOT met by this row: Refs, not Closes. Done_when 2''s cells turn green only through the model fixes under EPIC #3710; done_when 3 and 4 are aprender-f0''s (#3708). cells[] per verb x context (the widened bar, #3715 proposal) ships with the cop''s widened-bar delta, not here. WHERE THE GATE RUNS (so its rc=1 on the real contract is the release NO-GO, not a PR red): check_model_ladder.sh is declared ONLY in Cargo.toml [package.metadata.dogfood] gates (line 612, the T-2 pre-publish dogfood) and is listed in scripts/unwired_guards_baseline.txt (line 12): no PR workflow runs it, and guard_tree skips it (unwired-baseline). pv lint is a different tool that never runs this script; it reads the contract and PASSES with qwen3-8b at required: false. Measured on this head: guard_tree --no-cargo 0 failed, pv lint contracts/ PASS, cargo test -p aprender-contracts --lib 0 failed. THE GPU LOCK (cop ruling 2026-09-21, after a gx10 global OOM at 15:56:08Z killed CI containers): every apr call in model_ladder.sh goes through apr_locked = flock -E 75 -w ${MODEL_LADDER_LOCK_WAIT:-1800} /tmp/apr-gpu.lock choom -n 1000 --; a lock still held after the wait is an ENV decline (exit 2) naming the holder pid from /proc/locks, never a hang and never a model verdict. T-1 wraps the script in choom only (a second flock outside would deadlock). check_model_ladder.sh audits it statically in the REAL run (a raw "$APR" call is RED) and behaviourally in --self-test (fake apr via --lock-probe: lock held + oom 1000; a held lock declines naming the pid), with four producer mutants each killed: raw-apr-call, no-lock, no-choom, unbounded. FALSIFY-MCL-015.' + notes: 'ACCEPTANCE (hand-entered from gh#3712; this row = done_when 1 + 2, the GATE side; cop assignment 2026-09-21: worker B). Issue done_when 1, verbatim: "Universe = measured inventory, not a list. The release gate derives the model set from what''s ON each required host (every `*Q4_K*` GGUF/APR under the declared models dir on lambda AND gx10), unioned with the ladder. A model present on a host but missing from the gate''s run is a FAIL naming it. The ladder has no `required: false` for any Q4_K rung; a guard refuses the key on a Q4_K rung (case row + mutant)." Issue done_when 2, verbatim: "Every (model, host) cell is GREEN on CUDA: capability_match passed (not skipped), golden_output passed (not skipped; #3711), `apr run --gpu` rc 0 with no fallback line. One red cell = NO-GO. There is no known, optional or pre-existing exemption, and no threshold like >= N%." WHAT THIS ROW DOES: (a) contracts/model-capability-ladder-v1.yaml gains `ladder.inventory` {dirs, patterns (case-insensitive *q4_k*/*q4k* .gguf/.apr), backends [cuda]}; (b) scripts/model_ladder.sh measures ladder UNION the host''s inventory (every match, depth 1), judges each inventory model on cuda with the SAME measure() as a rung, and writes receipt schema apr-model-ladder-receipt/v2 carrying `inventory` [{file, sha256, bytes}], `file` per row, and `apr_sha` (full 40-hex, for #3715); (c) scripts/check_model_ladder.sh FAILs: `required` not true on a Q4_K rung; a Q4_K rung that does not claim cuda; a ladder with no inventory; a receipt that is not v2 or has no inventory; an EMPTY inventory; an inventory file with no present row (MISSING from the run, named); an inventory-only model not green on cuda (skip, fallback, rc != 0 are RED); (d) case table grows 18 -> 26 (every case red for exactly its own reason, one FAIL line each except the two-host cases), plus 3 self-mutants each killed by its case (q4k-required-false, q4k-without-cuda, inventory-missing); --case with no such case is now RED, and the root is derived from the script path, not git rev-parse (#3581). WHAT IT DOES NOT DO (cop ruling 2026-09-21): qwen3-8b-q4km stays `required: false` in the contract, and the gate refuses that key at T-2, so the release cannot ship on it. Flipping it here would turn pv lint red on EVERY PR: the armed ladder-green shape reads the committed 0.68.2 lambda receipt (golden_output: Empty output), measured rc 0 -> 1 with only that flip. The flip folds into the 0.69.1 batch with the qwen3-8b fix and a committed green lambda receipt. So done_when 1''s `no required: false` clause is NOT met by this row: Refs, not Closes. Done_when 2''s cells turn green only through the model fixes under EPIC #3710; done_when 3 and 4 are aprender-f0''s (#3708). cells[] per verb x context (the widened bar, #3715 proposal) ships with the cop''s widened-bar delta, not here. WHERE THE GATE RUNS (so its rc=1 on the real contract is the release NO-GO, not a PR red): check_model_ladder.sh is declared ONLY in Cargo.toml [package.metadata.dogfood] gates (line 612, the T-2 pre-publish dogfood) and is listed in scripts/unwired_guards_baseline.txt (line 12): no PR workflow runs it, and guard_tree skips it (unwired-baseline). pv lint is a different tool that never runs this script; it reads the contract and PASSES with qwen3-8b at required: false. Measured on this head: guard_tree --no-cargo 0 failed, pv lint contracts/ PASS, cargo test -p aprender-contracts --lib 0 failed. THE GPU LOCK (cop ruling 2026-09-21, after a gx10 global OOM at 15:56:08Z killed CI containers): every apr call in model_ladder.sh goes through apr_locked = flock -E 75 -w ${MODEL_LADDER_LOCK_WAIT:-1800} /tmp/apr-gpu.lock choom -n 1000 --; a lock still held after the wait is an ENV decline (exit 2) naming the holder pid from /proc/locks, never a hang and never a model verdict. T-1 wraps the script in choom only (a second flock outside would deadlock). check_model_ladder.sh audits it statically in the REAL run (a raw "$APR" call is RED) and behaviourally in --self-test (fake apr via --lock-probe: lock held + oom 1000; a held lock declines naming the pid), with four producer mutants each killed: raw-apr-call, no-lock, no-choom, unbounded. FALSIFY-MCL-015. ROW B1 (the judge side of cells; stacked on row A; operator bar raise 2026-09-21: "lets ensure this release has MORE than enough tokens for our needs (overdo it), and our verbs are dogfooded, i.e chat with and without thinking, ditto run, ditto code, etc,"). A cell is (model, host, verb, thinking, context rung). contracts/model-capability-ladder-v1.yaml gains `ladder.cells` {verbs [run, chat, serve, code], long_rungs_for {families [qwen35], representatives: one file per other arch}}. The RUNGS are deliberately NOT in the contract: they live in evidence/release/context-rungs.json (apr-release-context-rungs/v1, landing in #3715), one declaration shared with pv; consumer-max = max over consumers[].max_prompt_tokens. The judge (scripts/lib/model_ladder_cells.py, called from check_model_ladder.sh) derives the owed set from the MEASURED inventory: short rungs for every model up to its context_length, long rungs for qwen35 and each arch''s representative; thinking modes from the template evidence (enable_thinking -> both; always-open -> on; else off, #3723), never the producer''s claim; fit := weights + kv_bytes_per_token * tokens + workspace <= gpu_mem_total_bytes (operator ''a'', #3710; an unmeasured term keeps the cell owed in full). A fitting cell must pass (cuda, no fallback, rc 0, prompt_tokens >= rung, answer_chars > 0, think_closed when on); a non-fitting cell must be `refused` with required_bytes > the host TOTAL (a co-tenant refusal is RED); every owed cell must pass on >= 1 required host; every arch in any inventory has a representative some required host holds. A missing context-rungs file, a consumer without a number, a missing context_length (#3733) and a receipt with no cells are each a named FAIL, never an empty owed set. The cells block is an anti-shrink floor vs origin/main (block, verbs, rung ids, families). Case table 26 -> 45; 8 module mutants each killed by its case. The PRODUCER side (row B2: measuring the cells) is not in this row: it needs #3723 (thinking toggle), #3733 (inspect keys) and #3596 (capacity::plan, the pre-load refusal). Until it lands, the real T-2 run is RED on ''receipt carries no cells'', which is the operator''s bar, not a PR red: check_model_ladder.sh runs only in the release dogfood (Cargo.toml [package.metadata.dogfood]) and is listed in scripts/unwired_guards_baseline.txt.' diff --git a/docs/roadmaps/roadmap.yaml b/docs/roadmaps/roadmap.yaml index c044f2da80..0c02d574a1 100644 --- a/docs/roadmaps/roadmap.yaml +++ b/docs/roadmaps/roadmap.yaml @@ -19962,4 +19962,4 @@ roadmap: estimated_effort: null labels: - kind:code - notes: 'ACCEPTANCE (hand-entered from gh#3712; this row = done_when 1 + 2, the GATE side; cop assignment 2026-09-21: worker B). Issue done_when 1, verbatim: "Universe = measured inventory, not a list. The release gate derives the model set from what''s ON each required host (every `*Q4_K*` GGUF/APR under the declared models dir on lambda AND gx10), unioned with the ladder. A model present on a host but missing from the gate''s run is a FAIL naming it. The ladder has no `required: false` for any Q4_K rung; a guard refuses the key on a Q4_K rung (case row + mutant)." Issue done_when 2, verbatim: "Every (model, host) cell is GREEN on CUDA: capability_match passed (not skipped), golden_output passed (not skipped; #3711), `apr run --gpu` rc 0 with no fallback line. One red cell = NO-GO. There is no known, optional or pre-existing exemption, and no threshold like >= N%." WHAT THIS ROW DOES: (a) contracts/model-capability-ladder-v1.yaml gains `ladder.inventory` {dirs, patterns (case-insensitive *q4_k*/*q4k* .gguf/.apr), backends [cuda]}; (b) scripts/model_ladder.sh measures ladder UNION the host''s inventory (every match, depth 1), judges each inventory model on cuda with the SAME measure() as a rung, and writes receipt schema apr-model-ladder-receipt/v2 carrying `inventory` [{file, sha256, bytes}], `file` per row, and `apr_sha` (full 40-hex, for #3715); (c) scripts/check_model_ladder.sh FAILs: `required` not true on a Q4_K rung; a Q4_K rung that does not claim cuda; a ladder with no inventory; a receipt that is not v2 or has no inventory; an EMPTY inventory; an inventory file with no present row (MISSING from the run, named); an inventory-only model not green on cuda (skip, fallback, rc != 0 are RED); (d) case table grows 18 -> 26 (every case red for exactly its own reason, one FAIL line each except the two-host cases), plus 3 self-mutants each killed by its case (q4k-required-false, q4k-without-cuda, inventory-missing); --case with no such case is now RED, and the root is derived from the script path, not git rev-parse (#3581). WHAT IT DOES NOT DO (cop ruling 2026-09-21): qwen3-8b-q4km stays `required: false` in the contract, and the gate refuses that key at T-2, so the release cannot ship on it. Flipping it here would turn pv lint red on EVERY PR: the armed ladder-green shape reads the committed 0.68.2 lambda receipt (golden_output: Empty output), measured rc 0 -> 1 with only that flip. The flip folds into the 0.69.1 batch with the qwen3-8b fix and a committed green lambda receipt. So done_when 1''s `no required: false` clause is NOT met by this row: Refs, not Closes. Done_when 2''s cells turn green only through the model fixes under EPIC #3710; done_when 3 and 4 are aprender-f0''s (#3708). cells[] per verb x context (the widened bar, #3715 proposal) ships with the cop''s widened-bar delta, not here. WHERE THE GATE RUNS (so its rc=1 on the real contract is the release NO-GO, not a PR red): check_model_ladder.sh is declared ONLY in Cargo.toml [package.metadata.dogfood] gates (line 612, the T-2 pre-publish dogfood) and is listed in scripts/unwired_guards_baseline.txt (line 12): no PR workflow runs it, and guard_tree skips it (unwired-baseline). pv lint is a different tool that never runs this script; it reads the contract and PASSES with qwen3-8b at required: false. Measured on this head: guard_tree --no-cargo 0 failed, pv lint contracts/ PASS, cargo test -p aprender-contracts --lib 0 failed. THE GPU LOCK (cop ruling 2026-09-21, after a gx10 global OOM at 15:56:08Z killed CI containers): every apr call in model_ladder.sh goes through apr_locked = flock -E 75 -w ${MODEL_LADDER_LOCK_WAIT:-1800} /tmp/apr-gpu.lock choom -n 1000 --; a lock still held after the wait is an ENV decline (exit 2) naming the holder pid from /proc/locks, never a hang and never a model verdict. T-1 wraps the script in choom only (a second flock outside would deadlock). check_model_ladder.sh audits it statically in the REAL run (a raw "$APR" call is RED) and behaviourally in --self-test (fake apr via --lock-probe: lock held + oom 1000; a held lock declines naming the pid), with four producer mutants each killed: raw-apr-call, no-lock, no-choom, unbounded. FALSIFY-MCL-015.' + notes: 'ACCEPTANCE (hand-entered from gh#3712; this row = done_when 1 + 2, the GATE side; cop assignment 2026-09-21: worker B). Issue done_when 1, verbatim: "Universe = measured inventory, not a list. The release gate derives the model set from what''s ON each required host (every `*Q4_K*` GGUF/APR under the declared models dir on lambda AND gx10), unioned with the ladder. A model present on a host but missing from the gate''s run is a FAIL naming it. The ladder has no `required: false` for any Q4_K rung; a guard refuses the key on a Q4_K rung (case row + mutant)." Issue done_when 2, verbatim: "Every (model, host) cell is GREEN on CUDA: capability_match passed (not skipped), golden_output passed (not skipped; #3711), `apr run --gpu` rc 0 with no fallback line. One red cell = NO-GO. There is no known, optional or pre-existing exemption, and no threshold like >= N%." WHAT THIS ROW DOES: (a) contracts/model-capability-ladder-v1.yaml gains `ladder.inventory` {dirs, patterns (case-insensitive *q4_k*/*q4k* .gguf/.apr), backends [cuda]}; (b) scripts/model_ladder.sh measures ladder UNION the host''s inventory (every match, depth 1), judges each inventory model on cuda with the SAME measure() as a rung, and writes receipt schema apr-model-ladder-receipt/v2 carrying `inventory` [{file, sha256, bytes}], `file` per row, and `apr_sha` (full 40-hex, for #3715); (c) scripts/check_model_ladder.sh FAILs: `required` not true on a Q4_K rung; a Q4_K rung that does not claim cuda; a ladder with no inventory; a receipt that is not v2 or has no inventory; an EMPTY inventory; an inventory file with no present row (MISSING from the run, named); an inventory-only model not green on cuda (skip, fallback, rc != 0 are RED); (d) case table grows 18 -> 26 (every case red for exactly its own reason, one FAIL line each except the two-host cases), plus 3 self-mutants each killed by its case (q4k-required-false, q4k-without-cuda, inventory-missing); --case with no such case is now RED, and the root is derived from the script path, not git rev-parse (#3581). WHAT IT DOES NOT DO (cop ruling 2026-09-21): qwen3-8b-q4km stays `required: false` in the contract, and the gate refuses that key at T-2, so the release cannot ship on it. Flipping it here would turn pv lint red on EVERY PR: the armed ladder-green shape reads the committed 0.68.2 lambda receipt (golden_output: Empty output), measured rc 0 -> 1 with only that flip. The flip folds into the 0.69.1 batch with the qwen3-8b fix and a committed green lambda receipt. So done_when 1''s `no required: false` clause is NOT met by this row: Refs, not Closes. Done_when 2''s cells turn green only through the model fixes under EPIC #3710; done_when 3 and 4 are aprender-f0''s (#3708). cells[] per verb x context (the widened bar, #3715 proposal) ships with the cop''s widened-bar delta, not here. WHERE THE GATE RUNS (so its rc=1 on the real contract is the release NO-GO, not a PR red): check_model_ladder.sh is declared ONLY in Cargo.toml [package.metadata.dogfood] gates (line 612, the T-2 pre-publish dogfood) and is listed in scripts/unwired_guards_baseline.txt (line 12): no PR workflow runs it, and guard_tree skips it (unwired-baseline). pv lint is a different tool that never runs this script; it reads the contract and PASSES with qwen3-8b at required: false. Measured on this head: guard_tree --no-cargo 0 failed, pv lint contracts/ PASS, cargo test -p aprender-contracts --lib 0 failed. THE GPU LOCK (cop ruling 2026-09-21, after a gx10 global OOM at 15:56:08Z killed CI containers): every apr call in model_ladder.sh goes through apr_locked = flock -E 75 -w ${MODEL_LADDER_LOCK_WAIT:-1800} /tmp/apr-gpu.lock choom -n 1000 --; a lock still held after the wait is an ENV decline (exit 2) naming the holder pid from /proc/locks, never a hang and never a model verdict. T-1 wraps the script in choom only (a second flock outside would deadlock). check_model_ladder.sh audits it statically in the REAL run (a raw "$APR" call is RED) and behaviourally in --self-test (fake apr via --lock-probe: lock held + oom 1000; a held lock declines naming the pid), with four producer mutants each killed: raw-apr-call, no-lock, no-choom, unbounded. FALSIFY-MCL-015. ROW B1 (the judge side of cells; stacked on row A; operator bar raise 2026-09-21: "lets ensure this release has MORE than enough tokens for our needs (overdo it), and our verbs are dogfooded, i.e chat with and without thinking, ditto run, ditto code, etc,"). A cell is (model, host, verb, thinking, context rung). contracts/model-capability-ladder-v1.yaml gains `ladder.cells` {verbs [run, chat, serve, code], long_rungs_for {families [qwen35], representatives: one file per other arch}}. The RUNGS are deliberately NOT in the contract: they live in evidence/release/context-rungs.json (apr-release-context-rungs/v1, landing in #3715), one declaration shared with pv; consumer-max = max over consumers[].max_prompt_tokens. The judge (scripts/lib/model_ladder_cells.py, called from check_model_ladder.sh) derives the owed set from the MEASURED inventory: short rungs for every model up to its context_length, long rungs for qwen35 and each arch''s representative; thinking modes from the template evidence (enable_thinking -> both; always-open -> on; else off, #3723), never the producer''s claim; fit := weights + kv_bytes_per_token * tokens + workspace <= gpu_mem_total_bytes (operator ''a'', #3710; an unmeasured term keeps the cell owed in full). A fitting cell must pass (cuda, no fallback, rc 0, prompt_tokens >= rung, answer_chars > 0, think_closed when on); a non-fitting cell must be `refused` with required_bytes > the host TOTAL (a co-tenant refusal is RED); every owed cell must pass on >= 1 required host; every arch in any inventory has a representative some required host holds. A missing context-rungs file, a consumer without a number, a missing context_length (#3733) and a receipt with no cells are each a named FAIL, never an empty owed set. The cells block is an anti-shrink floor vs origin/main (block, verbs, rung ids, families). Case table 26 -> 45; 8 module mutants each killed by its case. The PRODUCER side (row B2: measuring the cells) is not in this row: it needs #3723 (thinking toggle), #3733 (inspect keys) and #3596 (capacity::plan, the pre-load refusal). Until it lands, the real T-2 run is RED on ''receipt carries no cells'', which is the operator''s bar, not a PR red: check_model_ladder.sh runs only in the release dogfood (Cargo.toml [package.metadata.dogfood]) and is listed in scripts/unwired_guards_baseline.txt.' diff --git a/scripts/check_model_ladder.sh b/scripts/check_model_ladder.sh index a190fe683d..c572716677 100755 --- a/scripts/check_model_ladder.sh +++ b/scripts/check_model_ladder.sh @@ -52,11 +52,13 @@ SELF="$(cd "$(dirname "$0")" && pwd)/$(basename "$0")" # before the cd: the mu cd "${MODEL_LADDER_ROOT:-$(dirname "$SELF")/..}" || exit 2 # ---------------------------------------------------------------- the judge -# judge → exit 0/1/2 +# judge [context-rungs.json] → exit 0/1/2 judge() { - python3 - "$1" "$2" "$3" "$4" <<'PY' + python3 - "$1" "$2" "$3" "$4" "${5:-}" <<'PY' import json, os, sys, yaml -ladder_p, main_p, rdir, version = sys.argv[1:5] +ladder_p, main_p, rdir, version, rungs_p = sys.argv[1:6] +sys.path.insert(0, os.environ.get("MODEL_LADDER_CELLS_LIB") or "scripts/lib") # a mutant copy of the module, in --self-test +import model_ladder_cells try: L = yaml.safe_load(open(ladder_p))["ladder"] except Exception as e: @@ -109,6 +111,15 @@ if main_p and os.path.exists(main_p): for r in rungs: if r.get("required") and r["id"] in mrh and not mrh[r["id"]] <= hostset(r, allh): print(f"FAIL hosts DROPPED on {r['id']} vs origin/main: {sorted(mrh[r['id']] - hostset(r, allh))} — a rung may gain hosts, never lose one"); rc = 1 + mc, hc = M.get("cells") or {}, L.get("cells") or {} + if mc and not hc: + print("FAIL the cells block DROPPED vs origin/main -- verbs x thinking x context would owe nothing"); rc = 1 + elif mc: + for what, a, b in (("verbs", mc.get("verbs"), hc.get("verbs")), + ("rungs", [r["id"] for r in mc.get("rungs") or []], [r["id"] for r in hc.get("rungs") or []]), + ("long-rung families", (mc.get("long_rungs_for") or {}).get("families"), (hc.get("long_rungs_for") or {}).get("families"))): + gone = set(a or []) - set(b or []) + if gone: print(f"FAIL cells {what} DROPPED vs origin/main: {sorted(gone)}"); rc = 1 mh = {h["id"] for h in M.get("hosts", []) if h.get("required")} for hid in mh - {h["id"] for h in hosts}: print(f"FAIL required host DROPPED vs origin/main: {hid}"); rc = 1 @@ -117,6 +128,7 @@ if main_p and os.path.exists(main_p): print(f"FAIL ladder at origin/main unreadable: {e}"); rc = 1 else: print("! BOOTSTRAP: no ladder at origin/main yet; the anti-shrink floor arms when this lands") +good = {} # host id -> a receipt that passed the host-level checks; the cells judge reads only these for h in hosts: f = os.path.join(rdir, f"{h['id']}.json") if not os.path.exists(f): @@ -134,6 +146,7 @@ for h in hosts: print(f"FAIL {h['id']:7} receipt carries no measured inventory (schema {R.get('schema')!r}) — the universe is what the host HOLDS, not a list (#3712)"); rc = 1; continue if not inv: print(f"FAIL {h['id']:7} measured inventory is EMPTY — a host holding no Q4_K model proved nothing (#3712)"); rc = 1; continue + good[h["id"]] = R by = {r.get("id"): r for r in R.get("rungs", [])} by_file = {x.get("file"): x for x in R.get("rungs", []) if x.get("file")} ladder_files = {r.get("gguf") for r in rungs} @@ -168,6 +181,15 @@ for h in hosts: else: print(f"warn {h['id']:7} {rid:22} ({tag}) " + "; ".join(why)) else: print(f"ok {h['id']:7} {rid:22} green on {','.join(r.get('backends', []))} ({R.get('gpu') or 'no-gpu'}, sha {R.get('sha')})") +# ---- cells: verb x thinking x context rung, per model per host (#3712 row B) +rungs_doc = None +if rungs_p and os.path.exists(rungs_p): + try: + rungs_doc = json.load(open(rungs_p)) + except Exception as e: + print(f"FAIL context rungs {rungs_p} unreadable: {e}") +if model_ladder_cells.judge(L, good, rungs_doc, print): + rc = 1 sys.exit(rc) PY } @@ -221,7 +243,7 @@ if [ "$SELF_TEST" = 1 ]; then want=$(cat "$c/expected_rc") lad="$c/ladder.yaml"; [ -f "$lad" ] || lad="$LADDER" main=""; [ -f "$c/ladder_main.yaml" ] && main="$c/ladder_main.yaml" - out=$(judge "$lad" "$main" "$c/receipts" "$(cat "$c/version" 2>/dev/null || echo 0.0.0-case)"); got=$? + out=$(judge "$lad" "$main" "$c/receipts" "$(cat "$c/version" 2>/dev/null || echo 0.0.0-case)" "$c/context-rungs.json"); got=$? n=$((n+1)) if [ "$got" = "$want" ] && { [ ! -f "$c/must_match" ] || grep -qE "$(cat "$c/must_match")" <<< "$out"; }; then printf 'ok case %-28s rc=%s\n' "$name" "$got" @@ -262,6 +284,23 @@ if [ "$SELF_TEST" = 1 ]; then pmutant no-lock 's/^apr_locked() { flock -E "\$LOCK_BUSY" -w "\$LOCK_WAIT" "\$GPU_LOCK" choom/apr_locked() { choom/' pmutant no-choom 's/ choom -n 1000 -- "\$APR" "\$@"/ "$APR" "$@"/' pmutant unbounded 's/ -w "\$LOCK_WAIT"//' + # The cells module (scripts/lib/model_ladder_cells.py): each rule deleted in a copy, imported through + # MODEL_LADDER_CELLS_LIB, and the case that names the rule must go RED under the copy. + cmutant() { # cmutant