From 275a8129e660cb5be3e1fbff761e3efd933b49fe Mon Sep 17 00:00:00 2001 From: Noah Gift Date: Mon, 28 Sep 2026 22:26:25 +0200 Subject: [PATCH 1/4] =?UTF-8?q?fix(ladder):=20de-claim=20by=20exact=20arti?= =?UTF-8?q?fact=20(sha256=20+=20file),=20not=20arch=20alone=20=E2=80=94=20?= =?UTF-8?q?D2=20for=20#3715?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit cells.declaimed was keyed (host, arch). gx10's failing Qwen3.5-0.8B-UD-IQ2_XXS is qwen35, so an arch entry would withdraw the whole flagship family on that host. Operator ruling D2: claims keyed by exact artifact per BIN-001 identity. A de-claim now names either an arch or one artifact (sha256 + file), never both. The judge, the producer and the receipt writer all match a file entry by sha256 only. If the host holds a same-named file with different bytes, that file still owes every cell, and the judge FAILs with a message naming the stale entry. The contract withdraws gx10 IQ2_XXS a369165c… (0.70.1). All four verbs hit the 600 s timeout on GB10 in the H4 run. Every other gx10 Qwen3.5 file, and lambda, still owe their cells. Red/green: - The pre-change judge FAILs green-cells-declaimed-file. - check_model_ladder.sh --self-test: 156 cases, 0 bad. - 5 new judge mutants and 1 new producer mutant are killed. - On the real H4 inventory, only the 40 IQ2_XXS cells on gx10 are withdrawn. Refs #3715 Agent: aprender-89 Co-Authored-By: Claude Opus 5.5 --- contracts/model-capability-ladder-v1.yaml | 12 + scripts/check_ladder_cells_producer.sh | 17 +- scripts/check_model_ladder.sh | 19 +- .../certification.json | 11 + .../context-rungs.json | 38 + .../crux/gx10-cpu.json | 102 +++ .../crux/gx10-gpu.json | 102 +++ .../crux/lambda-cpu.json | 102 +++ .../crux/lambda-gpu.json | 102 +++ .../green-cells-declaimed-file/expected_rc | 1 + .../green-cells-declaimed-file/ladder.yaml | 52 ++ .../green-cells-declaimed-file/must_match | 1 + .../receipts/gx10.json | 592 ++++++++++++++++ .../receipts/lambda.json | 652 ++++++++++++++++++ .../green-cells-declaimed-file/version | 1 + .../certification.json | 11 + .../context-rungs.json | 38 + .../crux/gx10-cpu.json | 102 +++ .../crux/gx10-gpu.json | 102 +++ .../crux/lambda-cpu.json | 102 +++ .../crux/lambda-gpu.json | 102 +++ .../red-cells-declaimed-file-bare/expected_rc | 1 + .../red-cells-declaimed-file-bare/ladder.yaml | 53 ++ .../red-cells-declaimed-file-bare/must_match | 1 + .../receipts/gx10.json | 592 ++++++++++++++++ .../receipts/lambda.json | 652 ++++++++++++++++++ .../red-cells-declaimed-file-bare/version | 1 + .../certification.json | 11 + .../context-rungs.json | 38 + .../crux/gx10-cpu.json | 102 +++ .../crux/gx10-gpu.json | 102 +++ .../crux/lambda-cpu.json | 102 +++ .../crux/lambda-gpu.json | 102 +++ .../expected_rc | 1 + .../ladder.yaml | 52 ++ .../must_match | 1 + .../receipts/gx10.json | 609 ++++++++++++++++ .../receipts/lambda.json | 652 ++++++++++++++++++ .../red-cells-declaimed-file-narrow/version | 1 + .../certification.json | 11 + .../context-rungs.json | 38 + .../crux/gx10-cpu.json | 102 +++ .../crux/gx10-gpu.json | 102 +++ .../crux/lambda-cpu.json | 102 +++ .../crux/lambda-gpu.json | 102 +++ .../expected_rc | 1 + .../ladder.yaml | 52 ++ .../red-cells-declaimed-file-stale/must_match | 1 + .../receipts/gx10.json | 592 ++++++++++++++++ .../receipts/lambda.json | 652 ++++++++++++++++++ .../red-cells-declaimed-file-stale/version | 1 + scripts/lib/model_ladder_cells.py | 49 +- scripts/lib/model_ladder_cells_produce.py | 5 +- scripts/model_ladder.sh | 11 +- 54 files changed, 7130 insertions(+), 25 deletions(-) create mode 100644 scripts/lib/model_ladder_cases/green-cells-declaimed-file/certification.json create mode 100644 scripts/lib/model_ladder_cases/green-cells-declaimed-file/context-rungs.json create mode 100644 scripts/lib/model_ladder_cases/green-cells-declaimed-file/crux/gx10-cpu.json create mode 100644 scripts/lib/model_ladder_cases/green-cells-declaimed-file/crux/gx10-gpu.json create mode 100644 scripts/lib/model_ladder_cases/green-cells-declaimed-file/crux/lambda-cpu.json create mode 100644 scripts/lib/model_ladder_cases/green-cells-declaimed-file/crux/lambda-gpu.json create mode 100644 scripts/lib/model_ladder_cases/green-cells-declaimed-file/expected_rc create mode 100644 scripts/lib/model_ladder_cases/green-cells-declaimed-file/ladder.yaml create mode 100644 scripts/lib/model_ladder_cases/green-cells-declaimed-file/must_match create mode 100644 scripts/lib/model_ladder_cases/green-cells-declaimed-file/receipts/gx10.json create mode 100644 scripts/lib/model_ladder_cases/green-cells-declaimed-file/receipts/lambda.json create mode 100644 scripts/lib/model_ladder_cases/green-cells-declaimed-file/version create mode 100644 scripts/lib/model_ladder_cases/red-cells-declaimed-file-bare/certification.json create mode 100644 scripts/lib/model_ladder_cases/red-cells-declaimed-file-bare/context-rungs.json create mode 100644 scripts/lib/model_ladder_cases/red-cells-declaimed-file-bare/crux/gx10-cpu.json create mode 100644 scripts/lib/model_ladder_cases/red-cells-declaimed-file-bare/crux/gx10-gpu.json create mode 100644 scripts/lib/model_ladder_cases/red-cells-declaimed-file-bare/crux/lambda-cpu.json create mode 100644 scripts/lib/model_ladder_cases/red-cells-declaimed-file-bare/crux/lambda-gpu.json create mode 100644 scripts/lib/model_ladder_cases/red-cells-declaimed-file-bare/expected_rc create mode 100644 scripts/lib/model_ladder_cases/red-cells-declaimed-file-bare/ladder.yaml create mode 100644 scripts/lib/model_ladder_cases/red-cells-declaimed-file-bare/must_match create mode 100644 scripts/lib/model_ladder_cases/red-cells-declaimed-file-bare/receipts/gx10.json create mode 100644 scripts/lib/model_ladder_cases/red-cells-declaimed-file-bare/receipts/lambda.json create mode 100644 scripts/lib/model_ladder_cases/red-cells-declaimed-file-bare/version create mode 100644 scripts/lib/model_ladder_cases/red-cells-declaimed-file-narrow/certification.json create mode 100644 scripts/lib/model_ladder_cases/red-cells-declaimed-file-narrow/context-rungs.json create mode 100644 scripts/lib/model_ladder_cases/red-cells-declaimed-file-narrow/crux/gx10-cpu.json create mode 100644 scripts/lib/model_ladder_cases/red-cells-declaimed-file-narrow/crux/gx10-gpu.json create mode 100644 scripts/lib/model_ladder_cases/red-cells-declaimed-file-narrow/crux/lambda-cpu.json create mode 100644 scripts/lib/model_ladder_cases/red-cells-declaimed-file-narrow/crux/lambda-gpu.json create mode 100644 scripts/lib/model_ladder_cases/red-cells-declaimed-file-narrow/expected_rc create mode 100644 scripts/lib/model_ladder_cases/red-cells-declaimed-file-narrow/ladder.yaml create mode 100644 scripts/lib/model_ladder_cases/red-cells-declaimed-file-narrow/must_match create mode 100644 scripts/lib/model_ladder_cases/red-cells-declaimed-file-narrow/receipts/gx10.json create mode 100644 scripts/lib/model_ladder_cases/red-cells-declaimed-file-narrow/receipts/lambda.json create mode 100644 scripts/lib/model_ladder_cases/red-cells-declaimed-file-narrow/version create mode 100644 scripts/lib/model_ladder_cases/red-cells-declaimed-file-stale/certification.json create mode 100644 scripts/lib/model_ladder_cases/red-cells-declaimed-file-stale/context-rungs.json create mode 100644 scripts/lib/model_ladder_cases/red-cells-declaimed-file-stale/crux/gx10-cpu.json create mode 100644 scripts/lib/model_ladder_cases/red-cells-declaimed-file-stale/crux/gx10-gpu.json create mode 100644 scripts/lib/model_ladder_cases/red-cells-declaimed-file-stale/crux/lambda-cpu.json create mode 100644 scripts/lib/model_ladder_cases/red-cells-declaimed-file-stale/crux/lambda-gpu.json create mode 100644 scripts/lib/model_ladder_cases/red-cells-declaimed-file-stale/expected_rc create mode 100644 scripts/lib/model_ladder_cases/red-cells-declaimed-file-stale/ladder.yaml create mode 100644 scripts/lib/model_ladder_cases/red-cells-declaimed-file-stale/must_match create mode 100644 scripts/lib/model_ladder_cases/red-cells-declaimed-file-stale/receipts/gx10.json create mode 100644 scripts/lib/model_ladder_cases/red-cells-declaimed-file-stale/receipts/lambda.json create mode 100644 scripts/lib/model_ladder_cases/red-cells-declaimed-file-stale/version diff --git a/contracts/model-capability-ladder-v1.yaml b/contracts/model-capability-ladder-v1.yaml index 7e2e156e6a..70134430c2 100644 --- a/contracts/model-capability-ladder-v1.yaml +++ b/contracts/model-capability-ladder-v1.yaml @@ -246,6 +246,18 @@ ladder: GB10 dense models: prompt processing unbatched (11.4 tok/s); fixed in 0.70.1 (#4590). Correct output, prefill speed only: sm_12x prefills every dense arch one token at a time (select_prefill_path). Withdrawn, not waived: lambda still owes every Qwen2 cell, and gx10 still owes every Qwen3.5 (hybrid) cell. + # D2 (operator ruling 2026-09-28 20:10Z, #3715): ONE artifact, keyed by its bytes (BIN-001 identity), not + # its arch -- an arch entry for qwen35 would withdraw the flagship family, which passes on gx10 (44-286 s). + - sha256: a369165c8ec45a92d55cbad98b5377e2c32ca8dfc824c899d9d05b33b6b53e54 + file: Qwen3.5-0.8B-UD-IQ2_XXS.gguf + host: gx10 + issue: 3715 + until: "0.70.1" + why: >- + all 4 verbs time out at 600 s on GB10 (run/chat/serve/code at 20k think-on, rc 124; H4 run, + apr 0.70.0 8d021f61e, 3715-run/h4/gx10). Withdrawn by data, not waived: every other gx10 Qwen3.5 file + still owes every cell, lambda still owes this file's cells, and a re-quantised file under the same + name is a different artifact and owes them too. long_rungs_for: families: [qwen35] # every consumer's family: the full rung set, through declared representatives: # one file per other arch the fleet holds diff --git a/scripts/check_ladder_cells_producer.sh b/scripts/check_ladder_cells_producer.sh index 9546f16896..91a758cf84 100755 --- a/scripts/check_ladder_cells_producer.sh +++ b/scripts/check_ladder_cells_producer.sh @@ -138,10 +138,12 @@ PY3 # declaim -> a cells.declaimed arch x host is not measured on that host, and IS on every other (#4590): # hours of cells must not be spent on a claim the release does not make, nor may the entry leak to other hosts. -declaim() { - local lib=$1 d n_t n_u +# D2 (#3715): the entry may name ONE artifact by sha256 instead -- then only those bytes skip, and a +# same-named file with other bytes is measured (want_t 16), never withdrawn by its name. +declaim() { # declaim [entry key] [rows owed on t] [label] + local lib=$1 key=${2:-"arch: qwen2"} want_t=${3:-0} what=${4:-"a de-claimed arch"} d n_t n_u SEQ=$((SEQ + 1)); d="$WORK/dec.$SEQ"; mkdir -p "$d" || return 1 - sed -e 's/^ verbs:/ declaimed: [{arch: qwen2, host: t, issue: 4590, until: "0.70.1", why: fixture}]\n verbs:/' \ + sed -e "s/^ verbs:/ declaimed: [{$key, host: t, issue: 4590, until: \"0.70.1\", why: fixture}]\\n verbs:/" \ -e 's/hosts: \[{id: t, required: true}\]/hosts: [{id: t, required: true}, {id: u, required: true}]/' \ "$WORK/ladder.yaml" > "$d/ladder.yaml" FAKE_APR_MODE=good python3 "$lib/model_ladder_cells_produce.py" enrich --apr "$FAKE" \ @@ -154,10 +156,10 @@ declaim() { done n_t=$(python3 -c 'import json,sys; print(len(json.load(open(sys.argv[1]))))' "$d/cells-t.json") n_u=$(python3 -c 'import json,sys; print(len(json.load(open(sys.argv[1]))))' "$d/cells-u.json") - if [ "$n_t" = 0 ] && [ "$n_u" = 16 ] && grep -q 'DECLAIMED on t' "$d/log-t"; then - printf 'ok %s\n' "declaim: a de-claimed arch is not measured on its host (0 rows) and is on another (16 rows)"; return 0 + if [ "$n_t" = "$want_t" ] && [ "$n_u" = 16 ] && { [ "$want_t" != 0 ] || grep -q 'DECLAIMED on t' "$d/log-t"; }; then + printf 'ok %s\n' "declaim: $what gives $want_t cell rows on its host and 16 on another"; return 0 fi - printf 'FAIL %s (t=%s rows, u=%s rows)\n' "declaim: the de-claim was ignored on its host or leaked to another" "$n_t" "$n_u"; return 1 + printf 'FAIL %s (t=%s rows, want %s; u=%s rows)\n' "declaim: $what -- the de-claim was ignored, too wide, or leaked to another host" "$n_t" "$want_t" "$n_u"; return 1 } cases() { # cases -> 0 all as expected @@ -182,6 +184,8 @@ cases() { # cases -> 0 all as expected offmode "$lib" || r=1 selflock "$lib" || r=1 declaim "$lib" || r=1 + declaim "$lib" "sha256: \"$(printf '%064d' 0)\", file: m.gguf" 0 "a de-claimed artifact (sha256)" || r=1 + declaim "$lib" "sha256: \"$(printf '%064d' 1)\", file: m.gguf" 16 "a same-named file with other bytes" || r=1 return $r } @@ -213,6 +217,7 @@ mutant only-vacuity 's/if a.only and not matched:/if False:/' mutant rungs-silent 's/^ if rc:$/ if False:/' mutant serve-inline 's/deferred.append((len(rows) - 1, prompt, mode, budget, measured))/serve = serve or Serve(R, path, a.serve_ceiling); rows[-1] = measure_serve(serve, prompt, mode, budget, rows[-1], R.timeout)/' mutant declaim 's/ if d: # the judge owes nothing here/ if False: # the judge owes nothing here/' +mutant declaim-name 's/ d = J.declaim_of(dec, a.host, it)/ d = dec.get((a.host, J.FILE_KEY + it["file"])) or dec.get((a.host, it.get("arch")))/' mutant offmode-budget 's/target = min(tok, int(ctx) - budget - 1)/target = min(tok, int(ctx) - a.max_tokens_thinking - 1)/' [ "$bad" = 0 ] && { echo "check_ladder_cells_producer: all cases and mutants as expected"; exit 0; } diff --git a/scripts/check_model_ladder.sh b/scripts/check_model_ladder.sh index cba67f70fd..0fa7396202 100755 --- a/scripts/check_model_ladder.sh +++ b/scripts/check_model_ladder.sh @@ -303,14 +303,17 @@ if main_p and os.path.exists(main_p): mrh = {r["id"]: hostset(r, allh) for r in M.get("rungs", []) if r.get("required")} # #4590: the one exception is a (host, arch) the release DE-CLAIMS in cells.declaimed. It is how the # de-claim reaches released pv, which reads rung hosts and not declaimed; printed on every run. - dcl = {(d.get("host"), d.get("arch")): d for d in (L.get("cells") or {}).get("declaimed") or []} + # D2 (#3715): a file entry is keyed by the rung's exact artifact, (host, "sha256:"), never its name. + dcl = {(d.get("host"), "sha256:" + d["sha256"] if d.get("sha256") else d.get("arch")): d + for d in (L.get("cells") or {}).get("declaimed") or []} + def dkey(h, r): return (h, "sha256:" + str(r.get("sha256"))) if (h, "sha256:" + str(r.get("sha256"))) in dcl else (h, r.get("arch")) for r in rungs: if r.get("required") and r["id"] in mrh and not mrh[r["id"]] <= hostset(r, allh): gone = mrh[r["id"]] - hostset(r, allh) - for h in sorted(g for g in gone if (g, r.get("arch")) in dcl): # a de-claimed host, not a drop - d = dcl[(h, r.get("arch"))] + for h in sorted(g for g in gone if dkey(g, r) in dcl): # a de-claimed host, not a drop + d = dcl[dkey(h, r)] print(f"DECLAIMED rung {r['id']} not claimed on {h} -- #{d.get('issue')}, restored in {d.get('until')}") - gone = {g for g in gone if (g, r.get("arch")) not in dcl} + gone = {g for g in gone if dkey(g, r) not in dcl} if gone: print(f"FAIL hosts DROPPED on {r['id']} vs origin/main: {sorted(gone)} — a rung may gain hosts, never lose one"); rc = 1 mc, hc = M.get("cells") or {}, L.get("cells") or {} @@ -890,7 +893,13 @@ if [ "$SELF_TEST" = 1 ]; then cmutant rungs-floor red-cells-rung-dropped-vs-main 's/ if gone:/ if False:/' cmutant declaim-skip green-cells-declaimed 's/ if d: # printed on every run/ if False: # printed on every run/' cmutant declaim-bare red-cells-declaimed-bare 's/ if bad: # a bare de-claim/ if False: # a bare de-claim/' - mutant declaim-host declaimed-host-on-rung 's/ gone = {g for g in gone if (g, r.get("arch")) not in dcl}/ gone = set(gone)/' + # D2 (#3715): a file de-claim is keyed by the artifact's sha256 -- never its name, never wider than the file. + cmutant declaim-file green-cells-declaimed-file 's/ got\[(d\["host"\], SHA_KEY + d\["sha256"\])\] = d/ pass/' + cmutant declaim-by-name red-cells-declaimed-file-stale 's/return dec.get((host, SHA_KEY + str(item.get("sha256") or "")))/return dec.get((host, FILE_KEY + str(item.get("file"))))/' + cmutant declaim-widen red-cells-declaimed-file-narrow 's/return dec.get((host, SHA_KEY + str(item.get("sha256") or "")))/return next((d for (h, k), d in dec.items() if h == host and k.startswith(SHA_KEY)), None)/' + cmutant declaim-stale red-cells-declaimed-file-stale 's/ if stale: # the entry/ if False: # the entry/' + cmutant declaim-sha-hex red-cells-declaimed-file-bare 's/if by_file and d.get("sha256") and not re.fullmatch/if False and not re.fullmatch/' + mutant declaim-host declaimed-host-on-rung 's/ gone = {g for g in gone if dkey(g, r) not in dcl}/ gone = set(gone)/' # #3957 F4/F8: the CRUX join (scripts/lib/model_ladder_crux.py), each rule deleted in a copy # imported through MODEL_LADDER_CRUX_LIB; the case that names the rule must go RED. xmutant() { # xmutant