diff --git a/CHANGELOG.md b/CHANGELOG.md index 5bd5ff011b..4dcf1a665c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -29,6 +29,33 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 host still owes every Qwen3 cell. Planned for 0.70.1: native sm_121 SASS, then a re-test of batched prefill on GB10. +## [0.70.2] - 2026-10-07 + +0.70.2 ships under a recorded operator emergency scope, `crux-smoke`, as 0.70.1 did: "0.70.2 ships on +CRUX smoke on lambda and gx10 GPU. Everything bigger is nightly." The record is the 0.70.2 entry under +`emergency_scopes` in `contracts/model-capability-ladder-v1.yaml`. The prompt certification is carried +from 0.70.1 unchanged (same models, prompts and oracle engines). The full notes, with the reason for +carrying it, are in `evidence/release/0.70.2/RELEASE-NOTES-0.70.2.md`. + +### Fixed +- `apr serve`: tools in a chat request reach the chat template, and Qwen3.5 XML tool calls are parsed (#4650). +- `apr serve`: KV caches no longer commit every byte when they are built (#4769). +- `apr train` / `apr finetune` on CPU: an intermittent crash (SIGSEGV) in the AVX2 matrix-multiply kernel + is fixed (#4891). + +### Release +- The PP-26 witness marker is re-measured on this release's base, so its release-phase check judges fresh + evidence (#4888). +- The coverage gate reads the nightly's sha-keyed receipt and is checked before the tag (#4734); merge, tag + and publish requirements are one list in code (#4688); no public GitHub release before its assets, + clean-room and preflight (#4690). + +### Known issues (moved to 0.71.0) +- Six models still fail and ship as known failures, each tracked on its ticket: #4661 and #4662 (cuda serve + /api/chat gibberish), #4663 (no measured thinking budget for an IQ4_XS quant), #4664 (an IQ2_XXS GPU forward + falls back to CPU), #4665 (no CUDA forward for qwen35moe), #4666 (think block unclosed within 2048 tokens). +- One unattended autopilot pass: the autopilot's readiness step and tag cut do not read a recorded scope. + ## [0.69.3] - 2026-09-24 0.69.3 is an emergency early release, authorized by the operator: "we need near parity apr serve diff --git a/Cargo.lock b/Cargo.lock index c6f9c3eb03..0a9166d4fa 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -248,7 +248,7 @@ dependencies = [ [[package]] name = "apr-cli" -version = "0.70.1" +version = "0.70.2" dependencies = [ "anyhow", "aprender-build-sha", @@ -325,7 +325,7 @@ dependencies = [ [[package]] name = "apr-format" -version = "0.70.1" +version = "0.70.2" dependencies = [ "bincode", "cargo_metadata", @@ -343,7 +343,7 @@ dependencies = [ [[package]] name = "aprender" -version = "0.70.1" +version = "0.70.2" dependencies = [ "apr-cli", "aprender-core", @@ -351,7 +351,7 @@ dependencies = [ [[package]] name = "aprender-bench-compute" -version = "0.70.1" +version = "0.70.2" dependencies = [ "aprender-core", "criterion 0.7.0", @@ -360,7 +360,7 @@ dependencies = [ [[package]] name = "aprender-bench-tokenizer" -version = "0.70.1" +version = "0.70.2" dependencies = [ "aprender-core", "criterion 0.7.0", @@ -369,14 +369,14 @@ dependencies = [ [[package]] name = "aprender-build-sha" -version = "0.70.1" +version = "0.70.2" dependencies = [ "tempfile", ] [[package]] name = "aprender-cbtop" -version = "0.70.1" +version = "0.70.2" dependencies = [ "anyhow", "aprender-build-sha", @@ -402,7 +402,7 @@ dependencies = [ [[package]] name = "aprender-cgp" -version = "0.70.1" +version = "0.70.2" dependencies = [ "anyhow", "aprender-build-sha", @@ -423,7 +423,7 @@ dependencies = [ [[package]] name = "aprender-common" -version = "0.70.1" +version = "0.70.2" dependencies = [ "clap", "lz4_flex 0.11.6", @@ -432,7 +432,7 @@ dependencies = [ [[package]] name = "aprender-compute" -version = "0.70.1" +version = "0.70.2" dependencies = [ "anyhow", "aprender-contracts", @@ -482,7 +482,7 @@ dependencies = [ [[package]] name = "aprender-compute-xtask" -version = "0.70.1" +version = "0.70.2" dependencies = [ "anyhow", "aprender-build-sha", @@ -497,7 +497,7 @@ dependencies = [ [[package]] name = "aprender-contracts" -version = "0.70.1" +version = "0.70.2" dependencies = [ "aprender-contracts-macros", "blake3", @@ -517,7 +517,7 @@ dependencies = [ [[package]] name = "aprender-contracts-cli" -version = "0.70.1" +version = "0.70.2" dependencies = [ "aprender-build-sha", "aprender-common", @@ -533,7 +533,7 @@ dependencies = [ [[package]] name = "aprender-contracts-macros" -version = "0.70.1" +version = "0.70.2" dependencies = [ "proc-macro2", "quote", @@ -543,7 +543,7 @@ dependencies = [ [[package]] name = "aprender-contrastive-data" -version = "0.70.1" +version = "0.70.2" dependencies = [ "aprender-contracts-macros", "aprender-rand", @@ -558,7 +558,7 @@ dependencies = [ [[package]] name = "aprender-core" -version = "0.70.1" +version = "0.70.2" dependencies = [ "aes-gcm", "alsa", @@ -612,7 +612,7 @@ dependencies = [ [[package]] name = "aprender-cuda-edge" -version = "0.70.1" +version = "0.70.2" dependencies = [ "aprender-gpu", "proptest", @@ -623,7 +623,7 @@ dependencies = [ [[package]] name = "aprender-cupti" -version = "0.70.1" +version = "0.70.2" dependencies = [ "bindgen", "bitflags 2.13.0", @@ -633,7 +633,7 @@ dependencies = [ [[package]] name = "aprender-data" -version = "0.70.1" +version = "0.70.2" dependencies = [ "aes-gcm", "aprender-build-sha", @@ -686,7 +686,7 @@ dependencies = [ [[package]] name = "aprender-db" -version = "0.70.1" +version = "0.70.2" dependencies = [ "anyhow", "aprender-build-sha", @@ -733,7 +733,7 @@ dependencies = [ [[package]] name = "aprender-distribute" -version = "0.70.1" +version = "0.70.2" dependencies = [ "aprender-compute", "aprender-db", @@ -763,7 +763,7 @@ dependencies = [ [[package]] name = "aprender-explain" -version = "0.70.1" +version = "0.70.2" dependencies = [ "aprender-build-sha", "aprender-gpu", @@ -782,7 +782,7 @@ dependencies = [ [[package]] name = "aprender-fft" -version = "0.70.1" +version = "0.70.2" dependencies = [ "criterion 0.7.0", "proptest", @@ -791,7 +791,7 @@ dependencies = [ [[package]] name = "aprender-gemm-codegen" -version = "0.70.1" +version = "0.70.2" dependencies = [ "proc-macro2", "quote", @@ -800,7 +800,7 @@ dependencies = [ [[package]] name = "aprender-gpu" -version = "0.70.1" +version = "0.70.2" dependencies = [ "aprender-common", "aprender-simulate", @@ -818,7 +818,7 @@ dependencies = [ [[package]] name = "aprender-graph" -version = "0.70.1" +version = "0.70.2" dependencies = [ "anyhow", "aprender-compute", @@ -839,7 +839,7 @@ dependencies = [ [[package]] name = "aprender-image" -version = "0.70.1" +version = "0.70.2" dependencies = [ "criterion 0.7.0", "proptest", @@ -848,7 +848,7 @@ dependencies = [ [[package]] name = "aprender-mcp" -version = "0.70.1" +version = "0.70.2" dependencies = [ "inventory", "jsonschema", @@ -861,7 +861,7 @@ dependencies = [ [[package]] name = "aprender-monte-carlo" -version = "0.70.1" +version = "0.70.2" dependencies = [ "aprender", "assert_cmd", @@ -880,7 +880,7 @@ dependencies = [ [[package]] name = "aprender-orchestrate" -version = "0.70.1" +version = "0.70.2" dependencies = [ "anyhow", "aprender-build-sha", @@ -951,7 +951,7 @@ dependencies = [ [[package]] name = "aprender-present-cli" -version = "0.70.1" +version = "0.70.2" dependencies = [ "aprender-build-sha", "aprender-contracts", @@ -966,7 +966,7 @@ dependencies = [ [[package]] name = "aprender-present-core" -version = "0.70.1" +version = "0.70.2" dependencies = [ "aprender-contracts", "aprender-contracts-macros", @@ -979,7 +979,7 @@ dependencies = [ [[package]] name = "aprender-present-layout" -version = "0.70.1" +version = "0.70.2" dependencies = [ "aprender-present-core", "criterion 0.7.0", @@ -990,7 +990,7 @@ dependencies = [ [[package]] name = "aprender-present-lib" -version = "0.70.1" +version = "0.70.2" dependencies = [ "aprender-contracts", "aprender-present-core", @@ -1019,7 +1019,7 @@ dependencies = [ [[package]] name = "aprender-present-terminal" -version = "0.70.1" +version = "0.70.2" dependencies = [ "aprender-build-sha", "aprender-contracts", @@ -1041,7 +1041,7 @@ dependencies = [ [[package]] name = "aprender-present-test" -version = "0.70.1" +version = "0.70.2" dependencies = [ "aprender-present-core", "aprender-present-test-macros", @@ -1052,7 +1052,7 @@ dependencies = [ [[package]] name = "aprender-present-test-macros" -version = "0.70.1" +version = "0.70.2" dependencies = [ "proc-macro2", "quote", @@ -1061,7 +1061,7 @@ dependencies = [ [[package]] name = "aprender-present-widgets" -version = "0.70.1" +version = "0.70.2" dependencies = [ "aprender-present-core", "aprender-present-yaml", @@ -1073,7 +1073,7 @@ dependencies = [ [[package]] name = "aprender-present-yaml" -version = "0.70.1" +version = "0.70.2" dependencies = [ "aprender-present-core", "proptest", @@ -1083,7 +1083,7 @@ dependencies = [ [[package]] name = "aprender-profile" -version = "0.70.1" +version = "0.70.2" dependencies = [ "addr2line 0.25.1", "anyhow", @@ -1136,7 +1136,7 @@ dependencies = [ [[package]] name = "aprender-profile-core" -version = "0.70.1" +version = "0.70.2" dependencies = [ "hex", "serde", @@ -1146,7 +1146,7 @@ dependencies = [ [[package]] name = "aprender-ptx-debug" -version = "0.70.1" +version = "0.70.2" dependencies = [ "aprender-build-sha", "clap", @@ -1156,7 +1156,7 @@ dependencies = [ [[package]] name = "aprender-qa-certify" -version = "0.70.1" +version = "0.70.2" dependencies = [ "aprender-build-sha", "chrono", @@ -1168,7 +1168,7 @@ dependencies = [ [[package]] name = "aprender-qa-cli" -version = "0.70.1" +version = "0.70.2" dependencies = [ "aprender-build-sha", "aprender-qa-certify", @@ -1188,7 +1188,7 @@ dependencies = [ [[package]] name = "aprender-qa-gen" -version = "0.70.1" +version = "0.70.2" dependencies = [ "anyhow", "chrono", @@ -1205,7 +1205,7 @@ dependencies = [ [[package]] name = "aprender-qa-report" -version = "0.70.1" +version = "0.70.2" dependencies = [ "anyhow", "aprender-qa-gen", @@ -1223,7 +1223,7 @@ dependencies = [ [[package]] name = "aprender-qa-runner" -version = "0.70.1" +version = "0.70.2" dependencies = [ "anyhow", "aprender-qa-gen", @@ -1248,7 +1248,7 @@ dependencies = [ [[package]] name = "aprender-quant" -version = "0.70.1" +version = "0.70.2" dependencies = [ "half", "serde_json", @@ -1256,7 +1256,7 @@ dependencies = [ [[package]] name = "aprender-rag" -version = "0.70.1" +version = "0.70.2" dependencies = [ "aprender-common", "aprender-compute", @@ -1285,7 +1285,7 @@ dependencies = [ [[package]] name = "aprender-rag-cli" -version = "0.70.1" +version = "0.70.2" dependencies = [ "anyhow", "aprender-build-sha", @@ -1305,7 +1305,7 @@ dependencies = [ [[package]] name = "aprender-rand" -version = "0.70.1" +version = "0.70.2" dependencies = [ "criterion 0.7.0", "proptest", @@ -1314,7 +1314,7 @@ dependencies = [ [[package]] name = "aprender-registry" -version = "0.70.1" +version = "0.70.2" dependencies = [ "anyhow", "aprender-core", @@ -1344,7 +1344,7 @@ dependencies = [ [[package]] name = "aprender-review-experiment" -version = "0.70.1" +version = "0.70.2" dependencies = [ "serde", "serde_json", @@ -1354,7 +1354,7 @@ dependencies = [ [[package]] name = "aprender-serve" -version = "0.70.1" +version = "0.70.2" dependencies = [ "anyhow", "approx", @@ -1425,7 +1425,7 @@ dependencies = [ [[package]] name = "aprender-shell" -version = "0.70.1" +version = "0.70.2" dependencies = [ "aprender-core", "clap", @@ -1440,7 +1440,7 @@ dependencies = [ [[package]] name = "aprender-simulate" -version = "0.70.1" +version = "0.70.2" dependencies = [ "aprender-build-sha", "aprender-contracts", @@ -1482,7 +1482,7 @@ dependencies = [ [[package]] name = "aprender-solve" -version = "0.70.1" +version = "0.70.2" dependencies = [ "criterion 0.7.0", "half", @@ -1492,7 +1492,7 @@ dependencies = [ [[package]] name = "aprender-sparse" -version = "0.70.1" +version = "0.70.2" dependencies = [ "criterion 0.7.0", "proptest", @@ -1501,7 +1501,7 @@ dependencies = [ [[package]] name = "aprender-tensor" -version = "0.70.1" +version = "0.70.2" dependencies = [ "criterion 0.7.0", "proptest", @@ -1510,7 +1510,7 @@ dependencies = [ [[package]] name = "aprender-test-cli" -version = "0.70.1" +version = "0.70.2" dependencies = [ "aprender-build-sha", "aprender-test-lib", @@ -1539,7 +1539,7 @@ dependencies = [ [[package]] name = "aprender-test-derive" -version = "0.70.1" +version = "0.70.2" dependencies = [ "proc-macro2", "quote", @@ -1549,7 +1549,7 @@ dependencies = [ [[package]] name = "aprender-test-js-gen" -version = "0.70.1" +version = "0.70.2" dependencies = [ "blake3", "chrono", @@ -1565,7 +1565,7 @@ dependencies = [ [[package]] name = "aprender-test-lib" -version = "0.70.1" +version = "0.70.2" dependencies = [ "aprender-compute", "aprender-present-core", @@ -1611,7 +1611,7 @@ dependencies = [ [[package]] name = "aprender-test-showcase" -version = "0.70.1" +version = "0.70.2" dependencies = [ "aprender-present-terminal", "aprender-test-lib", @@ -1628,7 +1628,7 @@ dependencies = [ [[package]] name = "aprender-train" -version = "0.70.1" +version = "0.70.2" dependencies = [ "approx", "aprender-common", @@ -1697,7 +1697,7 @@ dependencies = [ [[package]] name = "aprender-train-bench" -version = "0.70.1" +version = "0.70.2" dependencies = [ "aprender-build-sha", "aprender-train", @@ -1715,7 +1715,7 @@ dependencies = [ [[package]] name = "aprender-train-common" -version = "0.70.1" +version = "0.70.2" dependencies = [ "aprender-viz", "clap", @@ -1727,7 +1727,7 @@ dependencies = [ [[package]] name = "aprender-train-distill" -version = "0.70.1" +version = "0.70.2" dependencies = [ "aprender-build-sha", "aprender-train", @@ -1749,7 +1749,7 @@ dependencies = [ [[package]] name = "aprender-train-inspect" -version = "0.70.1" +version = "0.70.2" dependencies = [ "aprender-build-sha", "aprender-train", @@ -1766,7 +1766,7 @@ dependencies = [ [[package]] name = "aprender-train-lora" -version = "0.70.1" +version = "0.70.2" dependencies = [ "aprender-build-sha", "aprender-train", @@ -1783,7 +1783,7 @@ dependencies = [ [[package]] name = "aprender-train-shell" -version = "0.70.1" +version = "0.70.2" dependencies = [ "aprender-build-sha", "aprender-train", @@ -1800,7 +1800,7 @@ dependencies = [ [[package]] name = "aprender-train-wasm" -version = "0.70.1" +version = "0.70.2" dependencies = [ "proptest", "serde", @@ -1811,7 +1811,7 @@ dependencies = [ [[package]] name = "aprender-tsp" -version = "0.70.1" +version = "0.70.2" dependencies = [ "aprender", "assert_cmd", @@ -1826,7 +1826,7 @@ dependencies = [ [[package]] name = "aprender-update" -version = "0.70.1" +version = "0.70.2" dependencies = [ "flate2", "semver", @@ -1840,7 +1840,7 @@ dependencies = [ [[package]] name = "aprender-verify" -version = "0.70.1" +version = "0.70.2" dependencies = [ "chrono", "criterion 0.5.1", @@ -1852,7 +1852,7 @@ dependencies = [ [[package]] name = "aprender-verify-ml" -version = "0.70.1" +version = "0.70.2" dependencies = [ "aprender-build-sha", "aprender-compute", @@ -1883,7 +1883,7 @@ dependencies = [ [[package]] name = "aprender-viz" -version = "0.70.1" +version = "0.70.2" dependencies = [ "approx", "aprender-common", @@ -1912,7 +1912,7 @@ dependencies = [ [[package]] name = "aprender-zram" -version = "0.70.1" +version = "0.70.2" dependencies = [ "aprender-zram-adaptive", "aprender-zram-core", @@ -1922,7 +1922,7 @@ dependencies = [ [[package]] name = "aprender-zram-adaptive" -version = "0.70.1" +version = "0.70.2" dependencies = [ "aprender-zram-core", "proptest", @@ -1931,7 +1931,7 @@ dependencies = [ [[package]] name = "aprender-zram-cli" -version = "0.70.1" +version = "0.70.2" dependencies = [ "aprender-build-sha", "aprender-zram-adaptive", @@ -1946,7 +1946,7 @@ dependencies = [ [[package]] name = "aprender-zram-core" -version = "0.70.1" +version = "0.70.2" dependencies = [ "aprender-gpu", "criterion 0.7.0", @@ -1958,7 +1958,7 @@ dependencies = [ [[package]] name = "aprender-zram-generator" -version = "0.70.1" +version = "0.70.2" dependencies = [ "aprender-build-sha", "aprender-zram-core", diff --git a/Cargo.toml b/Cargo.toml index 897eac7e0b..6bf8012cef 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -157,7 +157,7 @@ exclude = [ resolver = "2" [workspace.package] -version = "0.70.1" +version = "0.70.2" edition = "2021" license = "MIT" repository = "https://github.com/paiml/aprender" @@ -238,7 +238,7 @@ tokenizers = { version = "0.23.1", default-features = false, features = ["fancy- # aprender-rand (lib name `trueno_rand`) is a workspace MEMBER but had no # [workspace.dependencies] entry, so every consumer had to re-declare the path. # It is the counter-based Philox RNG behind the SetFit encoder's dropout masks. -aprender-rand = { path = "crates/aprender-rand", version = "0.70.1" } +aprender-rand = { path = "crates/aprender-rand", version = "0.70.2" } # Chat templates minijinja = { version = "2.14", features = ["loader", "serde"] } @@ -248,9 +248,9 @@ minijinja = { version = "2.14", features = ["loader", "serde"] } # `crates/aprender-contracts` (lib name `provable_contracts`). Declaring it from # crates.io compiled a SECOND copy of the contract engine alongside the in-tree # one — exactly the duplicate-crate + publish-cycle the consolidation removed. -provable-contracts = { path = "crates/aprender-contracts", version = "0.70.1", package = "aprender-contracts" } -provable-contracts-macros = { path = "crates/aprender-contracts-macros", version = "0.70.1", package = "aprender-contracts-macros" } -aprender-contracts-macros = { path = "crates/aprender-contracts-macros", version = "0.70.1" } +provable-contracts = { path = "crates/aprender-contracts", version = "0.70.2", package = "aprender-contracts" } +provable-contracts-macros = { path = "crates/aprender-contracts-macros", version = "0.70.2", package = "aprender-contracts-macros" } +aprender-contracts-macros = { path = "crates/aprender-contracts-macros", version = "0.70.2" } # YAML serde_yaml = "0.9" @@ -262,69 +262,69 @@ hf-hub = { version = "0.4", default-features = false, features = ["ureq"] } hf-xet = "1.5.1" # Workspace-internal crates (paths within the monorepo) -aprender-compute = { path = "crates/aprender-compute", version = "0.70.1" } -aprender-gpu = { path = "crates/aprender-gpu", version = "0.70.1" } -aprender-quant = { path = "crates/aprender-quant", version = "0.70.1" } -aprender-serve = { path = "crates/aprender-serve", version = "0.70.1" } -realizar = { path = "crates/aprender-serve", version = "0.70.1", package = "aprender-serve" } -alimentar = { path = "crates/aprender-data", version = "0.70.1", package = "aprender-data" } -pacha = { path = "crates/aprender-registry", version = "0.70.1", package = "aprender-registry" } +aprender-compute = { path = "crates/aprender-compute", version = "0.70.2" } +aprender-gpu = { path = "crates/aprender-gpu", version = "0.70.2" } +aprender-quant = { path = "crates/aprender-quant", version = "0.70.2" } +aprender-serve = { path = "crates/aprender-serve", version = "0.70.2" } +realizar = { path = "crates/aprender-serve", version = "0.70.2", package = "aprender-serve" } +alimentar = { path = "crates/aprender-data", version = "0.70.2", package = "aprender-data" } +pacha = { path = "crates/aprender-registry", version = "0.70.2", package = "aprender-registry" } # APR-MONO self-containment: legacy sibling lib-name aliases -> in-monorepo crates (2026-06-12) -trueno-quant = { path = "crates/aprender-quant", version = "0.70.1", package = "aprender-quant" } -trueno-db = { path = "crates/aprender-db", version = "0.70.1", package = "aprender-db" } -trueno-viz = { path = "crates/aprender-viz", version = "0.70.1", package = "aprender-viz" } -trueno-rag = { path = "crates/aprender-rag", version = "0.70.1", package = "aprender-rag" } -trueno-cuda-edge = { path = "crates/aprender-cuda-edge", version = "0.70.1", package = "aprender-cuda-edge" } -trueno-sparse = { path = "crates/aprender-sparse", version = "0.70.1", package = "aprender-sparse" } -trueno-solve = { path = "crates/aprender-solve", version = "0.70.1", package = "aprender-solve" } -renacer-core = { path = "crates/aprender-profile-core", version = "0.70.1", package = "aprender-profile-core" } -repartir = { path = "crates/aprender-distribute", version = "0.70.1", package = "aprender-distribute" } -simular = { path = "crates/aprender-simulate", version = "0.70.1", package = "aprender-simulate" } -probar = { path = "crates/aprender-test-lib", version = "0.70.1", package = "aprender-test-lib" } +trueno-quant = { path = "crates/aprender-quant", version = "0.70.2", package = "aprender-quant" } +trueno-db = { path = "crates/aprender-db", version = "0.70.2", package = "aprender-db" } +trueno-viz = { path = "crates/aprender-viz", version = "0.70.2", package = "aprender-viz" } +trueno-rag = { path = "crates/aprender-rag", version = "0.70.2", package = "aprender-rag" } +trueno-cuda-edge = { path = "crates/aprender-cuda-edge", version = "0.70.2", package = "aprender-cuda-edge" } +trueno-sparse = { path = "crates/aprender-sparse", version = "0.70.2", package = "aprender-sparse" } +trueno-solve = { path = "crates/aprender-solve", version = "0.70.2", package = "aprender-solve" } +renacer-core = { path = "crates/aprender-profile-core", version = "0.70.2", package = "aprender-profile-core" } +repartir = { path = "crates/aprender-distribute", version = "0.70.2", package = "aprender-distribute" } +simular = { path = "crates/aprender-simulate", version = "0.70.2", package = "aprender-simulate" } +probar = { path = "crates/aprender-test-lib", version = "0.70.2", package = "aprender-test-lib" } # English key for the same crate, named after the command it serves (`apr test`). # The dep KEY, not the package name, is what becomes the extern crate name when # the two differ — the lib target is still `jugar_probar`, so keying it by its # package name would have imported Spanish anyway. Renaming that lib target is # 196 call sites in 17 crates and gets its own PR; this key is how new code # reads English today. -apr-test = { path = "crates/aprender-test-lib", version = "0.70.1", package = "aprender-test-lib", default-features = false, features = ["llm"] } +apr-test = { path = "crates/aprender-test-lib", version = "0.70.2", package = "aprender-test-lib", default-features = false, features = ["llm"] } # `jugar-probar` is the crates.io/lib name of the same in-tree crate as `probar` # above. Both aliases resolve to ONE package; consumers that spelled it # `jugar-probar = "0.4"|"0.5"|"1.0"` were building three extra copies. -jugar-probar = { path = "crates/aprender-test-lib", version = "0.70.1", package = "aprender-test-lib" } +jugar-probar = { path = "crates/aprender-test-lib", version = "0.70.2", package = "aprender-test-lib" } # NOTE: the matching `jugar-probar-derive` alias already exists further down this # same table (it points at crates/aprender-test-derive). crates/aprender-test-lib # was pinning `jugar-probar-derive = "=1.0.3"` from crates.io regardless — the # alias was sitting there unused. -aprender-train = { path = "crates/aprender-train", version = "0.70.1" } -entrenar = { path = "crates/aprender-train", version = "0.70.1", package = "aprender-train" } -aprender-train-common = { path = "crates/aprender-train-common", version = "0.70.1" } -aprender-train-distill = { path = "crates/aprender-train-distill", version = "0.70.1" } -aprender-orchestrate = { path = "crates/aprender-orchestrate", version = "0.70.1" } -aprender-contracts = { path = "crates/aprender-contracts", version = "0.70.1" } -aprender-build-sha = { path = "crates/aprender-build-sha", version = "0.70.1" } -aprender-profile = { path = "crates/aprender-profile", version = "0.70.1" } -aprender-profile-core = { path = "crates/aprender-profile-core", version = "0.70.1" } -aprender-zram-core = { path = "crates/aprender-zram-core", version = "0.70.1" } -aprender-zram-adaptive = { path = "crates/aprender-zram-adaptive", version = "0.70.1" } -aprender-zram-cli = { path = "crates/aprender-zram-cli", version = "0.70.1" } -aprender-present-core = { path = "crates/aprender-present-core", version = "0.70.1" } -aprender-present-terminal = { path = "crates/aprender-present-terminal", version = "0.70.1" } -aprender-present-widgets = { path = "crates/aprender-present-widgets", version = "0.70.1" } -aprender-present-layout = { path = "crates/aprender-present-layout", version = "0.70.1" } -aprender-present-yaml = { path = "crates/aprender-present-yaml", version = "0.70.1" } -aprender-present-test = { path = "crates/aprender-present-test", version = "0.70.1" } -aprender-db = { path = "crates/aprender-db", version = "0.70.1" } -aprender-graph = { path = "crates/aprender-graph", version = "0.70.1" } -aprender-rag = { path = "crates/aprender-rag", version = "0.70.1" } -aprender-rag-cli = { path = "crates/aprender-rag-cli", version = "0.70.1" } -aprender-qa-cli = { path = "crates/aprender-qa-cli", version = "0.70.1" } -aprender-cgp = { path = "crates/aprender-cgp", version = "0.70.1" } -aprender-contrastive-data = { path = "crates/aprender-contrastive-data", version = "0.70.1" } -aprender-contracts-cli = { path = "crates/aprender-contracts-cli", version = "0.70.1" } -aprender-update = { path = "crates/aprender-update", version = "0.70.1" } -aprender-data = { path = "crates/aprender-data", version = "0.70.1" } -aprender-ptx-debug = { path = "crates/aprender-ptx-debug", version = "0.70.1" } +aprender-train = { path = "crates/aprender-train", version = "0.70.2" } +entrenar = { path = "crates/aprender-train", version = "0.70.2", package = "aprender-train" } +aprender-train-common = { path = "crates/aprender-train-common", version = "0.70.2" } +aprender-train-distill = { path = "crates/aprender-train-distill", version = "0.70.2" } +aprender-orchestrate = { path = "crates/aprender-orchestrate", version = "0.70.2" } +aprender-contracts = { path = "crates/aprender-contracts", version = "0.70.2" } +aprender-build-sha = { path = "crates/aprender-build-sha", version = "0.70.2" } +aprender-profile = { path = "crates/aprender-profile", version = "0.70.2" } +aprender-profile-core = { path = "crates/aprender-profile-core", version = "0.70.2" } +aprender-zram-core = { path = "crates/aprender-zram-core", version = "0.70.2" } +aprender-zram-adaptive = { path = "crates/aprender-zram-adaptive", version = "0.70.2" } +aprender-zram-cli = { path = "crates/aprender-zram-cli", version = "0.70.2" } +aprender-present-core = { path = "crates/aprender-present-core", version = "0.70.2" } +aprender-present-terminal = { path = "crates/aprender-present-terminal", version = "0.70.2" } +aprender-present-widgets = { path = "crates/aprender-present-widgets", version = "0.70.2" } +aprender-present-layout = { path = "crates/aprender-present-layout", version = "0.70.2" } +aprender-present-yaml = { path = "crates/aprender-present-yaml", version = "0.70.2" } +aprender-present-test = { path = "crates/aprender-present-test", version = "0.70.2" } +aprender-db = { path = "crates/aprender-db", version = "0.70.2" } +aprender-graph = { path = "crates/aprender-graph", version = "0.70.2" } +aprender-rag = { path = "crates/aprender-rag", version = "0.70.2" } +aprender-rag-cli = { path = "crates/aprender-rag-cli", version = "0.70.2" } +aprender-qa-cli = { path = "crates/aprender-qa-cli", version = "0.70.2" } +aprender-cgp = { path = "crates/aprender-cgp", version = "0.70.2" } +aprender-contrastive-data = { path = "crates/aprender-contrastive-data", version = "0.70.2" } +aprender-contracts-cli = { path = "crates/aprender-contracts-cli", version = "0.70.2" } +aprender-update = { path = "crates/aprender-update", version = "0.70.2" } +aprender-data = { path = "crates/aprender-data", version = "0.70.2" } +aprender-ptx-debug = { path = "crates/aprender-ptx-debug", version = "0.70.2" } # Additional external deps needed by sub-crates serde_yaml_ng = "0.10" @@ -383,25 +383,25 @@ wasmtime = "48.0.3" bollard = "0.17" # Old crate name aliases (workspace = true refs in migrated sub-crates) -trueno-zram-core = { path = "crates/aprender-zram-core", version = "0.70.1", package = "aprender-zram-core" } -trueno-zram-adaptive = { path = "crates/aprender-zram-adaptive", version = "0.70.1", package = "aprender-zram-adaptive" } -trueno = { path = "crates/aprender-compute", version = "0.70.1", package = "aprender-compute" } -presentar-core = { path = "crates/aprender-present-core", version = "0.70.1", package = "aprender-present-core" } -presentar-terminal = { path = "crates/aprender-present-terminal", version = "0.70.1", package = "aprender-present-terminal" } -presentar-widgets = { path = "crates/aprender-present-widgets", version = "0.70.1", package = "aprender-present-widgets" } -presentar-layout = { path = "crates/aprender-present-layout", version = "0.70.1", package = "aprender-present-layout" } -presentar-yaml = { path = "crates/aprender-present-yaml", version = "0.70.1", package = "aprender-present-yaml" } -presentar-test = { path = "crates/aprender-present-test", version = "0.70.1", package = "aprender-present-test" } -presentar-test-macros = { path = "crates/aprender-present-test-macros", version = "0.70.1", package = "aprender-present-test-macros" } -jugar-probar-derive = { path = "crates/aprender-test-derive", version = "0.70.1", package = "aprender-test-derive" } -batuta-common = { path = "crates/aprender-common", version = "0.70.1", package = "aprender-common" } +trueno-zram-core = { path = "crates/aprender-zram-core", version = "0.70.2", package = "aprender-zram-core" } +trueno-zram-adaptive = { path = "crates/aprender-zram-adaptive", version = "0.70.2", package = "aprender-zram-adaptive" } +trueno = { path = "crates/aprender-compute", version = "0.70.2", package = "aprender-compute" } +presentar-core = { path = "crates/aprender-present-core", version = "0.70.2", package = "aprender-present-core" } +presentar-terminal = { path = "crates/aprender-present-terminal", version = "0.70.2", package = "aprender-present-terminal" } +presentar-widgets = { path = "crates/aprender-present-widgets", version = "0.70.2", package = "aprender-present-widgets" } +presentar-layout = { path = "crates/aprender-present-layout", version = "0.70.2", package = "aprender-present-layout" } +presentar-yaml = { path = "crates/aprender-present-yaml", version = "0.70.2", package = "aprender-present-yaml" } +presentar-test = { path = "crates/aprender-present-test", version = "0.70.2", package = "aprender-present-test" } +presentar-test-macros = { path = "crates/aprender-present-test-macros", version = "0.70.2", package = "aprender-present-test-macros" } +jugar-probar-derive = { path = "crates/aprender-test-derive", version = "0.70.2", package = "aprender-test-derive" } +batuta-common = { path = "crates/aprender-common", version = "0.70.2", package = "aprender-common" } # APR-MONO sibling aliases — MUST use workspace refs, NEVER crates.io versions. # Hard-pinning these to crates.io creates a diamond: sub-crates pull older # aprender versions → multiple `aprender` crates in Cargo.lock = MUDA. -renacer = { path = "crates/aprender-profile", version = "0.70.1", package = "aprender-profile" } -entrenar-lora = { path = "crates/aprender-train-lora", version = "0.70.1", package = "aprender-train-lora" } -batuta = { path = "crates/aprender-orchestrate", version = "0.70.1", package = "aprender-orchestrate" } -trueno-graph = { path = "crates/aprender-graph", version = "0.70.1", package = "aprender-graph" } +renacer = { path = "crates/aprender-profile", version = "0.70.2", package = "aprender-profile" } +entrenar-lora = { path = "crates/aprender-train-lora", version = "0.70.2", package = "aprender-train-lora" } +batuta = { path = "crates/aprender-orchestrate", version = "0.70.2", package = "aprender-orchestrate" } +trueno-graph = { path = "crates/aprender-graph", version = "0.70.2", package = "aprender-graph" } [workspace.lints.rust] # Safety @@ -583,7 +583,7 @@ semicolon_if_nothing_returned = "allow" # Style preference [package] name = "aprender" -version = "0.70.1" +version = "0.70.2" edition = "2021" rust-version = "1.91" authors = ["Noah Gift "] @@ -746,9 +746,9 @@ required-features = ["cli"] # (e.g. apr-cookbook, downstream apps) don't pull apr-cli's heavy transitive # dep graph (renacer/profile/inference/zram/visualization). `cargo install # aprender` still works because `cli` is in default features. See GH-1599. -apr-cli = { path = "crates/apr-cli", version = "0.70.1", default-features = true, optional = true } +apr-cli = { path = "crates/apr-cli", version = "0.70.2", default-features = true, optional = true } # Core ML library (lib name = "aprender") -aprender_ml = { path = "crates/aprender-core", version = "0.70.1", package = "aprender-core" } +aprender_ml = { path = "crates/aprender-core", version = "0.70.2", package = "aprender-core" } [features] default = ["cli"] diff --git a/contracts/model-capability-ladder-v1.yaml b/contracts/model-capability-ladder-v1.yaml index 859815ff96..5caa9a71f0 100644 --- a/contracts/model-capability-ladder-v1.yaml +++ b/contracts/model-capability-ladder-v1.yaml @@ -97,6 +97,12 @@ ladder: # model (from the prompt certification), every thinking mode below, all GREEN with a GREEN positive # control, receipts bound to the cut exactly. Refused for any other release. emergency_scopes: + - name: crux-smoke + release: "0.70.2" + date: "2026-10-07" + quote: '"0.70.2 ships on CRUX smoke on lambda and gx10 GPU. Everything bigger is nightly."' + hosts: [lambda, gx10] + thinking: ["off", "on"] - name: crux-smoke release: "0.70.1" date: "2026-10-03" diff --git a/crates/apr-cli/Cargo.toml b/crates/apr-cli/Cargo.toml index b2276a6780..ca75cb48e9 100644 --- a/crates/apr-cli/Cargo.toml +++ b/crates/apr-cli/Cargo.toml @@ -130,12 +130,12 @@ libc = "0.2" # performed. A surface and its capability must not be able to drift. apr-test = { workspace = true } # Core aprender library -aprender = { path = "../aprender-core", version = "0.70.1", package = "aprender-core", default-features = true, features = ["format-compression"] } +aprender = { path = "../aprender-core", version = "0.70.2", package = "aprender-core", default-features = true, features = ["format-compression"] } # Provable-contracts engine — `apr beat-run` parses + evaluates BeatBenchmark contracts (PMAT-741) aprender-contracts = { workspace = true } # MCP (Model Context Protocol) server — `apr mcp` subcommand -aprender-mcp = { path = "../aprender-mcp", version = "0.70.1", features = ["apr-tools"] } # apr mcp serves apr's own tools (PMAT-3954) +aprender-mcp = { path = "../aprender-mcp", version = "0.70.2", features = ["apr-tools"] } # apr mcp serves apr's own tools (PMAT-3954) async-stream = "0.3" provable-contracts-macros = { workspace = true } @@ -146,7 +146,7 @@ batuta-common = { workspace = true, features = ["cli-roles"] } # Non-optional since claude-code-parity-apr M150 (outcome-parity Phase 3 # requires apr code in default builds). GH-344: path dep via # .cargo/config.toml.dev-overrides; CI uses checkout+symlink. -batuta = { version = "0.70.1", path = "../aprender-orchestrate", package = "aprender-orchestrate", default-features = false, features = ["agents", "agents-inference", "agents-mcp", "rag"] } +batuta = { version = "0.70.2", path = "../aprender-orchestrate", package = "aprender-orchestrate", default-features = false, features = ["agents", "agents-inference", "agents-mcp", "rag"] } # Inference engine (workspace path dep — enables cuda/gpu feature forwarding) # GH-573: shim (0.9.1) didn't forward cuda feature → 0.7 tok/s instead of 273 tok/s @@ -242,7 +242,7 @@ pacha = { workspace = true } # published-crate dependency cycle the APR-MONO consolidation removed, and # (b) was never actually linked, so every `trueno_explain::` path in # commands/ptx_explain.rs failed to resolve. -trueno-explain = { path = "../aprender-explain", version = "0.70.1", package = "aprender-explain", optional = true } +trueno-explain = { path = "../aprender-explain", version = "0.70.2", package = "aprender-explain", optional = true } # ML tuning: LoRA/QLoRA configuration and memory planning (GH-176, PMAT-184) entrenar-lora = { workspace = true, optional = true } diff --git a/crates/aprender-bench-compute/Cargo.toml b/crates/aprender-bench-compute/Cargo.toml index 1956918936..66eaec19e9 100644 --- a/crates/aprender-bench-compute/Cargo.toml +++ b/crates/aprender-bench-compute/Cargo.toml @@ -11,7 +11,7 @@ disallowed_methods = "allow" [dependencies] # The implementation under test -aprender = { path = "../aprender-core", version = "0.70.1", package = "aprender-core", features = ["format-quantize"] } +aprender = { path = "../aprender-core", version = "0.70.2", package = "aprender-core", features = ["format-quantize"] } # Reference implementation for A/B comparison # Pure Rust (no BLAS) to keep comparison fair — both are pure Rust SIMD diff --git a/crates/aprender-bench-tokenizer/Cargo.toml b/crates/aprender-bench-tokenizer/Cargo.toml index 74b991ee5f..e468396908 100644 --- a/crates/aprender-bench-tokenizer/Cargo.toml +++ b/crates/aprender-bench-tokenizer/Cargo.toml @@ -11,7 +11,7 @@ disallowed_methods = "allow" [dependencies] # The implementation under test -aprender = { path = "../aprender-core", version = "0.70.1", package = "aprender-core" } +aprender = { path = "../aprender-core", version = "0.70.2", package = "aprender-core" } # HuggingFace reference for A/B comparison # Regular dep (not dev-dep) because [[bench]] harness=false binaries can't see dev-deps diff --git a/crates/aprender-cbtop/Cargo.toml b/crates/aprender-cbtop/Cargo.toml index ea1784234e..1e09fedbe8 100644 --- a/crates/aprender-cbtop/Cargo.toml +++ b/crates/aprender-cbtop/Cargo.toml @@ -32,7 +32,7 @@ presentar-terminal = { workspace = true } # Compute backends trueno = { workspace = true } -trueno-gpu = { version = "0.70.1", path = "../aprender-gpu", package = "aprender-gpu", optional = true } +trueno-gpu = { version = "0.70.2", path = "../aprender-gpu", package = "aprender-gpu", optional = true } # Terminal handling crossterm = "0.28" diff --git a/crates/aprender-cgp/Cargo.toml b/crates/aprender-cgp/Cargo.toml index 8851669a7b..2bf5938bdd 100644 --- a/crates/aprender-cgp/Cargo.toml +++ b/crates/aprender-cgp/Cargo.toml @@ -36,7 +36,7 @@ num_cpus = "1.17" comfy-table = "7" colored = "3" # GPU profiling: direct cuBLAS measurement (OWN THE STACK) -trueno-gpu = { path = "../aprender-gpu", version = "0.70.1", features = ["cuda"], optional = true, package = "aprender-gpu" } +trueno-gpu = { path = "../aprender-gpu", version = "0.70.2", features = ["cuda"], optional = true, package = "aprender-gpu" } [features] default = [] diff --git a/crates/aprender-compute/Cargo.toml b/crates/aprender-compute/Cargo.toml index b975bc0680..b155c3d754 100644 --- a/crates/aprender-compute/Cargo.toml +++ b/crates/aprender-compute/Cargo.toml @@ -41,11 +41,11 @@ futures-intrusive = { version = "0.5", optional = true } wasm-bindgen-futures = { version = "0.4", optional = true } wasm-bindgen = { version = "0.2", optional = true } # Native CUDA monitoring via trueno-gpu (TRUENO-SPEC-010) -trueno-gpu = { version = "0.70.1", path = "../aprender-gpu", optional = true, package = "aprender-gpu" } +trueno-gpu = { version = "0.70.2", path = "../aprender-gpu", optional = true, package = "aprender-gpu" } # PMAT-336: Provable contracts compile-time enforcement provable-contracts-macros = { workspace = true } # P1a: Sovereign GEMM microkernel codegen (compile-time shape specialization) -trueno-gemm-codegen = { version = "0.70.1", path = "../aprender-gemm-codegen", package = "aprender-gemm-codegen" } +trueno-gemm-codegen = { version = "0.70.2", path = "../aprender-gemm-codegen", package = "aprender-gemm-codegen" } # Tracing/profiling support (renacer integration) tracing = { version = "0.1", optional = true } tracing-subscriber = { version = "0.3", features = ["env-filter", "fmt"], optional = true } diff --git a/crates/aprender-compute/src/blis/microkernels/avx2.rs b/crates/aprender-compute/src/blis/microkernels/avx2.rs index e98e7f86cd..9282ac0d07 100644 --- a/crates/aprender-compute/src/blis/microkernels/avx2.rs +++ b/crates/aprender-compute/src/blis/microkernels/avx2.rs @@ -314,6 +314,15 @@ pub unsafe fn microkernel_8x6_true_asm( ".p2align 4", // Align loop for better I-cache "3:", + // Reload source for this iteration. On the last iteration there is + // no next block: A[k+4..k+7] lies past the panel, and for the last + // ir panel past the allocation (#4891, SIGSEGV on a guard page). + // So the last iteration reloads the current, in-bounds block; the + // values are dead, the k % 4 remainder reads A from memory. + "lea {tmp}, [{a_ptr} - 128]", + "cmp {k_cnt}, 1", + "cmovne {tmp}, {a_ptr}", + // --- K iteration 0: Use ymm6 (A[0]), load next A[4] into ymm6 --- "vbroadcastss ymm10, dword ptr [{b_ptr}]", "vbroadcastss ymm11, dword ptr [{b_ptr} + 4]", @@ -327,7 +336,7 @@ pub unsafe fn microkernel_8x6_true_asm( "vfmadd231ps ymm3, ymm6, ymm13", // c3 += a0 * b3 "vfmadd231ps ymm4, ymm6, ymm14", // c4 += a0 * b4 "vfmadd231ps ymm5, ymm6, ymm15", // c5 += a0 * b5 - "vmovups ymm6, [{a_ptr}]", // Reload A[4] -> ymm6 (reuse register) + "vmovups ymm6, [{tmp}]", // Reload A[4] -> ymm6 (reuse register) // --- K iteration 1: Use ymm7 (A[1]), load next A[5] into ymm7 --- "vbroadcastss ymm10, dword ptr [{b_ptr} + 24]", @@ -342,7 +351,7 @@ pub unsafe fn microkernel_8x6_true_asm( "vfmadd231ps ymm3, ymm7, ymm13", "vfmadd231ps ymm4, ymm7, ymm14", "vfmadd231ps ymm5, ymm7, ymm15", - "vmovups ymm7, [{a_ptr} + 32]", // Reload A[5] -> ymm7 + "vmovups ymm7, [{tmp} + 32]", // Reload A[5] -> ymm7 // --- K iteration 2: Use ymm8 (A[2]), load next A[6] into ymm8 --- "vbroadcastss ymm10, dword ptr [{b_ptr} + 48]", @@ -357,7 +366,7 @@ pub unsafe fn microkernel_8x6_true_asm( "vfmadd231ps ymm3, ymm8, ymm13", "vfmadd231ps ymm4, ymm8, ymm14", "vfmadd231ps ymm5, ymm8, ymm15", - "vmovups ymm8, [{a_ptr} + 64]", // Reload A[6] -> ymm8 + "vmovups ymm8, [{tmp} + 64]", // Reload A[6] -> ymm8 // --- K iteration 3: Use ymm9 (A[3]), load next A[7] into ymm9 --- "vbroadcastss ymm10, dword ptr [{b_ptr} + 72]", @@ -372,7 +381,7 @@ pub unsafe fn microkernel_8x6_true_asm( "vfmadd231ps ymm3, ymm9, ymm13", "vfmadd231ps ymm4, ymm9, ymm14", "vfmadd231ps ymm5, ymm9, ymm15", - "vmovups ymm9, [{a_ptr} + 96]", // Reload A[7] -> ymm9 + "vmovups ymm9, [{tmp} + 96]", // Reload A[7] -> ymm9 // Advance pointers for next 4 K iterations "add {a_ptr}, 128", // 4 * MR * sizeof(f32) = 4 * 8 * 4 = 128 diff --git a/crates/aprender-compute/src/blis/tests/microkernel.rs b/crates/aprender-compute/src/blis/tests/microkernel.rs index 2591ce043d..9244d65ed4 100644 --- a/crates/aprender-compute/src/blis/tests/microkernel.rs +++ b/crates/aprender-compute/src/blis/tests/microkernel.rs @@ -407,3 +407,128 @@ fn test_microkernel_true_asm_accumulates_into_c() { ); } } + +// ======================================================================== +// FALSIFY-BLIS-ASM-OOB: the asm kernel must not read past its A or B panel +// ======================================================================== +// +// The last ir panel of a packed A buffer ends exactly at the end of its +// allocation. A kernel that reads even one byte past the panel can cross +// into an unmapped page and fault, and only when the allocator happens to +// put the buffer there. These tests put the panel end on a PROT_NONE page, +// so any read past it faults on every run instead of one run in many. + +#[cfg(all(target_arch = "x86_64", target_os = "linux"))] +mod guard_page { + use std::ffi::c_void; + + const PROT_NONE: i32 = 0; + const PROT_READ: i32 = 1; + const PROT_WRITE: i32 = 2; + const MAP_PRIVATE: i32 = 0x02; + const MAP_ANONYMOUS: i32 = 0x20; + const SC_PAGESIZE: i32 = 30; + + extern "C" { + fn mmap( + addr: *mut c_void, + len: usize, + prot: i32, + flags: i32, + fd: i32, + off: i64, + ) -> *mut c_void; + fn mprotect(addr: *mut c_void, len: usize, prot: i32) -> i32; + fn munmap(addr: *mut c_void, len: usize) -> i32; + fn sysconf(name: i32) -> i64; + } + + /// `len` f32s whose last element is the last readable byte before a + /// PROT_NONE page. + pub struct GuardedF32 { + base: *mut c_void, + map_len: usize, + data: *mut f32, + len: usize, + } + + impl GuardedF32 { + pub fn new(values: &[f32]) -> Self { + // SAFETY: plain libc calls; every result is checked before use. + unsafe { + let page = usize::try_from(sysconf(SC_PAGESIZE)).expect("page size"); + let bytes = values.len() * 4; + let data_pages = bytes.div_ceil(page).max(1); + let map_len = (data_pages + 1) * page; + let base = mmap( + std::ptr::null_mut(), + map_len, + PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, + -1, + 0, + ); + assert!(base as isize != -1, "mmap failed"); + let guard = base.cast::().add(data_pages * page); + assert_eq!(mprotect(guard.cast(), page, PROT_NONE), 0, "mprotect failed"); + let data = guard.sub(bytes).cast::(); + std::ptr::copy_nonoverlapping(values.as_ptr(), data, values.len()); + Self { base, map_len, data, len: values.len() } + } + } + + pub fn as_ptr(&self) -> *const f32 { + self.data + } + + pub fn len(&self) -> usize { + self.len + } + } + + impl Drop for GuardedF32 { + fn drop(&mut self) { + // SAFETY: base/map_len are exactly what mmap returned. + unsafe { + munmap(self.base, self.map_len); + } + } + } +} + +#[test] +#[cfg(all(target_arch = "x86_64", target_os = "linux"))] +fn falsify_blis_asm_oob_001_true_asm_stays_inside_panels() { + if !is_x86_feature_detected!("avx2") || !is_x86_feature_detected!("fma") { + return; + } + + // Multiples of 4 (the unroll), each remainder, and the KC block size. + for k in [4usize, 5, 6, 7, 8, 12, 64, 255, 256] { + let a_vals: Vec = (0..MR * k).map(|i| ((i % 17) as f32) * 0.25 - 2.0).collect(); + let b_vals: Vec = (0..k * NR).map(|i| ((i % 13) as f32) * 0.5 - 3.0).collect(); + let a = guard_page::GuardedF32::new(&a_vals); + let b = guard_page::GuardedF32::new(&b_vals); + assert_eq!(a.len(), MR * k); + assert_eq!(b.len(), k * NR); + + let mut c_ref = vec![1.0f32; MR * NR]; + let mut c_asm = vec![1.0f32; MR * NR]; + microkernel_scalar(k, &a_vals, &b_vals, &mut c_ref, MR); + + // SAFETY: A holds MR*k and B holds k*NR f32s; C holds MR*NR with ldc = MR. + unsafe { + microkernel_8x6_true_asm(k, a.as_ptr(), b.as_ptr(), c_asm.as_mut_ptr(), MR); + } + + for i in 0..MR * NR { + let tol = 1e-4 * c_ref[i].abs().max(1.0); + assert!( + (c_ref[i] - c_asm[i]).abs() <= tol, + "FALSIFY-BLIS-ASM-OOB-001 k={k} i={i}: scalar {} vs asm {}", + c_ref[i], + c_asm[i] + ); + } + } +} diff --git a/crates/aprender-contracts-cli/Cargo.toml b/crates/aprender-contracts-cli/Cargo.toml index 8e16c8e976..73a9cccd3e 100644 --- a/crates/aprender-contracts-cli/Cargo.toml +++ b/crates/aprender-contracts-cli/Cargo.toml @@ -18,7 +18,7 @@ name = "pv-sat" path = "src/bin/pv-sat/main.rs" [dependencies] -aprender-contracts = { path = "../aprender-contracts", version = "0.70.1" } +aprender-contracts = { path = "../aprender-contracts", version = "0.70.2" } # #3745 S1: `--model` is declared through the shared role type `ModelPath`, so # `apr surface` reads `apr pv verify-structure --model` as a model argument. batuta-common = { workspace = true, features = ["cli-roles"] } diff --git a/crates/aprender-contracts/Cargo.toml b/crates/aprender-contracts/Cargo.toml index 9f4a4b0e77..91615b9809 100644 --- a/crates/aprender-contracts/Cargo.toml +++ b/crates/aprender-contracts/Cargo.toml @@ -26,7 +26,7 @@ kani = [] name = "provable_contracts" [dependencies] -provable-contracts-macros = { path = "../aprender-contracts-macros", version = "0.70.1", package = "aprender-contracts-macros" } +provable-contracts-macros = { path = "../aprender-contracts-macros", version = "0.70.2", package = "aprender-contracts-macros" } regex = "1" sha2 = { workspace = true } blake3 = "1" diff --git a/crates/aprender-cuda-edge/Cargo.toml b/crates/aprender-cuda-edge/Cargo.toml index 27d1bef852..3340a5415a 100644 --- a/crates/aprender-cuda-edge/Cargo.toml +++ b/crates/aprender-cuda-edge/Cargo.toml @@ -15,7 +15,7 @@ name = "trueno_cuda_edge" [dependencies] thiserror = "2.0" serde = { version = "1.0", features = ["derive"] } -trueno-gpu = { version = "0.70.1", path = "../aprender-gpu", package = "aprender-gpu", optional = true } +trueno-gpu = { version = "0.70.2", path = "../aprender-gpu", package = "aprender-gpu", optional = true } [dev-dependencies] proptest = "1.6" diff --git a/crates/aprender-explain/Cargo.toml b/crates/aprender-explain/Cargo.toml index f0da6e4cba..4dc6f8eeee 100644 --- a/crates/aprender-explain/Cargo.toml +++ b/crates/aprender-explain/Cargo.toml @@ -25,7 +25,7 @@ presentar-terminal = { workspace = true } crossterm = "0.28" # Internal dependency for PTX generation -trueno-gpu = { version = "0.70.1", path = "../aprender-gpu", package = "aprender-gpu", features = ["cuda"] } +trueno-gpu = { version = "0.70.2", path = "../aprender-gpu", package = "aprender-gpu", features = ["cuda"] } [build-dependencies] aprender-build-sha = { workspace = true } # #4219: APR_GIT_SHA for --version diff --git a/crates/aprender-graph/Cargo.toml b/crates/aprender-graph/Cargo.toml index 706fd69d27..acfe6f2dc5 100644 --- a/crates/aprender-graph/Cargo.toml +++ b/crates/aprender-graph/Cargo.toml @@ -28,7 +28,7 @@ anyhow = "1" thiserror = "2" # Phase 2+ dependencies -aprender = { path = "../aprender-core", version = "0.70.1", package = "aprender-core" } # Sparse matrix ops, PageRank +aprender = { path = "../aprender-core", version = "0.70.2", package = "aprender-core" } # Sparse matrix ops, PageRank trueno = { workspace = true } # Phase 3: GPU acceleration (optional - requires wgpu-capable hardware) diff --git a/crates/aprender-monte-carlo/Cargo.toml b/crates/aprender-monte-carlo/Cargo.toml index f253f9ddae..c9eacbea2d 100644 --- a/crates/aprender-monte-carlo/Cargo.toml +++ b/crates/aprender-monte-carlo/Cargo.toml @@ -18,7 +18,7 @@ path = "src/lib.rs" name = "aprender-monte-carlo" [dependencies] -aprender = { path = "../../", version = "0.70.1" } +aprender = { path = "../../", version = "0.70.2" } clap = { version = "4", features = ["derive"] } rand = "0.9" rand_chacha = "0.9" diff --git a/crates/aprender-orchestrate/Cargo.toml b/crates/aprender-orchestrate/Cargo.toml index 9fa336db64..3958197def 100644 --- a/crates/aprender-orchestrate/Cargo.toml +++ b/crates/aprender-orchestrate/Cargo.toml @@ -132,7 +132,7 @@ pacha = { workspace = true, optional = true } realizar = { workspace = true, optional = true } # ML algorithms and training (native-only) -aprender = { path = "../aprender-core", version = "0.70.1", package = "aprender-core", optional = true } +aprender = { path = "../aprender-core", version = "0.70.2", package = "aprender-core", optional = true } entrenar = { workspace = true, optional = true } alimentar = { workspace = true, optional = true } diff --git a/crates/aprender-profile/Cargo.toml b/crates/aprender-profile/Cargo.toml index f2a9e2ace0..257be96d5e 100644 --- a/crates/aprender-profile/Cargo.toml +++ b/crates/aprender-profile/Cargo.toml @@ -74,7 +74,7 @@ trueno = { workspace = true } # Machine learning for anomaly detection (Sprint 23) # GH-581: Use path dep to avoid trueno version skew with local workspace -aprender = { path = "../aprender-core", version = "0.70.1", package = "aprender-core" } +aprender = { path = "../aprender-core", version = "0.70.2", package = "aprender-core" } # Lock-free data structures for ring buffer (Sprint 40) crossbeam = { version = "0.8", default-features = false, features = ["crossbeam-channel", "crossbeam-queue", "std"] } @@ -112,7 +112,7 @@ crossterm = "0.28" # SIMD-accelerated visualization primitives (sister project) # NOTE: `monitor` feature removed upstream (#1979); this crate uses only viz primitives. -trueno-viz = { version = "0.70.1", path = "../aprender-viz", package = "aprender-viz", default-features = false } +trueno-viz = { version = "0.70.2", path = "../aprender-viz", package = "aprender-viz", default-features = false } # OpenTelemetry for distributed tracing (Sprint 30) opentelemetry = { version = "0.31.0", optional = true } diff --git a/crates/aprender-qa-cli/Cargo.toml b/crates/aprender-qa-cli/Cargo.toml index 799e21d8cf..ced155fe12 100644 --- a/crates/aprender-qa-cli/Cargo.toml +++ b/crates/aprender-qa-cli/Cargo.toml @@ -23,10 +23,10 @@ serde_json = { workspace = true } serde_yaml = { workspace = true } clap = { workspace = true } chrono = { workspace = true } -aprender-qa-gen = { version = "0.70.1", path = "../aprender-qa-gen" } -aprender-qa-runner = { version = "0.70.1", path = "../aprender-qa-runner" } -aprender-qa-report = { version = "0.70.1", path = "../aprender-qa-report" } -aprender-qa-certify = { version = "0.70.1", path = "../aprender-qa-certify" } +aprender-qa-gen = { version = "0.70.2", path = "../aprender-qa-gen" } +aprender-qa-runner = { version = "0.70.2", path = "../aprender-qa-runner" } +aprender-qa-report = { version = "0.70.2", path = "../aprender-qa-report" } +aprender-qa-certify = { version = "0.70.2", path = "../aprender-qa-certify" } ctrlc = "3.4" safetensors = { workspace = true } colored = "2.2" diff --git a/crates/aprender-qa-report/Cargo.toml b/crates/aprender-qa-report/Cargo.toml index abed239da5..ee4918c2ae 100644 --- a/crates/aprender-qa-report/Cargo.toml +++ b/crates/aprender-qa-report/Cargo.toml @@ -18,8 +18,8 @@ anyhow = { workspace = true } chrono = { workspace = true } colored = "2.2" csv = "1.3" -aprender-qa-gen = { version = "0.70.1", path = "../aprender-qa-gen" } -aprender-qa-runner = { version = "0.70.1", path = "../aprender-qa-runner" } +aprender-qa-gen = { version = "0.70.2", path = "../aprender-qa-gen" } +aprender-qa-runner = { version = "0.70.2", path = "../aprender-qa-runner" } [dev-dependencies] proptest = { workspace = true } diff --git a/crates/aprender-qa-runner/Cargo.toml b/crates/aprender-qa-runner/Cargo.toml index bcaa9d3263..ad530dfb94 100644 --- a/crates/aprender-qa-runner/Cargo.toml +++ b/crates/aprender-qa-runner/Cargo.toml @@ -22,14 +22,14 @@ anyhow = { workspace = true } tokio = { workspace = true } chrono = { workspace = true, features = ["serde"] } tracing = { workspace = true } -aprender-qa-gen = { version = "0.70.1", path = "../aprender-qa-gen" } +aprender-qa-gen = { version = "0.70.2", path = "../aprender-qa-gen" } hostname = "0.4" rayon = "1.10" sha2 = "0.10" regex = { workspace = true } num_cpus = "1.16" safetensors = { workspace = true } -jugar-probar = { version = "0.70.1", path = "../aprender-test-lib", package = "aprender-test-lib", features = ["llm-types"] } +jugar-probar = { version = "0.70.2", path = "../aprender-test-lib", package = "aprender-test-lib", features = ["llm-types"] } # Test fixtures dependencies (optional) tempfile = { version = "3", optional = true } half = { version = "2.4", optional = true } diff --git a/crates/aprender-rag-cli/Cargo.toml b/crates/aprender-rag-cli/Cargo.toml index 3f0ed1f2c0..82203e4408 100644 --- a/crates/aprender-rag-cli/Cargo.toml +++ b/crates/aprender-rag-cli/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "aprender-rag-cli" -version = "0.70.1" +version = "0.70.2" edition = "2021" authors = ["Pragmatic AI Labs"] description = "CLI for Trueno-RAG pipeline" diff --git a/crates/aprender-rag/Cargo.toml b/crates/aprender-rag/Cargo.toml index a10f7a5484..6b743d7da6 100644 --- a/crates/aprender-rag/Cargo.toml +++ b/crates/aprender-rag/Cargo.toml @@ -63,7 +63,7 @@ fastembed = { version = "5", optional = true } # NVIDIA Embed Nemotron 8B via realizar GGUF infrastructure (GH-3) # Large model (~4-32GB depending on quantization), requires GGUF file -realizar = { version = "0.70.1", path = "../aprender-serve", package = "aprender-serve", default-features = false, optional = true } +realizar = { version = "0.70.2", path = "../aprender-serve", package = "aprender-serve", default-features = false, optional = true } # Eval: LLM-as-judge via Anthropic API (PMAT-015) diff --git a/crates/aprender-registry/Cargo.toml b/crates/aprender-registry/Cargo.toml index 58c7241082..66bcd11524 100644 --- a/crates/aprender-registry/Cargo.toml +++ b/crates/aprender-registry/Cargo.toml @@ -42,7 +42,7 @@ zstd = { version = "0.13", optional = true } clap = { version = "4", features = ["derive"], optional = true } # Ecosystem integration -aprender = { path = "../aprender-core", version = "0.70.1", package = "aprender-core", optional = true } +aprender = { path = "../aprender-core", version = "0.70.2", package = "aprender-core", optional = true } alimentar = { workspace = true, optional = true } # HTTP client (optional) diff --git a/crates/aprender-serve/Cargo.toml b/crates/aprender-serve/Cargo.toml index e1ee800866..660e7364bb 100644 --- a/crates/aprender-serve/Cargo.toml +++ b/crates/aprender-serve/Cargo.toml @@ -51,7 +51,7 @@ trueno = { workspace = true, features = ["gpu"] } unicode-properties = { workspace = true } # CUDA PTX generation and runtime for NVIDIA GPUs (optional, pure Rust, no LLVM/nvcc) # v0.4.12: BatchedQ6K, MultiWarp, RopeNeox, FlashDecoding kernels -trueno-gpu = { version = "0.70.1", path = "../aprender-gpu", package = "aprender-gpu", optional = true, features = ["cuda"] } # APR-MONO single-source-of-truth +trueno-gpu = { version = "0.70.2", path = "../aprender-gpu", package = "aprender-gpu", optional = true, features = ["cuda"] } # APR-MONO single-source-of-truth # KV cache storage (for attention caching) trueno-db = { workspace = true, optional = true } # K-quantization formats (Q4_K, Q5_K, Q6_K) - Toyota Way: ONE source of truth @@ -65,7 +65,7 @@ renacer-core = { workspace = true } # ML algorithms and .apr model format (optional for aprender model serving) # v0.24.1: chat templates, GGUF tokenizer preservation (local may be newer) -aprender = { path = "../aprender-core", version = "0.70.1", package = "aprender-core", optional = true } +aprender = { path = "../aprender-core", version = "0.70.2", package = "aprender-core", optional = true } # Data loading library (optional for data pipeline examples) alimentar = { workspace = true, optional = true } diff --git a/crates/aprender-shell/Cargo.toml b/crates/aprender-shell/Cargo.toml index 0b96a37adc..ce83cdc96a 100644 --- a/crates/aprender-shell/Cargo.toml +++ b/crates/aprender-shell/Cargo.toml @@ -24,7 +24,7 @@ harness = false [dependencies] # Use path for local dev, crates.io for release -aprender = { path = "../aprender-core", version = "0.70.1", package = "aprender-core", features = ["format-encryption", "format-compression", "format-homomorphic"] } +aprender = { path = "../aprender-core", version = "0.70.2", package = "aprender-core", features = ["format-encryption", "format-compression", "format-homomorphic"] } clap = { version = "4", features = ["derive"] } dirs = "5" serde = { version = "1", features = ["derive"] } diff --git a/crates/aprender-simulate/Cargo.toml b/crates/aprender-simulate/Cargo.toml index 589b2412e2..356fd3a023 100644 --- a/crates/aprender-simulate/Cargo.toml +++ b/crates/aprender-simulate/Cargo.toml @@ -59,7 +59,7 @@ crossterm = { version = "0.28", optional = true } # ComputeBlocks from presentar-terminal (SIMD-optimized TUI widgets) # Path deps to avoid jugar-probar prod dep chain (CB-081) presentar-terminal = { workspace = true, optional = true } -presentar-core = { version = "0.70.1", path = "../aprender-present-core", package = "aprender-present-core", default-features = false, optional = true } +presentar-core = { version = "0.70.2", path = "../aprender-present-core", package = "aprender-present-core", default-features = false, optional = true } # WASM (optional) wasm-bindgen = { version = "0.2", optional = true } diff --git a/crates/aprender-test-cli/Cargo.toml b/crates/aprender-test-cli/Cargo.toml index 023c799c44..c7287b0908 100644 --- a/crates/aprender-test-cli/Cargo.toml +++ b/crates/aprender-test-cli/Cargo.toml @@ -35,7 +35,7 @@ path = "src/main.rs" [dependencies] # Core library (was: jugar-probar) — dep key preserves Rust identifier -jugar-probar = { path = "../aprender-test-lib", version = "0.70.1", package = "aprender-test-lib" } +jugar-probar = { path = "../aprender-test-lib", version = "0.70.2", package = "aprender-test-lib" } # CLI framework clap = { workspace = true } diff --git a/crates/aprender-train-bench/Cargo.toml b/crates/aprender-train-bench/Cargo.toml index 449b58cfc4..9a3cfbd266 100644 --- a/crates/aprender-train-bench/Cargo.toml +++ b/crates/aprender-train-bench/Cargo.toml @@ -20,7 +20,7 @@ workspace = true [dependencies] entrenar = { workspace = true } -entrenar-common = { version = "0.70.1", path = "../aprender-train-common", package = "aprender-train-common" } +entrenar-common = { version = "0.70.2", path = "../aprender-train-common", package = "aprender-train-common" } clap = { version = "4.5", features = ["derive"] } serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" diff --git a/crates/aprender-train-distill/Cargo.toml b/crates/aprender-train-distill/Cargo.toml index 9d3db55f30..7523f6dd2d 100644 --- a/crates/aprender-train-distill/Cargo.toml +++ b/crates/aprender-train-distill/Cargo.toml @@ -33,7 +33,7 @@ shard-batch-source = [] [dependencies] entrenar = { workspace = true } -entrenar-common = { version = "0.70.1", path = "../aprender-train-common", package = "aprender-train-common" } +entrenar-common = { version = "0.70.2", path = "../aprender-train-common", package = "aprender-train-common" } clap = { version = "4.5", features = ["derive"] } serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" diff --git a/crates/aprender-train-inspect/Cargo.toml b/crates/aprender-train-inspect/Cargo.toml index 91990fa433..ca5c6bf9b4 100644 --- a/crates/aprender-train-inspect/Cargo.toml +++ b/crates/aprender-train-inspect/Cargo.toml @@ -20,7 +20,7 @@ workspace = true [dependencies] entrenar = { workspace = true } -entrenar-common = { version = "0.70.1", path = "../aprender-train-common", package = "aprender-train-common" } +entrenar-common = { version = "0.70.2", path = "../aprender-train-common", package = "aprender-train-common" } clap = { version = "4.5", features = ["derive"] } serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" diff --git a/crates/aprender-train-lora/Cargo.toml b/crates/aprender-train-lora/Cargo.toml index 84252645b0..c12eba23cb 100644 --- a/crates/aprender-train-lora/Cargo.toml +++ b/crates/aprender-train-lora/Cargo.toml @@ -20,7 +20,7 @@ workspace = true [dependencies] entrenar = { workspace = true } -entrenar-common = { version = "0.70.1", path = "../aprender-train-common", package = "aprender-train-common" } +entrenar-common = { version = "0.70.2", path = "../aprender-train-common", package = "aprender-train-common" } clap = { version = "4.5", features = ["derive"] } serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" diff --git a/crates/aprender-train-shell/Cargo.toml b/crates/aprender-train-shell/Cargo.toml index 3992313c60..cacaa82c37 100644 --- a/crates/aprender-train-shell/Cargo.toml +++ b/crates/aprender-train-shell/Cargo.toml @@ -20,7 +20,7 @@ workspace = true [dependencies] entrenar = { workspace = true } -entrenar-common = { version = "0.70.1", path = "../aprender-train-common", package = "aprender-train-common" } +entrenar-common = { version = "0.70.2", path = "../aprender-train-common", package = "aprender-train-common" } clap = { version = "4.5", features = ["derive"] } rustyline = "14" serde = { version = "1.0", features = ["derive"] } diff --git a/crates/aprender-train/Cargo.toml b/crates/aprender-train/Cargo.toml index bf58fd6270..d5b4e0064e 100644 --- a/crates/aprender-train/Cargo.toml +++ b/crates/aprender-train/Cargo.toml @@ -80,9 +80,9 @@ provable-contracts-macros = { workspace = true } # Core PAIML stack ndarray = "0.16" trueno = { workspace = true, features = ["parallel"] } -trueno-gpu = { version = "0.70.1", path = "../aprender-gpu", package = "aprender-gpu", optional = true } # CUDA compute (APR-MONO single-source-of-truth) +trueno-gpu = { version = "0.70.2", path = "../aprender-gpu", package = "aprender-gpu", optional = true } # CUDA compute (APR-MONO single-source-of-truth) realizar = { workspace = true, optional = true } -aprender = { path = "../aprender-core", version = "0.70.1", package = "aprender-core" } # ML interpret module + .apr format + tokenizers +aprender = { path = "../aprender-core", version = "0.70.2", package = "aprender-core" } # ML interpret module + .apr format + tokenizers trueno-viz = { workspace = true, features = ["terminal"] } batuta-common = { workspace = true, optional = true } # WithDimensions trait for `viz` feature (APR-MONO §S #1978) presentar-terminal = { workspace = true, optional = true } diff --git a/crates/aprender-tsp/Cargo.toml b/crates/aprender-tsp/Cargo.toml index c31a408322..6847ff5982 100644 --- a/crates/aprender-tsp/Cargo.toml +++ b/crates/aprender-tsp/Cargo.toml @@ -18,7 +18,7 @@ path = "src/lib.rs" name = "aprender-tsp" [dependencies] -aprender = { path = "../../", version = "0.70.1" } +aprender = { path = "../../", version = "0.70.2" } clap = { version = "4", features = ["derive"] } rand = "0.9" crc32fast = "1.4" diff --git a/crates/aprender-verify-ml/Cargo.toml b/crates/aprender-verify-ml/Cargo.toml index af21b8aec7..9cd8e266fb 100644 --- a/crates/aprender-verify-ml/Cargo.toml +++ b/crates/aprender-verify-ml/Cargo.toml @@ -56,7 +56,7 @@ uuid = { version = "1.11", features = ["v4", "serde"] } trueno = { workspace = true } # Machine learning for bug prediction (optional) -aprender = { path = "../aprender-core", version = "0.70.1", package = "aprender-core", optional = true } +aprender = { path = "../aprender-core", version = "0.70.2", package = "aprender-core", optional = true } # LLM fine-tuning integration (optional) entrenar = { workspace = true, optional = true } diff --git a/crates/aprender-viz/Cargo.toml b/crates/aprender-viz/Cargo.toml index 674cf58ab0..d1125598dd 100644 --- a/crates/aprender-viz/Cargo.toml +++ b/crates/aprender-viz/Cargo.toml @@ -14,7 +14,7 @@ categories = ["visualization", "graphics", "science", "wasm"] [dependencies] # Core SIMD/GPU acceleration -trueno = { version = "0.70.1", path = "../aprender-compute", package = "aprender-compute", default-features = false } +trueno = { version = "0.70.2", path = "../aprender-compute", package = "aprender-compute", default-features = false } # Shared Batuta stack utilities (display traits, formatting) batuta-common = { workspace = true } @@ -52,10 +52,10 @@ kurbo = { version = "0.13", optional = true } resvg = { version = "0.45", optional = true, default-features = false } # Optional: ML integration -aprender = { path = "../aprender-core", version = "0.70.1", package = "aprender-core", optional = true } +aprender = { path = "../aprender-core", version = "0.70.2", package = "aprender-core", optional = true } # Optional: Graph integration -trueno-graph = { version = "0.70.1", path = "../aprender-graph", package = "aprender-graph", default-features = false, optional = true } +trueno-graph = { version = "0.70.2", path = "../aprender-graph", package = "aprender-graph", default-features = false, optional = true } # Optional: Database integration trueno-db = { workspace = true, optional = true } diff --git a/crates/facades/Cargo.lock b/crates/facades/Cargo.lock index 83ffdce172..19914fe620 100644 --- a/crates/facades/Cargo.lock +++ b/crates/facades/Cargo.lock @@ -13,7 +13,7 @@ dependencies = [ [[package]] name = "aprender-contracts" -version = "0.70.1" +version = "0.70.2" dependencies = [ "aprender-contracts-macros", "blake3", @@ -29,7 +29,7 @@ dependencies = [ [[package]] name = "aprender-contracts-macros" -version = "0.70.1" +version = "0.70.2" dependencies = [ "proc-macro2", "quote", diff --git a/crates/facades/provable-contracts-macros/Cargo.toml b/crates/facades/provable-contracts-macros/Cargo.toml index 869cd8765d..909b33e441 100644 --- a/crates/facades/provable-contracts-macros/Cargo.toml +++ b/crates/facades/provable-contracts-macros/Cargo.toml @@ -30,7 +30,7 @@ path = "src/lib.rs" # facade resolves. `scripts/check_facade_compat.sh` fails if this version ever # drifts from the aprender workspace version — these manifests are excluded # from the root workspace, so `cargo set-version` does NOT reach them. -upstream = { path = "../../aprender-contracts-macros", version = "0.70.1", package = "aprender-contracts-macros" } +upstream = { path = "../../aprender-contracts-macros", version = "0.70.2", package = "aprender-contracts-macros" } [features] default = [] diff --git a/crates/facades/provable-contracts/Cargo.toml b/crates/facades/provable-contracts/Cargo.toml index ddd593093b..afcf98ea65 100644 --- a/crates/facades/provable-contracts/Cargo.toml +++ b/crates/facades/provable-contracts/Cargo.toml @@ -31,7 +31,7 @@ path = "src/lib.rs" # facade resolves. `scripts/check_facade_compat.sh` fails if this version ever # drifts from the aprender workspace version — these manifests are excluded # from the root workspace, so `cargo set-version` does NOT reach them. -upstream = { path = "../../aprender-contracts", version = "0.70.1", package = "aprender-contracts" } +upstream = { path = "../../aprender-contracts", version = "0.70.2", package = "aprender-contracts" } [dev-dependencies] # Used by three of the vendored 0.3.1 example programs. They were regular diff --git a/evidence/crux/0.70.2/prompt-certification-inventory.json b/evidence/crux/0.70.2/prompt-certification-inventory.json new file mode 100644 index 0000000000..93705629cf --- /dev/null +++ b/evidence/crux/0.70.2/prompt-certification-inventory.json @@ -0,0 +1,13 @@ +[ + {"model": "Qwen3.5-2B", + "source": {"repo": "Qwen/Qwen3.5-2B", "revision": "15852e8c16360a2fea060d615a32b45270f8a8fc"}, + "bf16_gguf": "dc11be2ca4519954a3e4b013ef59497b5522209987d67891851113eda89619f1", + "quants": {"Q4_K_M": "aaf42c8b7c3cab2bf3d69c355048d4a0ee9973d48f16c731c0520ee914699223"}, + "thinking": ["on", "off"]}, + {"model": "Qwen3.5-4B", + "source": {"repo": "Qwen/Qwen3.5-4B", "revision": "851bf6e806efd8d0a36b00ddf55e13ccb7b8cd0a"}, + "bf16_gguf": "9e6e2841a75f503ccb330831832fd7861266e187e0dbf149a954219ccb8c197a", + "quants": {"Q4_K_M": "00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4", + "UD-Q4_K_XL": "b252c5610a42ca82d20fe2a12813e9d069eed89292907e26c783eeb0bc961bc7"}, + "thinking": ["on", "off"]} +] diff --git a/evidence/crux/0.70.2/prompt-certification.json b/evidence/crux/0.70.2/prompt-certification.json new file mode 100644 index 0000000000..5f2efe0a08 --- /dev/null +++ b/evidence/crux/0.70.2/prompt-certification.json @@ -0,0 +1,8906 @@ +{ + "schema": "crux-prompt-certification/v1", + "prompts": "scripts/crux_inference_prompts.v2.json", + "prompts_sha256": "cea80df5e785213667e27697fe3d139b99dcba95cfc6e970d76ac005d50fc806", + "apr_commit": "cc4463f7a609f04c0c4fd1ba4229a985ddaeeead", + "inventory_sha256": "b8e4c4626b20b7764041f5b5fe6d8a98166a952a64b3a844207b796b388778e0", + "manifests": { + "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl": "f56747953b33bb2e4adac4bac12be902224a202b2a07c35ede3e47121d178728", + "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl": "6a5657345a6a61ee95ca7a8d7237d7f9748ec8e6d652bdd95be042a586a4e45f" + }, + "admitted": { + "Qwen3.5-2B/Q4_K_M": [], + "Qwen3.5-4B/Q4_K_M": [ + "ctl-2plus2", + "ctl-code-add", + "fact-capital-japan", + "fact-capital-france", + "golden-paris", + "nonascii-water-kanji", + "arith-17x23", + "arith-precedence", + "arith-word-apples", + "hard-not-prime", + "hard-reverse-word", + "hard-count-letters", + "recall-arith-chain", + "code-count-vowels", + "long-primes-10", + "long-countdown-15" + ], + "Qwen3.5-4B/UD-Q4_K_XL": [ + "ctl-2plus2", + "ctl-code-add", + "fact-capital-japan", + "fact-capital-france", + "golden-paris", + "nonascii-water-kanji", + "arith-17x23", + "arith-precedence", + "arith-word-apples", + "hard-not-prime", + "hard-count-letters", + "recall-arith-chain", + "long-primes-10", + "long-countdown-15" + ] + }, + "admitted_by_sha": { + "aaf42c8b7c3cab2bf3d69c355048d4a0ee9973d48f16c731c0520ee914699223": [], + "00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4": [ + "ctl-2plus2", + "ctl-code-add", + "fact-capital-japan", + "fact-capital-france", + "golden-paris", + "nonascii-water-kanji", + "arith-17x23", + "arith-precedence", + "arith-word-apples", + "hard-not-prime", + "hard-reverse-word", + "hard-count-letters", + "recall-arith-chain", + "code-count-vowels", + "long-primes-10", + "long-countdown-15" + ], + "b252c5610a42ca82d20fe2a12813e9d069eed89292907e26c783eeb0bc961bc7": [ + "ctl-2plus2", + "ctl-code-add", + "fact-capital-japan", + "fact-capital-france", + "golden-paris", + "nonascii-water-kanji", + "arith-17x23", + "arith-precedence", + "arith-word-apples", + "hard-not-prime", + "hard-count-letters", + "recall-arith-chain", + "long-primes-10", + "long-countdown-15" + ] + }, + "admitted_by_sha_thinking": { + "aaf42c8b7c3cab2bf3d69c355048d4a0ee9973d48f16c731c0520ee914699223": { + "on": [], + "off": [ + "ctl-2plus2", + "ctl-recall-7", + "ctl-code-add", + "fact-capital-japan", + "fact-capital-france", + "golden-paris", + "arith-17x23", + "arith-precedence", + "arith-word-apples", + "hard-add-4digit", + "hard-not-prime", + "hard-count-letters", + "recall-cats", + "recall-name", + "recall-arith-chain", + "code-is-palindrome", + "code-fib", + "code-count-vowels", + "long-primes-10", + "long-countdown-15" + ] + }, + "00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4": { + "on": [ + "ctl-2plus2", + "ctl-code-add", + "fact-capital-japan", + "fact-capital-france", + "golden-paris", + "nonascii-water-kanji", + "arith-17x23", + "arith-precedence", + "arith-word-apples", + "hard-not-prime", + "hard-reverse-word", + "hard-count-letters", + "recall-arith-chain", + "code-count-vowels", + "long-primes-10", + "long-countdown-15" + ], + "off": [ + "ctl-2plus2", + "ctl-recall-7", + "ctl-code-add", + "fact-capital-japan", + "fact-capital-france", + "golden-paris", + "nonascii-water-kanji", + "arith-17x23", + "arith-precedence", + "arith-word-apples", + "hard-add-4digit", + "hard-not-prime", + "hard-reverse-word", + "hard-count-letters", + "recall-cats", + "recall-name", + "recall-arith-chain", + "code-is-palindrome", + "code-fib", + "code-count-vowels", + "long-primes-10", + "long-countdown-15" + ] + }, + "b252c5610a42ca82d20fe2a12813e9d069eed89292907e26c783eeb0bc961bc7": { + "on": [ + "ctl-2plus2", + "ctl-code-add", + "fact-capital-japan", + "fact-capital-france", + "golden-paris", + "nonascii-water-kanji", + "arith-17x23", + "arith-precedence", + "arith-word-apples", + "hard-not-prime", + "hard-count-letters", + "recall-arith-chain", + "long-primes-10", + "long-countdown-15" + ], + "off": [ + "ctl-2plus2", + "ctl-recall-7", + "ctl-code-add", + "fact-capital-japan", + "fact-capital-france", + "golden-paris", + "nonascii-water-kanji", + "arith-17x23", + "arith-precedence", + "arith-word-apples", + "hard-add-4digit", + "hard-not-prime", + "hard-reverse-word", + "hard-count-letters", + "recall-cats", + "recall-name", + "recall-arith-chain", + "code-is-palindrome", + "code-fib", + "code-count-vowels", + "long-primes-10", + "long-countdown-15" + ] + } + }, + "rejected": { + "Qwen3.5-2B/Q4_K_M": { + "ctl-2plus2": "ggml@bf16 llama.cpp serve run thinking=on on gx10: unclosed think (budget exhausted)", + "ctl-recall-7": "ggml@bf16 llama.cpp serve run thinking=on on gx10: unclosed think (budget exhausted)", + "ctl-code-add": "hf@bf16 thinking=on: no row", + "fact-capital-japan": "ggml@bf16 llama.cpp serve run thinking=on on gx10: unclosed think (budget exhausted)", + "fact-capital-france": "hf@bf16 thinking=on: no row", + "golden-paris": "ggml@bf16 thinking=on: no row", + "fact-water-formula": "ggml@bf16 llama.cpp serve run thinking=on on gx10: unclosed think (budget exhausted)", + "nonascii-water-kanji": "ggml@bf16 thinking=on: no row", + "arith-17x23": "ggml@bf16 llama.cpp serve run thinking=on on gx10: unclosed think (budget exhausted)", + "arith-precedence": "hf@bf16 thinking=on: no row", + "arith-word-apples": "hf@bf16 thinking=on: no row", + "hard-add-4digit": "hf@bf16 thinking=on: no row", + "hard-not-prime": "ggml@bf16 thinking=on: no row", + "hard-reverse-word": "ggml@bf16 thinking=on: no row", + "hard-count-letters": "ggml@bf16 thinking=on: no row", + "recall-cats": "ggml@bf16 thinking=on: no row", + "recall-name": "ggml@bf16 thinking=on: no row", + "recall-arith-chain": "ggml@bf16 thinking=on: no row", + "code-is-palindrome": "ggml@bf16 thinking=on: no row", + "code-fib": "ggml@bf16 thinking=on: no row", + "code-count-vowels": "ggml@bf16 thinking=on: no row", + "long-primes-10": "ggml@bf16 thinking=on: no row", + "long-countdown-15": "ggml@bf16 thinking=on: no row" + }, + "Qwen3.5-4B/Q4_K_M": { + "ctl-recall-7": "ggml@quant llama.cpp serve run thinking=on on gx10: unclosed think (budget exhausted)", + "fact-water-formula": "ggml@bf16 llama.cpp serve run thinking=on on gx10: unclosed think (budget exhausted)", + "hard-add-4digit": "ggml@quant llama.cpp serve run thinking=on on gx10: unclosed think (budget exhausted)", + "recall-cats": "ggml@bf16 llama.cpp serve run thinking=on on gx10: unclosed think (budget exhausted)", + "recall-name": "ggml@quant llama.cpp serve run thinking=on on gx10: unclosed think (budget exhausted)", + "code-is-palindrome": "vllm@bf16 vllm chat thinking=on on gx10: tests_failed: AssertionError", + "code-fib": "ggml@bf16 llama.cpp serve run thinking=on on gx10: unclosed think (budget exhausted)" + }, + "Qwen3.5-4B/UD-Q4_K_XL": { + "ctl-recall-7": "ggml@quant llama.cpp serve run thinking=on on gx10: unclosed think (budget exhausted)", + "fact-water-formula": "ggml@bf16 llama.cpp serve run thinking=on on gx10: unclosed think (budget exhausted)", + "hard-add-4digit": "ggml@quant llama.cpp serve run thinking=on on gx10: unclosed think (budget exhausted)", + "hard-reverse-word": "ggml@quant llama.cpp serve run thinking=on on gx10: mismatch", + "recall-cats": "ggml@bf16 llama.cpp serve run thinking=on on gx10: unclosed think (budget exhausted)", + "recall-name": "ggml@quant llama.cpp serve run thinking=on on gx10: unclosed think (budget exhausted)", + "code-is-palindrome": "vllm@bf16 vllm chat thinking=on on gx10: tests_failed: AssertionError", + "code-fib": "ggml@bf16 llama.cpp serve run thinking=on on gx10: unclosed think (budget exhausted)", + "code-count-vowels": "ggml@quant llama.cpp serve run thinking=on on gx10: unclosed think (budget exhausted)" + } + }, + "uncontrolled": [ + "Qwen3.5-2B/Q4_K_M" + ], + "uncontrolled_detail": [ + { + "model": "Qwen3.5-2B/Q4_K_M", + "sha256": "aaf42c8b7c3cab2bf3d69c355048d4a0ee9973d48f16c731c0520ee914699223", + "thinking": "on", + "verbs": [ + "chat", + "code", + "run", + "serve run", + "serve stream" + ] + } + ], + "think_closure": { + "Qwen3.5-2B/Q4_K_M|ctl-2plus2": { + "ggml@bf16:llama.cpp": "unclosed" + }, + "Qwen3.5-2B/Q4_K_M|ctl-recall-7": { + "ggml@bf16:llama.cpp": "unclosed" + }, + "Qwen3.5-2B/Q4_K_M|ctl-code-add": { + "ggml@bf16:llama.cpp": "closed" + }, + "Qwen3.5-2B/Q4_K_M|fact-capital-japan": { + "ggml@bf16:llama.cpp": "unclosed" + }, + "Qwen3.5-2B/Q4_K_M|fact-capital-france": { + "ggml@bf16:llama.cpp": "closed" + }, + "Qwen3.5-2B/Q4_K_M|fact-water-formula": { + "ggml@bf16:llama.cpp": "unclosed" + }, + "Qwen3.5-2B/Q4_K_M|arith-17x23": { + "ggml@bf16:llama.cpp": "unclosed" + }, + "Qwen3.5-2B/Q4_K_M|arith-precedence": { + "ggml@bf16:llama.cpp": "closed" + }, + "Qwen3.5-2B/Q4_K_M|arith-word-apples": { + "ggml@bf16:llama.cpp": "closed" + }, + "Qwen3.5-2B/Q4_K_M|hard-add-4digit": { + "ggml@bf16:llama.cpp": "closed" + }, + "Qwen3.5-4B/Q4_K_M|ctl-2plus2": { + "ggml@bf16:llama.cpp": "closed", + "hf@bf16:hf": "closed", + "vllm@bf16:vllm": "closed", + "ggml@quant:llama.cpp": "closed" + }, + "Qwen3.5-4B/Q4_K_M|ctl-recall-7": { + "ggml@bf16:llama.cpp": "closed", + "hf@bf16:hf": "closed", + "vllm@bf16:vllm": "closed", + "ggml@quant:llama.cpp": "unclosed" + }, + "Qwen3.5-4B/Q4_K_M|ctl-code-add": { + "ggml@bf16:llama.cpp": "closed", + "hf@bf16:hf": "closed", + "vllm@bf16:vllm": "closed", + "ggml@quant:llama.cpp": "closed" + }, + "Qwen3.5-4B/Q4_K_M|fact-capital-japan": { + "ggml@bf16:llama.cpp": "closed", + "hf@bf16:hf": "closed", + "vllm@bf16:vllm": "closed", + "ggml@quant:llama.cpp": "closed" + }, + "Qwen3.5-4B/Q4_K_M|fact-capital-france": { + "ggml@bf16:llama.cpp": "closed", + "hf@bf16:hf": "closed", + "vllm@bf16:vllm": "closed", + "ggml@quant:llama.cpp": "closed" + }, + "Qwen3.5-4B/Q4_K_M|golden-paris": { + "ggml@bf16:llama.cpp": "closed", + "hf@bf16:hf": "closed", + "vllm@bf16:vllm": "closed", + "ggml@quant:llama.cpp": "closed" + }, + "Qwen3.5-4B/Q4_K_M|fact-water-formula": { + "ggml@bf16:llama.cpp": "unclosed", + "hf@bf16:hf": "unclosed", + "vllm@bf16:vllm": "unclosed", + "ggml@quant:llama.cpp": "closed" + }, + "Qwen3.5-4B/Q4_K_M|nonascii-water-kanji": { + "ggml@bf16:llama.cpp": "closed", + "hf@bf16:hf": "closed", + "vllm@bf16:vllm": "closed", + "ggml@quant:llama.cpp": "closed" + }, + "Qwen3.5-4B/Q4_K_M|arith-17x23": { + "ggml@bf16:llama.cpp": "closed", + "hf@bf16:hf": "closed", + "vllm@bf16:vllm": "closed", + "ggml@quant:llama.cpp": "closed" + }, + "Qwen3.5-4B/Q4_K_M|arith-precedence": { + "ggml@bf16:llama.cpp": "closed", + "hf@bf16:hf": "closed", + "vllm@bf16:vllm": "closed", + "ggml@quant:llama.cpp": "closed" + }, + "Qwen3.5-4B/Q4_K_M|arith-word-apples": { + "ggml@bf16:llama.cpp": "closed", + "hf@bf16:hf": "closed", + "vllm@bf16:vllm": "closed", + "ggml@quant:llama.cpp": "closed" + }, + "Qwen3.5-4B/Q4_K_M|hard-add-4digit": { + "ggml@bf16:llama.cpp": "closed", + "hf@bf16:hf": "closed", + "vllm@bf16:vllm": "closed", + "ggml@quant:llama.cpp": "unclosed" + }, + "Qwen3.5-4B/Q4_K_M|hard-not-prime": { + "ggml@bf16:llama.cpp": "closed", + "hf@bf16:hf": "closed", + "vllm@bf16:vllm": "closed", + "ggml@quant:llama.cpp": "closed" + }, + "Qwen3.5-4B/Q4_K_M|hard-reverse-word": { + "ggml@bf16:llama.cpp": "closed", + "hf@bf16:hf": "closed", + "vllm@bf16:vllm": "closed", + "ggml@quant:llama.cpp": "closed" + }, + "Qwen3.5-4B/Q4_K_M|hard-count-letters": { + "ggml@bf16:llama.cpp": "closed", + "hf@bf16:hf": "closed", + "vllm@bf16:vllm": "closed", + "ggml@quant:llama.cpp": "closed" + }, + "Qwen3.5-4B/Q4_K_M|recall-cats": { + "ggml@bf16:llama.cpp": "unclosed", + "hf@bf16:hf": "closed", + "vllm@bf16:vllm": "closed", + "ggml@quant:llama.cpp": "unclosed" + }, + "Qwen3.5-4B/Q4_K_M|recall-name": { + "ggml@bf16:llama.cpp": "closed", + "hf@bf16:hf": "closed", + "vllm@bf16:vllm": "closed", + "ggml@quant:llama.cpp": "unclosed" + }, + "Qwen3.5-4B/Q4_K_M|recall-arith-chain": { + "ggml@bf16:llama.cpp": "closed", + "hf@bf16:hf": "closed", + "vllm@bf16:vllm": "closed", + "ggml@quant:llama.cpp": "closed" + }, + "Qwen3.5-4B/Q4_K_M|code-is-palindrome": { + "ggml@bf16:llama.cpp": "closed", + "hf@bf16:hf": "closed", + "vllm@bf16:vllm": "closed", + "ggml@quant:llama.cpp": "unclosed" + }, + "Qwen3.5-4B/Q4_K_M|code-fib": { + "ggml@bf16:llama.cpp": "unclosed", + "hf@bf16:hf": "unclosed", + "vllm@bf16:vllm": "unclosed", + "ggml@quant:llama.cpp": "unclosed" + }, + "Qwen3.5-4B/Q4_K_M|code-count-vowels": { + "ggml@bf16:llama.cpp": "closed", + "hf@bf16:hf": "closed", + "vllm@bf16:vllm": "closed", + "ggml@quant:llama.cpp": "closed" + }, + "Qwen3.5-4B/Q4_K_M|long-primes-10": { + "ggml@bf16:llama.cpp": "closed", + "hf@bf16:hf": "closed", + "vllm@bf16:vllm": "closed", + "ggml@quant:llama.cpp": "closed" + }, + "Qwen3.5-4B/Q4_K_M|long-countdown-15": { + "ggml@bf16:llama.cpp": "closed", + "hf@bf16:hf": "closed", + "vllm@bf16:vllm": "closed", + "ggml@quant:llama.cpp": "closed" + }, + "Qwen3.5-4B/UD-Q4_K_XL|ctl-2plus2": { + "ggml@bf16:llama.cpp": "closed", + "hf@bf16:hf": "closed", + "vllm@bf16:vllm": "closed", + "ggml@quant:llama.cpp": "closed" + }, + "Qwen3.5-4B/UD-Q4_K_XL|ctl-recall-7": { + "ggml@bf16:llama.cpp": "closed", + "hf@bf16:hf": "closed", + "vllm@bf16:vllm": "closed", + "ggml@quant:llama.cpp": "unclosed" + }, + "Qwen3.5-4B/UD-Q4_K_XL|ctl-code-add": { + "ggml@bf16:llama.cpp": "closed", + "hf@bf16:hf": "closed", + "vllm@bf16:vllm": "closed", + "ggml@quant:llama.cpp": "closed" + }, + "Qwen3.5-4B/UD-Q4_K_XL|fact-capital-japan": { + "ggml@bf16:llama.cpp": "closed", + "hf@bf16:hf": "closed", + "vllm@bf16:vllm": "closed", + "ggml@quant:llama.cpp": "closed" + }, + "Qwen3.5-4B/UD-Q4_K_XL|fact-capital-france": { + "ggml@bf16:llama.cpp": "closed", + "hf@bf16:hf": "closed", + "vllm@bf16:vllm": "closed", + "ggml@quant:llama.cpp": "closed" + }, + "Qwen3.5-4B/UD-Q4_K_XL|golden-paris": { + "ggml@bf16:llama.cpp": "closed", + "hf@bf16:hf": "closed", + "vllm@bf16:vllm": "closed", + "ggml@quant:llama.cpp": "closed" + }, + "Qwen3.5-4B/UD-Q4_K_XL|fact-water-formula": { + "ggml@bf16:llama.cpp": "unclosed", + "hf@bf16:hf": "unclosed", + "vllm@bf16:vllm": "unclosed", + "ggml@quant:llama.cpp": "unclosed" + }, + "Qwen3.5-4B/UD-Q4_K_XL|nonascii-water-kanji": { + "ggml@bf16:llama.cpp": "closed", + "hf@bf16:hf": "closed", + "vllm@bf16:vllm": "closed", + "ggml@quant:llama.cpp": "closed" + }, + "Qwen3.5-4B/UD-Q4_K_XL|arith-17x23": { + "ggml@bf16:llama.cpp": "closed", + "hf@bf16:hf": "closed", + "vllm@bf16:vllm": "closed", + "ggml@quant:llama.cpp": "closed" + }, + "Qwen3.5-4B/UD-Q4_K_XL|arith-precedence": { + "ggml@bf16:llama.cpp": "closed", + "hf@bf16:hf": "closed", + "vllm@bf16:vllm": "closed", + "ggml@quant:llama.cpp": "closed" + }, + "Qwen3.5-4B/UD-Q4_K_XL|arith-word-apples": { + "ggml@bf16:llama.cpp": "closed", + "hf@bf16:hf": "closed", + "vllm@bf16:vllm": "closed", + "ggml@quant:llama.cpp": "closed" + }, + "Qwen3.5-4B/UD-Q4_K_XL|hard-add-4digit": { + "ggml@bf16:llama.cpp": "closed", + "hf@bf16:hf": "closed", + "vllm@bf16:vllm": "closed", + "ggml@quant:llama.cpp": "unclosed" + }, + "Qwen3.5-4B/UD-Q4_K_XL|hard-not-prime": { + "ggml@bf16:llama.cpp": "closed", + "hf@bf16:hf": "closed", + "vllm@bf16:vllm": "closed", + "ggml@quant:llama.cpp": "closed" + }, + "Qwen3.5-4B/UD-Q4_K_XL|hard-reverse-word": { + "ggml@bf16:llama.cpp": "closed", + "hf@bf16:hf": "closed", + "vllm@bf16:vllm": "closed", + "ggml@quant:llama.cpp": "closed" + }, + "Qwen3.5-4B/UD-Q4_K_XL|hard-count-letters": { + "ggml@bf16:llama.cpp": "closed", + "hf@bf16:hf": "closed", + "vllm@bf16:vllm": "closed", + "ggml@quant:llama.cpp": "closed" + }, + "Qwen3.5-4B/UD-Q4_K_XL|recall-cats": { + "ggml@bf16:llama.cpp": "unclosed", + "hf@bf16:hf": "closed", + "vllm@bf16:vllm": "closed", + "ggml@quant:llama.cpp": "unclosed" + }, + "Qwen3.5-4B/UD-Q4_K_XL|recall-name": { + "ggml@bf16:llama.cpp": "closed", + "hf@bf16:hf": "closed", + "vllm@bf16:vllm": "closed", + "ggml@quant:llama.cpp": "unclosed" + }, + "Qwen3.5-4B/UD-Q4_K_XL|recall-arith-chain": { + "ggml@bf16:llama.cpp": "closed", + "hf@bf16:hf": "closed", + "vllm@bf16:vllm": "closed", + "ggml@quant:llama.cpp": "closed" + }, + "Qwen3.5-4B/UD-Q4_K_XL|code-is-palindrome": { + "ggml@bf16:llama.cpp": "closed", + "hf@bf16:hf": "closed", + "vllm@bf16:vllm": "closed", + "ggml@quant:llama.cpp": "unclosed" + }, + "Qwen3.5-4B/UD-Q4_K_XL|code-fib": { + "ggml@bf16:llama.cpp": "unclosed", + "hf@bf16:hf": "unclosed", + "vllm@bf16:vllm": "unclosed", + "ggml@quant:llama.cpp": "unclosed" + }, + "Qwen3.5-4B/UD-Q4_K_XL|code-count-vowels": { + "ggml@bf16:llama.cpp": "closed", + "hf@bf16:hf": "closed", + "vllm@bf16:vllm": "closed", + "ggml@quant:llama.cpp": "unclosed" + }, + "Qwen3.5-4B/UD-Q4_K_XL|long-primes-10": { + "ggml@bf16:llama.cpp": "closed", + "hf@bf16:hf": "closed", + "vllm@bf16:vllm": "closed", + "ggml@quant:llama.cpp": "closed" + }, + "Qwen3.5-4B/UD-Q4_K_XL|long-countdown-15": { + "ggml@bf16:llama.cpp": "closed", + "hf@bf16:hf": "closed", + "vllm@bf16:vllm": "closed", + "ggml@quant:llama.cpp": "closed" + } + }, + "cells": [ + { + "leg": "ggml@bf16", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": false, + "why": "unclosed think (budget exhausted)", + "extracted": null, + "think": "unclosed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L1", + "prompt_id": "ctl-2plus2", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": false, + "why": "unclosed think (budget exhausted)", + "extracted": null, + "think": "unclosed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L263", + "prompt_id": "ctl-2plus2", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "ctl-2plus2", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "ctl-2plus2", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "ctl-2plus2", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "4", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L11", + "prompt_id": "ctl-2plus2", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "4", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L1", + "prompt_id": "ctl-2plus2", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "4", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L23", + "prompt_id": "ctl-2plus2", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "4", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L33", + "prompt_id": "ctl-2plus2", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": false, + "why": "unclosed think (budget exhausted)", + "extracted": null, + "think": "unclosed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L2", + "prompt_id": "ctl-recall-7", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": false, + "why": "unclosed think (budget exhausted)", + "extracted": null, + "think": "unclosed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L264", + "prompt_id": "ctl-recall-7", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "ctl-recall-7", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "ctl-recall-7", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "ctl-recall-7", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "7", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L12", + "prompt_id": "ctl-recall-7", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "7", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L2", + "prompt_id": "ctl-recall-7", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "7", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L24", + "prompt_id": "ctl-recall-7", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "7", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L34", + "prompt_id": "ctl-recall-7", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "def add(a, b):\n return a + b\n", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L3", + "prompt_id": "ctl-code-add", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "ctl-code-add", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "ctl-code-add", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "ctl-code-add", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "def add(a, b):\n return a + b\n", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L13", + "prompt_id": "ctl-code-add", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "def add(a, b):\n return a + b\n", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L3", + "prompt_id": "ctl-code-add", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "def add(a, b):\n return a + b\n", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L25", + "prompt_id": "ctl-code-add", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "def add(a, b):\n return a + b\n", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L35", + "prompt_id": "ctl-code-add", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": false, + "why": "unclosed think (budget exhausted)", + "extracted": null, + "think": "unclosed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L4", + "prompt_id": "fact-capital-japan", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "fact-capital-japan", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "fact-capital-japan", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "fact-capital-japan", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "Tokyo", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L14", + "prompt_id": "fact-capital-japan", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "Tokyo", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L4", + "prompt_id": "fact-capital-japan", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "Tokyo", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L26", + "prompt_id": "fact-capital-japan", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "Tokyo", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L36", + "prompt_id": "fact-capital-japan", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "Paris", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L5", + "prompt_id": "fact-capital-france", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "fact-capital-france", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "fact-capital-france", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "fact-capital-france", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "Paris", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L15", + "prompt_id": "fact-capital-france", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "Paris", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L5", + "prompt_id": "fact-capital-france", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "Paris", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L27", + "prompt_id": "fact-capital-france", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "Paris", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L37", + "prompt_id": "fact-capital-france", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "golden-paris", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "golden-paris", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "golden-paris", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "golden-paris", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "Paris", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L16", + "prompt_id": "golden-paris", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "Paris", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L6", + "prompt_id": "golden-paris", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "Paris", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L28", + "prompt_id": "golden-paris", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "Paris", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L38", + "prompt_id": "golden-paris", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": false, + "why": "unclosed think (budget exhausted)", + "extracted": null, + "think": "unclosed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L6", + "prompt_id": "fact-water-formula", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "fact-water-formula", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "fact-water-formula", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "fact-water-formula", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": false, + "why": "mismatch", + "extracted": "H_2O", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L17", + "prompt_id": "fact-water-formula", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": false, + "why": "mismatch", + "extracted": "H_2O", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L7", + "prompt_id": "fact-water-formula", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "lambda", + "correct": false, + "why": "mismatch", + "extracted": "H_2O", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L29", + "prompt_id": "fact-water-formula", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": false, + "why": "mismatch", + "extracted": "H₂O", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L39", + "prompt_id": "fact-water-formula", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "nonascii-water-kanji", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "nonascii-water-kanji", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "nonascii-water-kanji", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "nonascii-water-kanji", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "nonascii-water-kanji", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "nonascii-water-kanji", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "nonascii-water-kanji", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "nonascii-water-kanji", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": false, + "why": "unclosed think (budget exhausted)", + "extracted": null, + "think": "unclosed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L7", + "prompt_id": "arith-17x23", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "arith-17x23", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "arith-17x23", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "arith-17x23", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "391", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L18", + "prompt_id": "arith-17x23", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "391", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L8", + "prompt_id": "arith-17x23", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "391", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L30", + "prompt_id": "arith-17x23", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "391", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L40", + "prompt_id": "arith-17x23", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "23", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L8", + "prompt_id": "arith-precedence", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "arith-precedence", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "arith-precedence", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "arith-precedence", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "23", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L19", + "prompt_id": "arith-precedence", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "23", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L9", + "prompt_id": "arith-precedence", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "23", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L31", + "prompt_id": "arith-precedence", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "23", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L41", + "prompt_id": "arith-precedence", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "56", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L9", + "prompt_id": "arith-word-apples", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "arith-word-apples", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "arith-word-apples", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "arith-word-apples", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "56", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L20", + "prompt_id": "arith-word-apples", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "56", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L10", + "prompt_id": "arith-word-apples", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "56", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L32", + "prompt_id": "arith-word-apples", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "56", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L42", + "prompt_id": "arith-word-apples", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "8613", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L10", + "prompt_id": "hard-add-4digit", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "hard-add-4digit", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "hard-add-4digit", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "hard-add-4digit", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "8613", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L21", + "prompt_id": "hard-add-4digit", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "8613", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L11", + "prompt_id": "hard-add-4digit", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "8613", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L33", + "prompt_id": "hard-add-4digit", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "8613", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L43", + "prompt_id": "hard-add-4digit", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "hard-not-prime", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "hard-not-prime", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "hard-not-prime", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "hard-not-prime", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "15", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L22", + "prompt_id": "hard-not-prime", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "15", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L12", + "prompt_id": "hard-not-prime", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "15", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L34", + "prompt_id": "hard-not-prime", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "15", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L44", + "prompt_id": "hard-not-prime", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "hard-reverse-word", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "hard-reverse-word", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "hard-reverse-word", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "hard-reverse-word", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": false, + "why": "mismatch", + "extracted": "tnipal", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L23", + "prompt_id": "hard-reverse-word", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": false, + "why": "mismatch", + "extracted": "tnipal", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L13", + "prompt_id": "hard-reverse-word", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "lambda", + "correct": false, + "why": "mismatch", + "extracted": "tnipal", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L35", + "prompt_id": "hard-reverse-word", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": false, + "why": "mismatch", + "extracted": "tenap", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L45", + "prompt_id": "hard-reverse-word", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "hard-count-letters", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "hard-count-letters", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "hard-count-letters", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "hard-count-letters", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "3", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L24", + "prompt_id": "hard-count-letters", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "3", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L14", + "prompt_id": "hard-count-letters", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "3", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L36", + "prompt_id": "hard-count-letters", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "3", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L46", + "prompt_id": "hard-count-letters", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "recall-cats", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "recall-cats", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "recall-cats", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "recall-cats", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "5", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L25", + "prompt_id": "recall-cats", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "5", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L15", + "prompt_id": "recall-cats", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "5", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L37", + "prompt_id": "recall-cats", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "5", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L47", + "prompt_id": "recall-cats", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "recall-name", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "recall-name", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "recall-name", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "recall-name", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "Ada", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L26", + "prompt_id": "recall-name", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "Ada", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L16", + "prompt_id": "recall-name", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "Ada", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L38", + "prompt_id": "recall-name", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "Ada", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L48", + "prompt_id": "recall-name", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "recall-arith-chain", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "recall-arith-chain", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "recall-arith-chain", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "recall-arith-chain", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "7", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L27", + "prompt_id": "recall-arith-chain", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "7", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L17", + "prompt_id": "recall-arith-chain", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "7", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L39", + "prompt_id": "recall-arith-chain", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "7", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L49", + "prompt_id": "recall-arith-chain", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "code-is-palindrome", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "code-is-palindrome", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "code-is-palindrome", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "code-is-palindrome", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "def is_palindrome(s):\n s = s.lower()\n return s == s[::-1]\n", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L28", + "prompt_id": "code-is-palindrome", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "def is_palindrome(s):\n s = s.lower()\n return s == s[::-1]\n", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L18", + "prompt_id": "code-is-palindrome", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "def is_palindrome(s):\n s = s.lower()\n return s == s[::-1]\n", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L40", + "prompt_id": "code-is-palindrome", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "def is_palindrome(s):\n s = s.lower()\n return s == s[::-1]\n", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L50", + "prompt_id": "code-is-palindrome", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "code-fib", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "code-fib", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "code-fib", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "code-fib", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "def fib(n):\n if n < 0:\n raise ValueError(\"n must be a non-negative integer\")\n if n == 0:\n return 0\n if n == 1:\n return 1\n a, b = 0, 1\n for _ in range(2, n + 1):\n a, b = b, a + b\n return b\n", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L29", + "prompt_id": "code-fib", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "def fib(n):\n if n < 0:\n raise ValueError(\"n must be a non-negative integer\")\n if n == 0:\n return 0\n if n == 1:\n return 1\n a, b = 0, 1\n for _ in range(2, n + 1):\n a, b = b, a + b\n return b\n", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L19", + "prompt_id": "code-fib", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "def fib(n):\n if n < 0:\n raise ValueError(\"n must be a non-negative integer\")\n if n == 0:\n return 0\n if n == 1:\n return 1\n a, b = 0, 1\n for _ in range(2, n + 1):\n a, b = b, a + b\n return b\n", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L41", + "prompt_id": "code-fib", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "def fib(n):\n if n < 0:\n raise ValueError(\"n must be a non-negative integer\")\n if n == 0:\n return 0\n if n == 1:\n return 1\n a, b = 0, 1\n for _ in range(2, n + 1):\n a, b = b, a + b\n return b\n", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L51", + "prompt_id": "code-fib", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "code-count-vowels", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "code-count-vowels", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "code-count-vowels", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "code-count-vowels", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "def count_vowels(s):\n vowels = set('aeiouAEIOU')\n return sum(1 for char in s if char.lower() in vowels)\n", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L30", + "prompt_id": "code-count-vowels", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "def count_vowels(s):\n vowels = set('aeiouAEIOU')\n return sum(1 for char in s if char.lower() in vowels)\n", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L20", + "prompt_id": "code-count-vowels", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "def count_vowels(s):\n vowels = set('aeiouAEIOU')\n return sum(1 for char in s if char.lower() in vowels)\n", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L42", + "prompt_id": "code-count-vowels", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "def count_vowels(s):\n vowels = set('aeiouAEIOU')\n return sum(1 for char in s if char in vowels)\n", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L52", + "prompt_id": "code-count-vowels", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "long-primes-10", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "long-primes-10", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "long-primes-10", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "long-primes-10", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": [ + "2", + "3", + "5", + "7", + "11", + "13", + "17", + "19", + "23", + "29" + ], + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L31", + "prompt_id": "long-primes-10", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": [ + "2", + "3", + "5", + "7", + "11", + "13", + "17", + "19", + "23", + "29" + ], + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L21", + "prompt_id": "long-primes-10", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": [ + "2", + "3", + "5", + "7", + "11", + "13", + "17", + "19", + "23", + "29" + ], + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L43", + "prompt_id": "long-primes-10", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": [ + "2", + "3", + "5", + "7", + "11", + "13", + "17", + "19", + "23", + "29" + ], + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L53", + "prompt_id": "long-primes-10", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "long-countdown-15", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "long-countdown-15", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "long-countdown-15", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "long-countdown-15", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": [ + "15", + "14", + "13", + "12", + "11", + "10", + "9", + "8", + "7", + "6", + "5", + "4", + "3", + "2", + "1" + ], + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L32", + "prompt_id": "long-countdown-15", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": [ + "15", + "14", + "13", + "12", + "11", + "10", + "9", + "8", + "7", + "6", + "5", + "4", + "3", + "2", + "1" + ], + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L22", + "prompt_id": "long-countdown-15", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": [ + "15", + "14", + "13", + "12", + "11", + "10", + "9", + "8", + "7", + "6", + "5", + "4", + "3", + "2", + "1" + ], + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L44", + "prompt_id": "long-countdown-15", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": [ + "15", + "14", + "13", + "12", + "11", + "10", + "9", + "8", + "7", + "6", + "5", + "4", + "3", + "2", + "1" + ], + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L54", + "prompt_id": "long-countdown-15", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "4", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L121", + "prompt_id": "ctl-2plus2", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "engine": "hf", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "4", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L187", + "prompt_id": "ctl-2plus2", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "4", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L235", + "prompt_id": "ctl-2plus2", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "4", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L143", + "prompt_id": "ctl-2plus2", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "4", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L55", + "prompt_id": "ctl-2plus2", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "4", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L45", + "prompt_id": "ctl-2plus2", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "4", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L210", + "prompt_id": "ctl-2plus2", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "4", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L77", + "prompt_id": "ctl-2plus2", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "7", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L122", + "prompt_id": "ctl-recall-7", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "engine": "hf", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "7", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L188", + "prompt_id": "ctl-recall-7", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "7", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L236", + "prompt_id": "ctl-recall-7", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": false, + "why": "unclosed think (budget exhausted)", + "extracted": null, + "think": "unclosed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L144", + "prompt_id": "ctl-recall-7", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "7", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L56", + "prompt_id": "ctl-recall-7", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "7", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L46", + "prompt_id": "ctl-recall-7", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "7", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L211", + "prompt_id": "ctl-recall-7", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "7", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L78", + "prompt_id": "ctl-recall-7", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "def add(a, b):\n return a + b\n", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L127", + "prompt_id": "ctl-code-add", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "engine": "hf", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "def add(a, b):\n return a + b\n", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L189", + "prompt_id": "ctl-code-add", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "def add(a, b):\n return a + b\n", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L237", + "prompt_id": "ctl-code-add", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "def add(a, b):\n return a + b\n", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L145", + "prompt_id": "ctl-code-add", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "def add(a, b):\n return a + b\n", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L57", + "prompt_id": "ctl-code-add", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "def add(a, b):\n return a + b\n", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L47", + "prompt_id": "ctl-code-add", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "def add(a, b):\n return a + b\n", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L212", + "prompt_id": "ctl-code-add", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "def add(a, b):\n return a + b\n", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L79", + "prompt_id": "ctl-code-add", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "Tokyo", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L123", + "prompt_id": "fact-capital-japan", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "engine": "hf", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "Tokyo", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L190", + "prompt_id": "fact-capital-japan", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "Tokyo", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L238", + "prompt_id": "fact-capital-japan", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "Tokyo", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L146", + "prompt_id": "fact-capital-japan", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "Tokyo", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L58", + "prompt_id": "fact-capital-japan", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "Tokyo", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L48", + "prompt_id": "fact-capital-japan", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "Tokyo", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L213", + "prompt_id": "fact-capital-japan", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "Tokyo", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L80", + "prompt_id": "fact-capital-japan", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "Paris", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L124", + "prompt_id": "fact-capital-france", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "engine": "hf", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "Paris", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L191", + "prompt_id": "fact-capital-france", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "Paris", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L239", + "prompt_id": "fact-capital-france", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "Paris", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L147", + "prompt_id": "fact-capital-france", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "Paris", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L59", + "prompt_id": "fact-capital-france", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "Paris", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L49", + "prompt_id": "fact-capital-france", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "Paris", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L214", + "prompt_id": "fact-capital-france", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "Paris", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L81", + "prompt_id": "fact-capital-france", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "Paris", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L128", + "prompt_id": "golden-paris", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "engine": "hf", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "Paris", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L192", + "prompt_id": "golden-paris", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "Paris", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L240", + "prompt_id": "golden-paris", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "Paris", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L148", + "prompt_id": "golden-paris", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "Paris", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L60", + "prompt_id": "golden-paris", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "Paris", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L50", + "prompt_id": "golden-paris", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "Paris", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L215", + "prompt_id": "golden-paris", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "Paris", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L82", + "prompt_id": "golden-paris", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": false, + "why": "unclosed think (budget exhausted)", + "extracted": null, + "think": "unclosed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L125", + "prompt_id": "fact-water-formula", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "engine": "hf", + "verb": "chat", + "host": "gx10", + "correct": false, + "why": "unclosed think (budget exhausted)", + "extracted": null, + "think": "unclosed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L193", + "prompt_id": "fact-water-formula", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": false, + "why": "unclosed think (budget exhausted)", + "extracted": null, + "think": "unclosed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L241", + "prompt_id": "fact-water-formula", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": false, + "why": "mismatch", + "extracted": "H_2O", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L149", + "prompt_id": "fact-water-formula", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": false, + "why": "mismatch", + "extracted": "H₂O", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L61", + "prompt_id": "fact-water-formula", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": false, + "why": "mismatch", + "extracted": "H₂O", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L51", + "prompt_id": "fact-water-formula", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": false, + "why": "mismatch", + "extracted": "H₂O", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L216", + "prompt_id": "fact-water-formula", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": false, + "why": "mismatch", + "extracted": "H2O", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L83", + "prompt_id": "fact-water-formula", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "水", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L233", + "prompt_id": "nonascii-water-kanji", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "engine": "hf", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "水", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L194", + "prompt_id": "nonascii-water-kanji", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "水", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L242", + "prompt_id": "nonascii-water-kanji", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "水", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L258", + "prompt_id": "nonascii-water-kanji", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "水", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L234", + "prompt_id": "nonascii-water-kanji", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "水", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L262", + "prompt_id": "nonascii-water-kanji", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "水", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L217", + "prompt_id": "nonascii-water-kanji", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "水", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L259", + "prompt_id": "nonascii-water-kanji", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "391", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L126", + "prompt_id": "arith-17x23", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "engine": "hf", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "391", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L195", + "prompt_id": "arith-17x23", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "391", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L243", + "prompt_id": "arith-17x23", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "391", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L150", + "prompt_id": "arith-17x23", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "391", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L62", + "prompt_id": "arith-17x23", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "391", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L52", + "prompt_id": "arith-17x23", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "391", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L218", + "prompt_id": "arith-17x23", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "391", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L84", + "prompt_id": "arith-17x23", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "23", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L129", + "prompt_id": "arith-precedence", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "engine": "hf", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "23", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L196", + "prompt_id": "arith-precedence", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "23", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L244", + "prompt_id": "arith-precedence", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "23", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L151", + "prompt_id": "arith-precedence", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "23", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L63", + "prompt_id": "arith-precedence", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "23", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L53", + "prompt_id": "arith-precedence", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "23", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L219", + "prompt_id": "arith-precedence", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "23", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L85", + "prompt_id": "arith-precedence", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "56", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L130", + "prompt_id": "arith-word-apples", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "engine": "hf", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "56", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L197", + "prompt_id": "arith-word-apples", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "56", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L245", + "prompt_id": "arith-word-apples", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "56", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L152", + "prompt_id": "arith-word-apples", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "56", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L64", + "prompt_id": "arith-word-apples", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "56", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L54", + "prompt_id": "arith-word-apples", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "56", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L220", + "prompt_id": "arith-word-apples", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "56", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L86", + "prompt_id": "arith-word-apples", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "8613", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L131", + "prompt_id": "hard-add-4digit", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "engine": "hf", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "8613", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L198", + "prompt_id": "hard-add-4digit", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "8613", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L246", + "prompt_id": "hard-add-4digit", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": false, + "why": "unclosed think (budget exhausted)", + "extracted": null, + "think": "unclosed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L153", + "prompt_id": "hard-add-4digit", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "8613", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L65", + "prompt_id": "hard-add-4digit", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "8613", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L55", + "prompt_id": "hard-add-4digit", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "8613", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L221", + "prompt_id": "hard-add-4digit", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "8613", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L87", + "prompt_id": "hard-add-4digit", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "15", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L132", + "prompt_id": "hard-not-prime", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "engine": "hf", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "15", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L199", + "prompt_id": "hard-not-prime", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "15", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L247", + "prompt_id": "hard-not-prime", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "15", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L154", + "prompt_id": "hard-not-prime", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "15", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L66", + "prompt_id": "hard-not-prime", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "15", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L56", + "prompt_id": "hard-not-prime", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "15", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L222", + "prompt_id": "hard-not-prime", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "15", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L88", + "prompt_id": "hard-not-prime", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "tenalp", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L133", + "prompt_id": "hard-reverse-word", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "engine": "hf", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "tenalp", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L200", + "prompt_id": "hard-reverse-word", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "tenalp", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L248", + "prompt_id": "hard-reverse-word", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "tenalp", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L155", + "prompt_id": "hard-reverse-word", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "tenalp", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L67", + "prompt_id": "hard-reverse-word", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "tenalp", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L57", + "prompt_id": "hard-reverse-word", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "tenalp", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L223", + "prompt_id": "hard-reverse-word", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "tenalp", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L89", + "prompt_id": "hard-reverse-word", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "3", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L134", + "prompt_id": "hard-count-letters", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "engine": "hf", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "3", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L201", + "prompt_id": "hard-count-letters", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "3", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L249", + "prompt_id": "hard-count-letters", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "3", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L156", + "prompt_id": "hard-count-letters", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "3", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L68", + "prompt_id": "hard-count-letters", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "3", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L58", + "prompt_id": "hard-count-letters", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "3", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L224", + "prompt_id": "hard-count-letters", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "3", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L90", + "prompt_id": "hard-count-letters", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": false, + "why": "unclosed think (budget exhausted)", + "extracted": null, + "think": "unclosed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L135", + "prompt_id": "recall-cats", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "engine": "hf", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "5", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L202", + "prompt_id": "recall-cats", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "5", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L250", + "prompt_id": "recall-cats", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": false, + "why": "unclosed think (budget exhausted)", + "extracted": null, + "think": "unclosed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L157", + "prompt_id": "recall-cats", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "5", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L69", + "prompt_id": "recall-cats", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "5", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L59", + "prompt_id": "recall-cats", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "5", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L225", + "prompt_id": "recall-cats", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "5", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L91", + "prompt_id": "recall-cats", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "Ada", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L136", + "prompt_id": "recall-name", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "engine": "hf", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "Ada", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L203", + "prompt_id": "recall-name", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "Ada", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L251", + "prompt_id": "recall-name", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": false, + "why": "unclosed think (budget exhausted)", + "extracted": null, + "think": "unclosed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L158", + "prompt_id": "recall-name", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "Ada", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L70", + "prompt_id": "recall-name", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "Ada", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L60", + "prompt_id": "recall-name", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "Ada", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L226", + "prompt_id": "recall-name", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "Ada", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L92", + "prompt_id": "recall-name", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "7", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L137", + "prompt_id": "recall-arith-chain", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "engine": "hf", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "7", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L204", + "prompt_id": "recall-arith-chain", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "7", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L252", + "prompt_id": "recall-arith-chain", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "7", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L159", + "prompt_id": "recall-arith-chain", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "7", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L71", + "prompt_id": "recall-arith-chain", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "7", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L61", + "prompt_id": "recall-arith-chain", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "7", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L227", + "prompt_id": "recall-arith-chain", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "7", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L93", + "prompt_id": "recall-arith-chain", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "def is_palindrome(s):\n s = s.lower()\n return s == s[::-1]\n", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L138", + "prompt_id": "code-is-palindrome", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "engine": "hf", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "def is_palindrome(s):\n return s.lower() == s[::-1].lower()\n", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L205", + "prompt_id": "code-is-palindrome", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": false, + "why": "tests_failed: AssertionError", + "extracted": "def is_palindrome(s):\n return s.lower() == s[::-1]\n", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L253", + "prompt_id": "code-is-palindrome", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": false, + "why": "unclosed think (budget exhausted)", + "extracted": null, + "think": "unclosed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L160", + "prompt_id": "code-is-palindrome", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "def is_palindrome(s):\n cleaned = ''.join(char.lower() for char in s if char.isalnum())\n return cleaned == cleaned[::-1]\n", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L72", + "prompt_id": "code-is-palindrome", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "def is_palindrome(s):\n cleaned = ''.join(char.lower() for char in s if char.isalnum())\n return cleaned == cleaned[::-1]\n", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L62", + "prompt_id": "code-is-palindrome", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "def is_palindrome(s):\n cleaned = ''.join(char.lower() for char in s if char.isalnum())\n return cleaned == cleaned[::-1]\n", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L228", + "prompt_id": "code-is-palindrome", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "def is_palindrome(s):\n cleaned = ''.join(char.lower() for char in s if char.isalnum())\n return cleaned == cleaned[::-1]\n", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L94", + "prompt_id": "code-is-palindrome", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": false, + "why": "unclosed think (budget exhausted)", + "extracted": null, + "think": "unclosed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L139", + "prompt_id": "code-fib", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "engine": "hf", + "verb": "chat", + "host": "gx10", + "correct": false, + "why": "unclosed think (budget exhausted)", + "extracted": null, + "think": "unclosed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L206", + "prompt_id": "code-fib", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": false, + "why": "unclosed think (budget exhausted)", + "extracted": null, + "think": "unclosed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L254", + "prompt_id": "code-fib", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": false, + "why": "unclosed think (budget exhausted)", + "extracted": null, + "think": "unclosed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L161", + "prompt_id": "code-fib", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "def fib(n):\n if n == 0:\n return 0\n elif n == 1:\n return 1\n else:\n a, b = 0, 1\n for _ in range(2, n + 1):\n a, b = b, a + b\n return b\n", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L73", + "prompt_id": "code-fib", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "def fib(n):\n if n == 0:\n return 0\n elif n == 1:\n return 1\n a, b = 0, 1\n for _ in range(2, n + 1):\n a, b = b, a + b\n return b\n", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L63", + "prompt_id": "code-fib", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "def fib(n):\n if n == 0:\n return 0\n elif n == 1:\n return 1\n a, b = 0, 1\n for _ in range(2, n + 1):\n a, b = b, a + b\n return b\n", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L229", + "prompt_id": "code-fib", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "def fib(n):\n if n == 0:\n return 0\n elif n == 1:\n return 1\n else:\n a, b = 0, 1\n for _ in range(2, n + 1):\n a, b = b, a + b\n return b\n", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L95", + "prompt_id": "code-fib", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "def count_vowels(s):\n count = 0\n for char in s:\n if char.lower() in 'aeiou':\n count += 1\n return count\n", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L140", + "prompt_id": "code-count-vowels", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "engine": "hf", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "def count_vowels(s):\n count = 0\n vowels = \"aeiou\"\n for char in s:\n if char.lower() in vowels:\n count += 1\n return count\n", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L207", + "prompt_id": "code-count-vowels", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "def count_vowels(s):\n return sum(1 for char in s if char.lower() in 'aeiou')\n", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L255", + "prompt_id": "code-count-vowels", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "def count_vowels(s):\n vowels = \"aeiou\"\n count = 0\n for char in s:\n if char.lower() in vowels:\n count += 1\n return count\n", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L162", + "prompt_id": "code-count-vowels", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "def count_vowels(s):\n vowels = set('aeiouAEIOU')\n return sum(1 for char in s if char in vowels)\n", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L74", + "prompt_id": "code-count-vowels", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "def count_vowels(s):\n vowels = set('aeiouAEIOU')\n return sum(1 for char in s if char in vowels)\n", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L64", + "prompt_id": "code-count-vowels", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "def count_vowels(s):\n vowels = set('aeiouAEIOU')\n return sum(1 for char in s if char in vowels)\n", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L230", + "prompt_id": "code-count-vowels", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "def count_vowels(s):\n vowels = \"aeiouAEIOU\"\n count = 0\n for char in s:\n if char in vowels:\n count += 1\n return count\n", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L96", + "prompt_id": "code-count-vowels", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": [ + "2", + "3", + "5", + "7", + "11", + "13", + "17", + "19", + "23", + "29" + ], + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L141", + "prompt_id": "long-primes-10", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "engine": "hf", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": [ + "2", + "3", + "5", + "7", + "11", + "13", + "17", + "19", + "23", + "29" + ], + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L208", + "prompt_id": "long-primes-10", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": [ + "2", + "3", + "5", + "7", + "11", + "13", + "17", + "19", + "23", + "29" + ], + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L256", + "prompt_id": "long-primes-10", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": [ + "2", + "3", + "5", + "7", + "11", + "13", + "17", + "19", + "23", + "29" + ], + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L163", + "prompt_id": "long-primes-10", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": [ + "2", + "3", + "5", + "7", + "11", + "13", + "17", + "19", + "23", + "29" + ], + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L75", + "prompt_id": "long-primes-10", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": [ + "2", + "3", + "5", + "7", + "11", + "13", + "17", + "19", + "23", + "29" + ], + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L65", + "prompt_id": "long-primes-10", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": [ + "2", + "3", + "5", + "7", + "11", + "13", + "17", + "19", + "23", + "29" + ], + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L231", + "prompt_id": "long-primes-10", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": [ + "2", + "3", + "5", + "7", + "11", + "13", + "17", + "19", + "23", + "29" + ], + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L97", + "prompt_id": "long-primes-10", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": [ + "15", + "14", + "13", + "12", + "11", + "10", + "9", + "8", + "7", + "6", + "5", + "4", + "3", + "2", + "1" + ], + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L142", + "prompt_id": "long-countdown-15", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "engine": "hf", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": [ + "15", + "14", + "13", + "12", + "11", + "10", + "9", + "8", + "7", + "6", + "5", + "4", + "3", + "2", + "1" + ], + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L209", + "prompt_id": "long-countdown-15", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": [ + "15", + "14", + "13", + "12", + "11", + "10", + "9", + "8", + "7", + "6", + "5", + "4", + "3", + "2", + "1" + ], + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L257", + "prompt_id": "long-countdown-15", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": [ + "15", + "14", + "13", + "12", + "11", + "10", + "9", + "8", + "7", + "6", + "5", + "4", + "3", + "2", + "1" + ], + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L164", + "prompt_id": "long-countdown-15", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": [ + "15", + "14", + "13", + "12", + "11", + "10", + "9", + "8", + "7", + "6", + "5", + "4", + "3", + "2", + "1" + ], + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L76", + "prompt_id": "long-countdown-15", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": [ + "15", + "14", + "13", + "12", + "11", + "10", + "9", + "8", + "7", + "6", + "5", + "4", + "3", + "2", + "1" + ], + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L66", + "prompt_id": "long-countdown-15", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": [ + "15", + "14", + "13", + "12", + "11", + "10", + "9", + "8", + "7", + "6", + "5", + "4", + "3", + "2", + "1" + ], + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L232", + "prompt_id": "long-countdown-15", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": [ + "15", + "14", + "13", + "12", + "11", + "10", + "9", + "8", + "7", + "6", + "5", + "4", + "3", + "2", + "1" + ], + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L98", + "prompt_id": "long-countdown-15", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "4", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L121", + "prompt_id": "ctl-2plus2", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "engine": "hf", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "4", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L187", + "prompt_id": "ctl-2plus2", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "4", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L235", + "prompt_id": "ctl-2plus2", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "4", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L165", + "prompt_id": "ctl-2plus2", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "4", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L55", + "prompt_id": "ctl-2plus2", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "4", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L45", + "prompt_id": "ctl-2plus2", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "4", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L210", + "prompt_id": "ctl-2plus2", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "4", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L99", + "prompt_id": "ctl-2plus2", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "7", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L122", + "prompt_id": "ctl-recall-7", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "engine": "hf", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "7", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L188", + "prompt_id": "ctl-recall-7", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "7", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L236", + "prompt_id": "ctl-recall-7", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": false, + "why": "unclosed think (budget exhausted)", + "extracted": null, + "think": "unclosed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L166", + "prompt_id": "ctl-recall-7", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "7", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L56", + "prompt_id": "ctl-recall-7", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "7", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L46", + "prompt_id": "ctl-recall-7", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "7", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L211", + "prompt_id": "ctl-recall-7", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "7", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L100", + "prompt_id": "ctl-recall-7", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "def add(a, b):\n return a + b\n", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L127", + "prompt_id": "ctl-code-add", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "engine": "hf", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "def add(a, b):\n return a + b\n", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L189", + "prompt_id": "ctl-code-add", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "def add(a, b):\n return a + b\n", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L237", + "prompt_id": "ctl-code-add", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "def add(a, b):\n return a + b\n", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L167", + "prompt_id": "ctl-code-add", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "def add(a, b):\n return a + b\n", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L57", + "prompt_id": "ctl-code-add", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "def add(a, b):\n return a + b\n", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L47", + "prompt_id": "ctl-code-add", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "def add(a, b):\n return a + b\n", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L212", + "prompt_id": "ctl-code-add", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "def add(a, b):\n return a + b\n", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L101", + "prompt_id": "ctl-code-add", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "Tokyo", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L123", + "prompt_id": "fact-capital-japan", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "engine": "hf", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "Tokyo", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L190", + "prompt_id": "fact-capital-japan", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "Tokyo", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L238", + "prompt_id": "fact-capital-japan", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "Tokyo", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L168", + "prompt_id": "fact-capital-japan", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "Tokyo", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L58", + "prompt_id": "fact-capital-japan", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "Tokyo", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L48", + "prompt_id": "fact-capital-japan", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "Tokyo", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L213", + "prompt_id": "fact-capital-japan", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "Tokyo", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L102", + "prompt_id": "fact-capital-japan", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "Paris", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L124", + "prompt_id": "fact-capital-france", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "engine": "hf", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "Paris", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L191", + "prompt_id": "fact-capital-france", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "Paris", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L239", + "prompt_id": "fact-capital-france", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "Paris", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L169", + "prompt_id": "fact-capital-france", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "Paris", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L59", + "prompt_id": "fact-capital-france", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "Paris", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L49", + "prompt_id": "fact-capital-france", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "Paris", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L214", + "prompt_id": "fact-capital-france", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "Paris", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L103", + "prompt_id": "fact-capital-france", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "Paris", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L128", + "prompt_id": "golden-paris", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "engine": "hf", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "Paris", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L192", + "prompt_id": "golden-paris", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "Paris", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L240", + "prompt_id": "golden-paris", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "Paris", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L170", + "prompt_id": "golden-paris", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "Paris", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L60", + "prompt_id": "golden-paris", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "Paris", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L50", + "prompt_id": "golden-paris", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "Paris", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L215", + "prompt_id": "golden-paris", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "Paris", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L104", + "prompt_id": "golden-paris", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": false, + "why": "unclosed think (budget exhausted)", + "extracted": null, + "think": "unclosed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L125", + "prompt_id": "fact-water-formula", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "engine": "hf", + "verb": "chat", + "host": "gx10", + "correct": false, + "why": "unclosed think (budget exhausted)", + "extracted": null, + "think": "unclosed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L193", + "prompt_id": "fact-water-formula", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": false, + "why": "unclosed think (budget exhausted)", + "extracted": null, + "think": "unclosed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L241", + "prompt_id": "fact-water-formula", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": false, + "why": "unclosed think (budget exhausted)", + "extracted": null, + "think": "unclosed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L171", + "prompt_id": "fact-water-formula", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": false, + "why": "mismatch", + "extracted": "H₂O", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L61", + "prompt_id": "fact-water-formula", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": false, + "why": "mismatch", + "extracted": "H₂O", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L51", + "prompt_id": "fact-water-formula", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": false, + "why": "mismatch", + "extracted": "H₂O", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L216", + "prompt_id": "fact-water-formula", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": false, + "why": "mismatch", + "extracted": "H₂O", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L105", + "prompt_id": "fact-water-formula", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "水", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L233", + "prompt_id": "nonascii-water-kanji", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "engine": "hf", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "水", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L194", + "prompt_id": "nonascii-water-kanji", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "水", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L242", + "prompt_id": "nonascii-water-kanji", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "水", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L260", + "prompt_id": "nonascii-water-kanji", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "水", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L234", + "prompt_id": "nonascii-water-kanji", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "水", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L262", + "prompt_id": "nonascii-water-kanji", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "水", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L217", + "prompt_id": "nonascii-water-kanji", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "水", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L261", + "prompt_id": "nonascii-water-kanji", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "391", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L126", + "prompt_id": "arith-17x23", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "engine": "hf", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "391", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L195", + "prompt_id": "arith-17x23", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "391", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L243", + "prompt_id": "arith-17x23", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "391", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L172", + "prompt_id": "arith-17x23", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "391", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L62", + "prompt_id": "arith-17x23", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "391", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L52", + "prompt_id": "arith-17x23", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "391", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L218", + "prompt_id": "arith-17x23", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "391", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L106", + "prompt_id": "arith-17x23", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "23", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L129", + "prompt_id": "arith-precedence", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "engine": "hf", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "23", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L196", + "prompt_id": "arith-precedence", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "23", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L244", + "prompt_id": "arith-precedence", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "23", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L173", + "prompt_id": "arith-precedence", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "23", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L63", + "prompt_id": "arith-precedence", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "23", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L53", + "prompt_id": "arith-precedence", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "23", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L219", + "prompt_id": "arith-precedence", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "23", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L107", + "prompt_id": "arith-precedence", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "56", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L130", + "prompt_id": "arith-word-apples", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "engine": "hf", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "56", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L197", + "prompt_id": "arith-word-apples", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "56", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L245", + "prompt_id": "arith-word-apples", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "56", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L174", + "prompt_id": "arith-word-apples", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "56", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L64", + "prompt_id": "arith-word-apples", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "56", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L54", + "prompt_id": "arith-word-apples", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "56", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L220", + "prompt_id": "arith-word-apples", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "56", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L108", + "prompt_id": "arith-word-apples", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "8613", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L131", + "prompt_id": "hard-add-4digit", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "engine": "hf", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "8613", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L198", + "prompt_id": "hard-add-4digit", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "8613", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L246", + "prompt_id": "hard-add-4digit", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": false, + "why": "unclosed think (budget exhausted)", + "extracted": null, + "think": "unclosed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L175", + "prompt_id": "hard-add-4digit", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "8613", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L65", + "prompt_id": "hard-add-4digit", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "8613", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L55", + "prompt_id": "hard-add-4digit", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "8613", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L221", + "prompt_id": "hard-add-4digit", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "8613", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L109", + "prompt_id": "hard-add-4digit", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "15", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L132", + "prompt_id": "hard-not-prime", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "engine": "hf", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "15", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L199", + "prompt_id": "hard-not-prime", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "15", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L247", + "prompt_id": "hard-not-prime", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "15", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L176", + "prompt_id": "hard-not-prime", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "15", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L66", + "prompt_id": "hard-not-prime", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "15", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L56", + "prompt_id": "hard-not-prime", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "15", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L222", + "prompt_id": "hard-not-prime", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "15", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L110", + "prompt_id": "hard-not-prime", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "tenalp", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L133", + "prompt_id": "hard-reverse-word", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "engine": "hf", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "tenalp", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L200", + "prompt_id": "hard-reverse-word", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "tenalp", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L248", + "prompt_id": "hard-reverse-word", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": false, + "why": "mismatch", + "extracted": "teanlap", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L177", + "prompt_id": "hard-reverse-word", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "tenalp", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L67", + "prompt_id": "hard-reverse-word", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "tenalp", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L57", + "prompt_id": "hard-reverse-word", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "tenalp", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L223", + "prompt_id": "hard-reverse-word", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "tenalp", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L111", + "prompt_id": "hard-reverse-word", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "3", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L134", + "prompt_id": "hard-count-letters", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "engine": "hf", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "3", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L201", + "prompt_id": "hard-count-letters", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "3", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L249", + "prompt_id": "hard-count-letters", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "3", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L178", + "prompt_id": "hard-count-letters", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "3", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L68", + "prompt_id": "hard-count-letters", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "3", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L58", + "prompt_id": "hard-count-letters", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "3", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L224", + "prompt_id": "hard-count-letters", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "3", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L112", + "prompt_id": "hard-count-letters", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": false, + "why": "unclosed think (budget exhausted)", + "extracted": null, + "think": "unclosed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L135", + "prompt_id": "recall-cats", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "engine": "hf", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "5", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L202", + "prompt_id": "recall-cats", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "5", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L250", + "prompt_id": "recall-cats", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": false, + "why": "unclosed think (budget exhausted)", + "extracted": null, + "think": "unclosed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L179", + "prompt_id": "recall-cats", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "5", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L69", + "prompt_id": "recall-cats", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "5", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L59", + "prompt_id": "recall-cats", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "5", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L225", + "prompt_id": "recall-cats", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "5", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L113", + "prompt_id": "recall-cats", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "Ada", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L136", + "prompt_id": "recall-name", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "engine": "hf", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "Ada", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L203", + "prompt_id": "recall-name", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "Ada", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L251", + "prompt_id": "recall-name", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": false, + "why": "unclosed think (budget exhausted)", + "extracted": null, + "think": "unclosed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L180", + "prompt_id": "recall-name", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "Ada", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L70", + "prompt_id": "recall-name", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "Ada", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L60", + "prompt_id": "recall-name", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "Ada", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L226", + "prompt_id": "recall-name", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "Ada", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L114", + "prompt_id": "recall-name", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "7", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L137", + "prompt_id": "recall-arith-chain", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "engine": "hf", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "7", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L204", + "prompt_id": "recall-arith-chain", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "7", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L252", + "prompt_id": "recall-arith-chain", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "7", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L181", + "prompt_id": "recall-arith-chain", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "7", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L71", + "prompt_id": "recall-arith-chain", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "7", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L61", + "prompt_id": "recall-arith-chain", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "7", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L227", + "prompt_id": "recall-arith-chain", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "7", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L115", + "prompt_id": "recall-arith-chain", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "def is_palindrome(s):\n s = s.lower()\n return s == s[::-1]\n", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L138", + "prompt_id": "code-is-palindrome", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "engine": "hf", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "def is_palindrome(s):\n return s.lower() == s[::-1].lower()\n", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L205", + "prompt_id": "code-is-palindrome", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": false, + "why": "tests_failed: AssertionError", + "extracted": "def is_palindrome(s):\n return s.lower() == s[::-1]\n", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L253", + "prompt_id": "code-is-palindrome", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": false, + "why": "unclosed think (budget exhausted)", + "extracted": null, + "think": "unclosed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L182", + "prompt_id": "code-is-palindrome", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "def is_palindrome(s):\n cleaned = ''.join(char.lower() for char in s if char.isalnum())\n return cleaned == cleaned[::-1]\n", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L72", + "prompt_id": "code-is-palindrome", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "def is_palindrome(s):\n cleaned = ''.join(char.lower() for char in s if char.isalnum())\n return cleaned == cleaned[::-1]\n", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L62", + "prompt_id": "code-is-palindrome", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "def is_palindrome(s):\n cleaned = ''.join(char.lower() for char in s if char.isalnum())\n return cleaned == cleaned[::-1]\n", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L228", + "prompt_id": "code-is-palindrome", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "def is_palindrome(s):\n cleaned = ''.join(char.lower() for char in s if char.isalnum())\n return cleaned == cleaned[::-1]\n", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L116", + "prompt_id": "code-is-palindrome", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": false, + "why": "unclosed think (budget exhausted)", + "extracted": null, + "think": "unclosed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L139", + "prompt_id": "code-fib", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "engine": "hf", + "verb": "chat", + "host": "gx10", + "correct": false, + "why": "unclosed think (budget exhausted)", + "extracted": null, + "think": "unclosed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L206", + "prompt_id": "code-fib", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": false, + "why": "unclosed think (budget exhausted)", + "extracted": null, + "think": "unclosed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L254", + "prompt_id": "code-fib", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": false, + "why": "unclosed think (budget exhausted)", + "extracted": null, + "think": "unclosed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L183", + "prompt_id": "code-fib", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "def fib(n):\n if n == 0:\n return 0\n elif n == 1:\n return 1\n else:\n a, b = 0, 1\n for _ in range(2, n + 1):\n a, b = b, a + b\n return b\n", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L73", + "prompt_id": "code-fib", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "def fib(n):\n if n == 0:\n return 0\n elif n == 1:\n return 1\n a, b = 0, 1\n for _ in range(2, n + 1):\n a, b = b, a + b\n return b\n", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L63", + "prompt_id": "code-fib", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "def fib(n):\n if n == 0:\n return 0\n elif n == 1:\n return 1\n a, b = 0, 1\n for _ in range(2, n + 1):\n a, b = b, a + b\n return b\n", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L229", + "prompt_id": "code-fib", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "def fib(n):\n if n == 0:\n return 0\n elif n == 1:\n return 1\n else:\n a, b = 0, 1\n for _ in range(2, n + 1):\n a, b = b, a + b\n return b\n", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L117", + "prompt_id": "code-fib", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "def count_vowels(s):\n count = 0\n for char in s:\n if char.lower() in 'aeiou':\n count += 1\n return count\n", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L140", + "prompt_id": "code-count-vowels", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "engine": "hf", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "def count_vowels(s):\n count = 0\n vowels = \"aeiou\"\n for char in s:\n if char.lower() in vowels:\n count += 1\n return count\n", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L207", + "prompt_id": "code-count-vowels", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "def count_vowels(s):\n return sum(1 for char in s if char.lower() in 'aeiou')\n", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L255", + "prompt_id": "code-count-vowels", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": false, + "why": "unclosed think (budget exhausted)", + "extracted": null, + "think": "unclosed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L184", + "prompt_id": "code-count-vowels", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "def count_vowels(s):\n vowels = set('aeiouAEIOU')\n return sum(1 for char in s if char in vowels)\n", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L74", + "prompt_id": "code-count-vowels", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "def count_vowels(s):\n vowels = set('aeiouAEIOU')\n return sum(1 for char in s if char in vowels)\n", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L64", + "prompt_id": "code-count-vowels", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "def count_vowels(s):\n vowels = set('aeiouAEIOU')\n return sum(1 for char in s if char in vowels)\n", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L230", + "prompt_id": "code-count-vowels", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "def count_vowels(s):\n vowels = \"aeiouAEIOU\"\n count = 0\n for char in s:\n if char in vowels:\n count += 1\n return count\n", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L118", + "prompt_id": "code-count-vowels", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": [ + "2", + "3", + "5", + "7", + "11", + "13", + "17", + "19", + "23", + "29" + ], + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L141", + "prompt_id": "long-primes-10", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "engine": "hf", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": [ + "2", + "3", + "5", + "7", + "11", + "13", + "17", + "19", + "23", + "29" + ], + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L208", + "prompt_id": "long-primes-10", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": [ + "2", + "3", + "5", + "7", + "11", + "13", + "17", + "19", + "23", + "29" + ], + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L256", + "prompt_id": "long-primes-10", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": [ + "2", + "3", + "5", + "7", + "11", + "13", + "17", + "19", + "23", + "29" + ], + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L185", + "prompt_id": "long-primes-10", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": [ + "2", + "3", + "5", + "7", + "11", + "13", + "17", + "19", + "23", + "29" + ], + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L75", + "prompt_id": "long-primes-10", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": [ + "2", + "3", + "5", + "7", + "11", + "13", + "17", + "19", + "23", + "29" + ], + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L65", + "prompt_id": "long-primes-10", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": [ + "2", + "3", + "5", + "7", + "11", + "13", + "17", + "19", + "23", + "29" + ], + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L231", + "prompt_id": "long-primes-10", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": [ + "2", + "3", + "5", + "7", + "11", + "13", + "17", + "19", + "23", + "29" + ], + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L119", + "prompt_id": "long-primes-10", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": [ + "15", + "14", + "13", + "12", + "11", + "10", + "9", + "8", + "7", + "6", + "5", + "4", + "3", + "2", + "1" + ], + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L142", + "prompt_id": "long-countdown-15", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "engine": "hf", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": [ + "15", + "14", + "13", + "12", + "11", + "10", + "9", + "8", + "7", + "6", + "5", + "4", + "3", + "2", + "1" + ], + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L209", + "prompt_id": "long-countdown-15", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": [ + "15", + "14", + "13", + "12", + "11", + "10", + "9", + "8", + "7", + "6", + "5", + "4", + "3", + "2", + "1" + ], + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L257", + "prompt_id": "long-countdown-15", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": [ + "15", + "14", + "13", + "12", + "11", + "10", + "9", + "8", + "7", + "6", + "5", + "4", + "3", + "2", + "1" + ], + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L186", + "prompt_id": "long-countdown-15", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": [ + "15", + "14", + "13", + "12", + "11", + "10", + "9", + "8", + "7", + "6", + "5", + "4", + "3", + "2", + "1" + ], + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L76", + "prompt_id": "long-countdown-15", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": [ + "15", + "14", + "13", + "12", + "11", + "10", + "9", + "8", + "7", + "6", + "5", + "4", + "3", + "2", + "1" + ], + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L66", + "prompt_id": "long-countdown-15", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": [ + "15", + "14", + "13", + "12", + "11", + "10", + "9", + "8", + "7", + "6", + "5", + "4", + "3", + "2", + "1" + ], + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L232", + "prompt_id": "long-countdown-15", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": [ + "15", + "14", + "13", + "12", + "11", + "10", + "9", + "8", + "7", + "6", + "5", + "4", + "3", + "2", + "1" + ], + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L120", + "prompt_id": "long-countdown-15", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + } + ] +} diff --git a/evidence/perf041/lambda/marker.json b/evidence/perf041/lambda/marker.json index 1b969721ff..9a1da39e1d 100644 --- a/evidence/perf041/lambda/marker.json +++ b/evidence/perf041/lambda/marker.json @@ -1,10 +1,10 @@ { "cc": "89", - "commit": "f6fa4cf0cf2424a6e69940a457af5fe8fd057ebc", + "commit": "40980071333be516b6ec6683079646897e9cbcfc", "exit": 0, "host": "noah-Lambda-Vector", - "max_m": 14, - "sha256": "9fde772f14bceafefae2fb53fdfa8c471d9ad7b54127bbfacd193b5b7cbb9f7b", - "started_utc": "2026-09-27T05:50:00Z", + "max_m": 16, + "sha256": "0ca721d8899670fa7997597b3c7d87aede4a184fc20d5abcdec9cbebdd3a0ad0", + "started_utc": "2026-10-07T06:37:57Z", "status": "PASS" } diff --git a/evidence/release/0.70.2/RELEASE-NOTES-0.70.2.md b/evidence/release/0.70.2/RELEASE-NOTES-0.70.2.md new file mode 100644 index 0000000000..ae7db08cf6 --- /dev/null +++ b/evidence/release/0.70.2/RELEASE-NOTES-0.70.2.md @@ -0,0 +1,69 @@ +# aprender 0.70.2 — release notes + +## How this release was gated + +0.70.2 ships under a recorded operator emergency scope, `crux-smoke`, as 0.70.1 and 0.69.1 did. +The operator's words: "0.70.2 ships on CRUX smoke on lambda and gx10 GPU. Everything bigger is nightly." +The record is the `crux-smoke` entry for release "0.70.2" under `emergency_scopes` in +`contracts/model-capability-ladder-v1.yaml`. It is data only: the judge +(`scripts/lib/crux_smoke_scope.py`) matches a recorded scope by its whole release string, so no code changed +to admit it. + +**The gate.** The CRUX smoke runs on the two GPUs the quote names (an RTX 4090 and a GB10), with nothing else on +either GPU. It covers every certified model (Qwen3.5-2B Q4_K_M, Qwen3.5-4B Q4_K_M and Qwen3.5-4B UD-Q4_K_XL), +every admitted thinking mode (off, on), on the control prompts, and the verbs run, chat, serve and code. The +smoke is run with the binary built at the release commit. A commit cannot hold results produced from itself, +so those receipts are published with the GitHub release for v0.70.2. + +**The prompt certification is carried from 0.70.1, not re-run.** +`evidence/crux/0.70.2/prompt-certification.json` and `prompt-certification-inventory.json` are byte copies of +the `evidence/crux/0.70.1/` files. Carrying is allowed because nothing the certification depends on changed: +- the models are the same: the three certified files have the same sha256 as for 0.70.1; +- the prompts are the same: between v0.70.1 and this release's base, `evidence/crux/0.70.1`, the 0.69.1 prompt + certification, `scripts/crux_inference_dogfood.sh`, `scripts/lib/crux_smoke_scope.py` and + `scripts/crux_sweep_shards.sh` have no diff; +- the oracle engines are the same: the oracle record is still the 0.69.1 manifests, pinned by sha256 inside + the certification, which 0.70.1 carried the same way; +- the smoke does not re-run the oracle engines. + +**Printed as evidence, not the gate for this release.** The full model matrix and the release-readiness grade +still run at publish, and their verdicts are printed. They stay red where they were red; see Known failures. + +**Unchanged.** Clean tree, tag on HEAD, release branch, dogfood GO for this commit, and the clean-room green on +the tagged commit. + +## Changed since 0.70.1 +- `apr serve`: tools in a chat request now reach the chat template, and Qwen3.5 XML tool calls are parsed + (#4650). +- `apr serve`: KV caches no longer commit every byte when they are built (#4769). +- `apr train` / `apr finetune` on CPU: an intermittent crash (SIGSEGV) in the AVX2 matrix-multiply kernel, + which read past the end of its input, is fixed (#4891). +- The PP-26 witness marker under `evidence/perf041/` is re-measured on the base of this release, + so the release-phase check `scripts/check_perf041_marker.sh` judges fresh evidence (#4888). +- Release, CI and guard fixes on main since 0.70.1, among them: the coverage gate reads the nightly's + sha-keyed receipt and is checked before the tag (#4734); one list in code of what merge, tag and publish + require (#4688); no public GitHub release before its assets, clean-room and preflight (#4690). + +## Known failures (shipped, moved to 0.71.0) +The operator refused all six for 0.70.2 under the scope above. None has a per-model contract edit, and none is +marked known red or green. Each stays red in the printed evidence and in its ticket, and each is on the 0.71.0 +milestone. Each was re-measured on the RTX 4090 on 2026-10-06, against 0.70.1 built from main, and still fails the same way. + +| model | GPU | check | failure | ticket | +|---|---|---|---|---| +| qwen35-0.8b-q4km | RTX 4090, GB10 | golden_output thinking_on | think block unclosed within 2048 tokens | https://github.com/paiml/aprender/issues/4666 | +| Qwen2.5-0.5B-Instruct-f16.gguf | RTX 4090 | cuda serve /api/chat | gibberish output | https://github.com/paiml/aprender/issues/4661 | +| Qwen3-Coder-30B-A3B-Instruct-Q4_K_M.gguf | RTX 4090, GB10 | cuda serve /api/chat | gibberish output | https://github.com/paiml/aprender/issues/4662 | +| Qwen3.5-0.8B-IQ4_XS.gguf | RTX 4090 | golden_output thinking_on | no measured thinking budget (named refusal) | https://github.com/paiml/aprender/issues/4663 | +| Qwen3.5-0.8B-UD-IQ2_XXS.gguf | RTX 4090, GB10 | golden_output | GPU forward fell back to CPU (run and chat rc=14); `apr qa` also fails tensor_contract | https://github.com/paiml/aprender/issues/4664 | +| Qwen3.5-35B-A3B-UD-IQ4_XS.gguf | RTX 4090 | capability_match | no CUDA forward for qwen35moe (named refusal) | https://github.com/paiml/aprender/issues/4665 | + +## Moved to 0.71 +- The six known failures above (#4661, #4662, #4663, #4664, #4665, #4666). +- One unattended autopilot pass. `scripts/release/autopilot.sh` (its readiness step and `cut_tag`) does not + read a recorded emergency scope; only the publish preflight does. No fix before this release. + +## Not measured for 0.70.2 +- Capability cells (`model_ladder.sh --cells`), as for 0.70.1: a gap, not a pass + (https://github.com/paiml/aprender/issues/4667). +- The full CRUX sweep (beyond the smoke above) and CPU lanes: nightly, per the scope. diff --git a/scripts/lib/crux_cells_serve_code.sh b/scripts/lib/crux_cells_serve_code.sh index 7f03ff9418..aba2062f81 100644 --- a/scripts/lib/crux_cells_serve_code.sh +++ b/scripts/lib/crux_cells_serve_code.sh @@ -143,6 +143,12 @@ serve_routes_cell() { pa=$(free_port) printf '#!/usr/bin/env bash\n# one CRUX serve cell (#3962): every route apr mounts x every mode x every serve prompt\n' > "$cell" crux_teardown_trap "$cell" "$d/teardown.state" "$d/apr-serve.pid" "$d/llama-serve.pid" + # The ground-truth comparators (hf, vllm) run FIRST, alone on the device: they start their own + # servers and need no apr or llama port. Run after apr serve + llama-server were resident, a 4B + # model on one 24 GB card left hf OOM and vllm short of its 0.5 utilization, so every serve cell + # went RED with "no ground-truth control" while apr and llama.cpp both answered. + [ "${#pids[@]}" -gt 0 ] && crux_plugin_lines "$cell" "$d" "serve run" "${pids[@]}" + [ "${#spids[@]}" -gt 0 ] && crux_plugin_lines "$cell" "$d/stream" "serve stream" "${spids[@]}" { printf '%q ' "$APR" serve run "$M" --port "$pa" "$APR_BE"; printf '> %q 2>&1 < /dev/null &\necho $! > %q\n' "$d/apr-serve.log" "$d/apr-serve.pid"; } >> "$cell" serve_wait_line "$cell" "$pa" /health "$d/apr-serve.pid" local render=() @@ -170,8 +176,6 @@ serve_routes_cell() { --out-dir "$d/ollama" --device "$OL_DEVICE" "${common[@]}" fi [ "$HAVE_OLLAMA" = 1 ] && [ -z "$OL_REFUSED" ] && cell_add_ollama_unload "$cell" "$d/ollama-serve" "$OL_NAME" - [ "${#pids[@]}" -gt 0 ] && crux_plugin_lines "$cell" "$d" "serve run" "${pids[@]}" - [ "${#spids[@]}" -gt 0 ] && crux_plugin_lines "$cell" "$d/stream" "serve stream" "${spids[@]}" printf 'exit 0\n' >> "$cell" for pid in "${pids[@]}"; do for eng in $(crux_plugin_engines); do before[$eng-$pid]=$(VERB_KEY="serve run" rows_for "$eng" "$pid"); done; done