diff --git a/CHANGELOG.md b/CHANGELOG.md index 34db076292..aaa1400518 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,93 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [0.71.0] - 2026-10-11 + +The 0.71.0 train: release checks move into code with a fixed stage budget, checks that cannot block a merge or a release run nightly instead of on the PR path, a nightly release rehearsal and nightly train judge one commit end to end, the CI helpers and the CRUX judge are ported to Rust, the Laya ModernBERT encoder lands, and Qwen3.5 hybrid GGUF export loads in llama.cpp. From this release on, a release ships on CRUX smoke; larger model rows run nightly and each red row is a ticket. + +### Added + +- feat(release): nightly assets rehearsal — build every release asset on a schedule, unable to upload (#4806) (#4813) +- feat(ci-tools): aprender-ci-tools crate — Rust ports of the CI helpers (PY-PORT-2, split 1/2 of #4847) (#4916) +- feat(crux-judge): Rust CRUX judge (PMAT-4802, split 2/2 of #4847) (#4917) +- feat(hooks): pre-push tag guard, accepting a release tag at exactly its release branch head (#4944) +- feat(release): the stage list lives in code - one data file, one reader, a budget self-test (merge 9, tag 5, publish 5) (#4967) +- feat(release): nightly release rehearsal on one commit, with the three-night streak (#4997) +- feat(export): Qwen3.5 hybrid GGUF export that llama.cpp loads (#4998) +- feat(code): apr code's JSON document reports the device it ran on (Refs #3719) (#5005) +- feat(contracts): findings-ledger-v1, a findings line under contract (#4455) (#5006) +- feat(serve): X-Trace-Level step|layer runs the tracer, and says measured only when it ran (#4496) (#5015) +- feat(ttft): the V1 TTFT ratio verdict, the TTFT split by phase, and a baseline script that refuses an untrustworthy run (Refs #4954) (#5037) +- feat(dogfood): write a dogfood-receipt/v1 and run the gate path with no Python (#4377) (#5065) +- feat(release): the pass's runner-wait rule reads and prints inside its own waits (#4929) (#5078) +- feat(nightly-train): a red or not_measured night opens or updates one ticket with one owner (#5081) (#5085) +- feat(decide): carry #4941, the Laya ModernBERT encoder and decision library (#5101) +- feat(nightly): dogfood and readiness lanes get their producers on main (#5103) +- feat(bench): V2 serve receipt — load_ms + TTFT beside pp512/tg128 (Refs #4953) (#5106) + +### Fixed + +- fix(release): deep-nodefault no longer reads green on a failing cargo exit (#4874) +- fix(release): T-2 can return GO on main's head — defer only the pre-bump CRUX ladder row (D5) (#4931) +- fix(release): the coverage version-only rule admits the carried CRUX prompt certification (P7 D3) (#4932) +- fix(release): T-1 dogfood ladder gate waits for the models lane and reads its CRUX receipts (#4930) (#4933) +- fix(release): the models lane's remote leg fetched only main, so a patch release could not build there (#4938) +- fix(gpu): a shared cubin temp file let a reader load a partial cubin (#4956) (#4964) +- fix(gpu): cuBLAS tests synchronized the context inside another test's capture (#4956) (#4966) +- fix(gpu): a test filling the device let neighbours' allocations OOM in-process (#4956) (#4970) +- fix(guards): pin origin/main to the run's build base, so a re-run judges the main it was built on (#4996) +- fix(dogfood): the release model gates read a declared model list, not the model dir (#5000) +- fix(test): MUT-05/06/07 assert the nightly mutants job; m001/m008 nested build drops CONTRACT_* env (#5004) +- fix(ci): libm-ban-live prints clippy's output when clippy fails for another reason (#5020) +- fix(ci): fat_driver case-table fixture lines never reach the runner as annotations (Refs #4596, #4610) (#5029) +- fix(flake): disable test_stack_unwinding_does_not_crash under FLAKE-0 (#5035) +- fix(guards): crux serve guard's EXIT trap runs only in its own shell (#5012) (#5036) +- fix(#4523): per-arch qwen35 LOGITS_BUDGET + a nightly sm_121 lane for the parity module (#5038) +- fix(tests): four cbtop tests passed while apr-cli did not compile (#5042) +- fix(cuda): drop graph-mode buffers after a failed decode-graph capture (C14) (#5043) +- fix(release): the fleet cells gate judges in bash and awk, not python (#4350) (#5049) +- fix(nightly): ci-pending is asked again the same night, then ends RED (#5053) (#5054) +- fix(ci): section git fetches read up to 3 times through fetch_p6.sh (P6) (#4936) (#5066) +- fix(ont): rebind or delete the ghost binding rows; allowlist 163 -> 3 (#4470) (#5067) +- fix(examples): examples-nightly 36 rows to pass or a skip class, fail 0 timeout 0 (#4756) (#5068) +- fix(guards): crux serve guard's stub server holds no caller fd and never outlives its case (#5019) (#5073) +- fix(ont): binding rows after the ghost rebind — dup row, stale signatures, partial fits (#4470 follow-up) (#5077) +- fix(train): judge the commit of the night's scheduled nightly.yml run, not main's head at read time (#5086) +- fix(dogfood): pinning guard scopes a gate path whole; receipt test covers the exit (#4377 follow-up) (#5090) +- fix(release): gates and train judge the producer's commit, not main's head (#5082 hits 2,3,4,5,8,12) (#5094) +- fix(package): three crates gitignored a source file they ship (#5095) +- fix(models-nightly): no_token tests the credentials readable, not present (#5062) (#5097) +- fix(nightly): every registered lane names a producer on the commit, as its own check (#5080) (#5104) +- fix(scripts): kill only the pid recorded at spawn, never a name pattern (#5112) +- fix(run): qwen35 --gpu tok/s no longer bills the F2 guard as generation (#4342) (#5115) +- fix(nightly): the assets lane names its producer, the nightly assets rehearsal (#5116) +- fix(rehearse): pick walks 200 first-parent commits, and the verdict prints C's age (#5062) (#5129) +- fix(silicon): an API read failure is NOT_MEASURED, and gh's stderr is kept (#5031) (#5130) +- fix(cuda): Qwen3.5 batched prefill f32 dequants for IQ2_S, IQ2_XXS, IQ3_S, IQ3_XXS and Q2_K (#5131) +- fix(train): a stale pin is RED, and the lanes run counts for the commit it asserted (Refs #5083, #5087) (#5132) +- fix(train): the self-test runs an unpinned copy, so it is green inside an installed bundle (Refs #5083) (#5136) +- fix(decide): inherit aprender-core from the workspace; guard that path pins name their target version (#5121) (#5137) +- fix(andon): read the day dir, not cache/: a printed verdict was reported as missing (#5141) +- fix(night): a dispatched run on main counts as the night, one rule at every site (#5142) +- fix(rehearse): wiring counts the train's distinct capture names, so the lanes read no longer fails pick (Refs #5062) (#5143) +- fix(serve): Qwen3.5 host-RAM guard + 1.27x host footprint, recalibrated (Refs #4947) (#5146) +- fix(decide): pack_laya runs a deterministic smoke path with no arguments (#5157) +- fix(release): models gates on crux; ladder and six night rows go to LAB (#5158) + +### Changed + +- release: policy reader refuses an unquoted #; nightly rows match inside the entry (#4923) +- release: 0.70.3 — streaming tool calls (#4927) +- ci: remove a fat job's section containers when its run is cancelled (#4962) +- perf(pv): cut guard-cargo's pv runtime (scoped binding lint, optimized YAML parser in dev builds) (#4993) +- ci(receipts): judge receipt presence nightly over merged PRs, off the PR path (#4999) +- test(gpu): run the six gdn device tests in the CUDA unit job again (#4956) (#5044) +- release: the pass-start streak needs one night, not three (#5061) +- release: close the last strip spellings of the rehearsal name (#5061 follow-up) (#5071) +- ci: move three non-blocking PR-path rows to nightly LAB (#5076) +- refactor(metrics): split davies_bouldin_score and calinski_harabasz_score under the complexity limits (#5111) +- test(metrics): pin that cluster scores ignore labels past the last row (#5113) + ### Known issues - **Qwen3 MoE (Qwen3-30B-A3B, Qwen3-Coder-30B-A3B) on GB10 is not claimed for 0.70 (#3715); planned for 0.70.1.** On GB10 (sm_121) every verb (run, chat, serve, code) times out at 600 s on the 20k-token rung. The 0.69.1 diff --git a/Cargo.lock b/Cargo.lock index 2bd2d9ed97..12091a73b2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -248,7 +248,7 @@ dependencies = [ [[package]] name = "apr-cli" -version = "0.70.3" +version = "0.71.0" dependencies = [ "anyhow", "aprender-build-sha", @@ -325,7 +325,7 @@ dependencies = [ [[package]] name = "apr-format" -version = "0.70.3" +version = "0.71.0" dependencies = [ "bincode", "cargo_metadata", @@ -343,7 +343,7 @@ dependencies = [ [[package]] name = "aprender" -version = "0.70.3" +version = "0.71.0" dependencies = [ "apr-cli", "aprender-core", @@ -351,7 +351,7 @@ dependencies = [ [[package]] name = "aprender-bench-compute" -version = "0.70.3" +version = "0.71.0" dependencies = [ "aprender-core", "criterion 0.7.0", @@ -360,7 +360,7 @@ dependencies = [ [[package]] name = "aprender-bench-tokenizer" -version = "0.70.3" +version = "0.71.0" dependencies = [ "aprender-core", "criterion 0.7.0", @@ -369,14 +369,14 @@ dependencies = [ [[package]] name = "aprender-build-sha" -version = "0.70.3" +version = "0.71.0" dependencies = [ "tempfile", ] [[package]] name = "aprender-cbtop" -version = "0.70.3" +version = "0.71.0" dependencies = [ "anyhow", "aprender-build-sha", @@ -402,7 +402,7 @@ dependencies = [ [[package]] name = "aprender-cgp" -version = "0.70.3" +version = "0.71.0" dependencies = [ "anyhow", "aprender-build-sha", @@ -423,7 +423,7 @@ dependencies = [ [[package]] name = "aprender-ci-tools" -version = "0.70.3" +version = "0.71.0" dependencies = [ "aprender-build-sha", "clap", @@ -436,7 +436,7 @@ dependencies = [ [[package]] name = "aprender-common" -version = "0.70.3" +version = "0.71.0" dependencies = [ "clap", "lz4_flex 0.11.6", @@ -445,7 +445,7 @@ dependencies = [ [[package]] name = "aprender-compute" -version = "0.70.3" +version = "0.71.0" dependencies = [ "anyhow", "aprender-contracts", @@ -495,7 +495,7 @@ dependencies = [ [[package]] name = "aprender-compute-xtask" -version = "0.70.3" +version = "0.71.0" dependencies = [ "anyhow", "aprender-build-sha", @@ -510,7 +510,7 @@ dependencies = [ [[package]] name = "aprender-contracts" -version = "0.70.3" +version = "0.71.0" dependencies = [ "aprender-contracts-macros", "blake3", @@ -530,7 +530,7 @@ dependencies = [ [[package]] name = "aprender-contracts-cli" -version = "0.70.3" +version = "0.71.0" dependencies = [ "aprender-build-sha", "aprender-common", @@ -546,7 +546,7 @@ dependencies = [ [[package]] name = "aprender-contracts-macros" -version = "0.70.3" +version = "0.71.0" dependencies = [ "proc-macro2", "quote", @@ -556,7 +556,7 @@ dependencies = [ [[package]] name = "aprender-contrastive-data" -version = "0.70.3" +version = "0.71.0" dependencies = [ "aprender-contracts-macros", "aprender-rand", @@ -571,7 +571,7 @@ dependencies = [ [[package]] name = "aprender-core" -version = "0.70.3" +version = "0.71.0" dependencies = [ "aes-gcm", "alsa", @@ -625,7 +625,7 @@ dependencies = [ [[package]] name = "aprender-cuda-edge" -version = "0.70.3" +version = "0.71.0" dependencies = [ "aprender-gpu", "proptest", @@ -636,7 +636,7 @@ dependencies = [ [[package]] name = "aprender-cupti" -version = "0.70.3" +version = "0.71.0" dependencies = [ "bindgen", "bitflags 2.13.0", @@ -646,7 +646,7 @@ dependencies = [ [[package]] name = "aprender-data" -version = "0.70.3" +version = "0.71.0" dependencies = [ "aes-gcm", "aprender-build-sha", @@ -699,7 +699,7 @@ dependencies = [ [[package]] name = "aprender-db" -version = "0.70.3" +version = "0.71.0" dependencies = [ "anyhow", "aprender-build-sha", @@ -746,7 +746,7 @@ dependencies = [ [[package]] name = "aprender-decide" -version = "0.70.3" +version = "0.71.0" dependencies = [ "aprender-contracts-macros", "aprender-core", @@ -765,7 +765,7 @@ dependencies = [ [[package]] name = "aprender-distribute" -version = "0.70.3" +version = "0.71.0" dependencies = [ "aprender-compute", "aprender-db", @@ -795,7 +795,7 @@ dependencies = [ [[package]] name = "aprender-explain" -version = "0.70.3" +version = "0.71.0" dependencies = [ "aprender-build-sha", "aprender-gpu", @@ -814,7 +814,7 @@ dependencies = [ [[package]] name = "aprender-fft" -version = "0.70.3" +version = "0.71.0" dependencies = [ "criterion 0.7.0", "proptest", @@ -823,7 +823,7 @@ dependencies = [ [[package]] name = "aprender-gemm-codegen" -version = "0.70.3" +version = "0.71.0" dependencies = [ "proc-macro2", "quote", @@ -832,7 +832,7 @@ dependencies = [ [[package]] name = "aprender-gpu" -version = "0.70.3" +version = "0.71.0" dependencies = [ "aprender-common", "aprender-simulate", @@ -850,7 +850,7 @@ dependencies = [ [[package]] name = "aprender-graph" -version = "0.70.3" +version = "0.71.0" dependencies = [ "anyhow", "aprender-compute", @@ -871,7 +871,7 @@ dependencies = [ [[package]] name = "aprender-image" -version = "0.70.3" +version = "0.71.0" dependencies = [ "criterion 0.7.0", "proptest", @@ -880,7 +880,7 @@ dependencies = [ [[package]] name = "aprender-mcp" -version = "0.70.3" +version = "0.71.0" dependencies = [ "inventory", "jsonschema", @@ -893,7 +893,7 @@ dependencies = [ [[package]] name = "aprender-monte-carlo" -version = "0.70.3" +version = "0.71.0" dependencies = [ "aprender", "assert_cmd", @@ -912,7 +912,7 @@ dependencies = [ [[package]] name = "aprender-orchestrate" -version = "0.70.3" +version = "0.71.0" dependencies = [ "anyhow", "aprender-build-sha", @@ -983,7 +983,7 @@ dependencies = [ [[package]] name = "aprender-present-cli" -version = "0.70.3" +version = "0.71.0" dependencies = [ "aprender-build-sha", "aprender-contracts", @@ -998,7 +998,7 @@ dependencies = [ [[package]] name = "aprender-present-core" -version = "0.70.3" +version = "0.71.0" dependencies = [ "aprender-contracts", "aprender-contracts-macros", @@ -1011,7 +1011,7 @@ dependencies = [ [[package]] name = "aprender-present-layout" -version = "0.70.3" +version = "0.71.0" dependencies = [ "aprender-present-core", "criterion 0.7.0", @@ -1022,7 +1022,7 @@ dependencies = [ [[package]] name = "aprender-present-lib" -version = "0.70.3" +version = "0.71.0" dependencies = [ "aprender-contracts", "aprender-present-core", @@ -1051,7 +1051,7 @@ dependencies = [ [[package]] name = "aprender-present-terminal" -version = "0.70.3" +version = "0.71.0" dependencies = [ "aprender-build-sha", "aprender-contracts", @@ -1073,7 +1073,7 @@ dependencies = [ [[package]] name = "aprender-present-test" -version = "0.70.3" +version = "0.71.0" dependencies = [ "aprender-present-core", "aprender-present-test-macros", @@ -1084,7 +1084,7 @@ dependencies = [ [[package]] name = "aprender-present-test-macros" -version = "0.70.3" +version = "0.71.0" dependencies = [ "proc-macro2", "quote", @@ -1093,7 +1093,7 @@ dependencies = [ [[package]] name = "aprender-present-widgets" -version = "0.70.3" +version = "0.71.0" dependencies = [ "aprender-present-core", "aprender-present-yaml", @@ -1105,7 +1105,7 @@ dependencies = [ [[package]] name = "aprender-present-yaml" -version = "0.70.3" +version = "0.71.0" dependencies = [ "aprender-present-core", "proptest", @@ -1115,7 +1115,7 @@ dependencies = [ [[package]] name = "aprender-profile" -version = "0.70.3" +version = "0.71.0" dependencies = [ "addr2line 0.25.1", "anyhow", @@ -1168,7 +1168,7 @@ dependencies = [ [[package]] name = "aprender-profile-core" -version = "0.70.3" +version = "0.71.0" dependencies = [ "hex", "serde", @@ -1178,7 +1178,7 @@ dependencies = [ [[package]] name = "aprender-ptx-debug" -version = "0.70.3" +version = "0.71.0" dependencies = [ "aprender-build-sha", "clap", @@ -1188,7 +1188,7 @@ dependencies = [ [[package]] name = "aprender-qa-certify" -version = "0.70.3" +version = "0.71.0" dependencies = [ "aprender-build-sha", "chrono", @@ -1200,7 +1200,7 @@ dependencies = [ [[package]] name = "aprender-qa-cli" -version = "0.70.3" +version = "0.71.0" dependencies = [ "aprender-build-sha", "aprender-qa-certify", @@ -1220,7 +1220,7 @@ dependencies = [ [[package]] name = "aprender-qa-gen" -version = "0.70.3" +version = "0.71.0" dependencies = [ "anyhow", "chrono", @@ -1237,7 +1237,7 @@ dependencies = [ [[package]] name = "aprender-qa-report" -version = "0.70.3" +version = "0.71.0" dependencies = [ "anyhow", "aprender-qa-gen", @@ -1255,7 +1255,7 @@ dependencies = [ [[package]] name = "aprender-qa-runner" -version = "0.70.3" +version = "0.71.0" dependencies = [ "anyhow", "aprender-qa-gen", @@ -1280,7 +1280,7 @@ dependencies = [ [[package]] name = "aprender-quant" -version = "0.70.3" +version = "0.71.0" dependencies = [ "half", "serde_json", @@ -1288,7 +1288,7 @@ dependencies = [ [[package]] name = "aprender-rag" -version = "0.70.3" +version = "0.71.0" dependencies = [ "aprender-common", "aprender-compute", @@ -1317,7 +1317,7 @@ dependencies = [ [[package]] name = "aprender-rag-cli" -version = "0.70.3" +version = "0.71.0" dependencies = [ "anyhow", "aprender-build-sha", @@ -1337,7 +1337,7 @@ dependencies = [ [[package]] name = "aprender-rand" -version = "0.70.3" +version = "0.71.0" dependencies = [ "criterion 0.7.0", "proptest", @@ -1346,7 +1346,7 @@ dependencies = [ [[package]] name = "aprender-registry" -version = "0.70.3" +version = "0.71.0" dependencies = [ "anyhow", "aprender-core", @@ -1376,7 +1376,7 @@ dependencies = [ [[package]] name = "aprender-review-experiment" -version = "0.70.3" +version = "0.71.0" dependencies = [ "serde", "serde_json", @@ -1386,7 +1386,7 @@ dependencies = [ [[package]] name = "aprender-serve" -version = "0.70.3" +version = "0.71.0" dependencies = [ "anyhow", "approx", @@ -1457,7 +1457,7 @@ dependencies = [ [[package]] name = "aprender-shell" -version = "0.70.3" +version = "0.71.0" dependencies = [ "aprender-core", "clap", @@ -1472,7 +1472,7 @@ dependencies = [ [[package]] name = "aprender-simulate" -version = "0.70.3" +version = "0.71.0" dependencies = [ "aprender-build-sha", "aprender-contracts", @@ -1514,7 +1514,7 @@ dependencies = [ [[package]] name = "aprender-solve" -version = "0.70.3" +version = "0.71.0" dependencies = [ "criterion 0.7.0", "half", @@ -1524,7 +1524,7 @@ dependencies = [ [[package]] name = "aprender-sparse" -version = "0.70.3" +version = "0.71.0" dependencies = [ "criterion 0.7.0", "proptest", @@ -1533,7 +1533,7 @@ dependencies = [ [[package]] name = "aprender-tensor" -version = "0.70.3" +version = "0.71.0" dependencies = [ "criterion 0.7.0", "proptest", @@ -1542,7 +1542,7 @@ dependencies = [ [[package]] name = "aprender-test-cli" -version = "0.70.3" +version = "0.71.0" dependencies = [ "aprender-build-sha", "aprender-test-lib", @@ -1571,7 +1571,7 @@ dependencies = [ [[package]] name = "aprender-test-derive" -version = "0.70.3" +version = "0.71.0" dependencies = [ "proc-macro2", "quote", @@ -1581,7 +1581,7 @@ dependencies = [ [[package]] name = "aprender-test-js-gen" -version = "0.70.3" +version = "0.71.0" dependencies = [ "blake3", "chrono", @@ -1597,7 +1597,7 @@ dependencies = [ [[package]] name = "aprender-test-lib" -version = "0.70.3" +version = "0.71.0" dependencies = [ "aprender-compute", "aprender-present-core", @@ -1643,7 +1643,7 @@ dependencies = [ [[package]] name = "aprender-test-showcase" -version = "0.70.3" +version = "0.71.0" dependencies = [ "aprender-present-terminal", "aprender-test-lib", @@ -1660,7 +1660,7 @@ dependencies = [ [[package]] name = "aprender-train" -version = "0.70.3" +version = "0.71.0" dependencies = [ "approx", "aprender-common", @@ -1729,7 +1729,7 @@ dependencies = [ [[package]] name = "aprender-train-bench" -version = "0.70.3" +version = "0.71.0" dependencies = [ "aprender-build-sha", "aprender-train", @@ -1747,7 +1747,7 @@ dependencies = [ [[package]] name = "aprender-train-common" -version = "0.70.3" +version = "0.71.0" dependencies = [ "aprender-viz", "clap", @@ -1759,7 +1759,7 @@ dependencies = [ [[package]] name = "aprender-train-distill" -version = "0.70.3" +version = "0.71.0" dependencies = [ "aprender-build-sha", "aprender-train", @@ -1781,7 +1781,7 @@ dependencies = [ [[package]] name = "aprender-train-inspect" -version = "0.70.3" +version = "0.71.0" dependencies = [ "aprender-build-sha", "aprender-train", @@ -1798,7 +1798,7 @@ dependencies = [ [[package]] name = "aprender-train-lora" -version = "0.70.3" +version = "0.71.0" dependencies = [ "aprender-build-sha", "aprender-train", @@ -1815,7 +1815,7 @@ dependencies = [ [[package]] name = "aprender-train-shell" -version = "0.70.3" +version = "0.71.0" dependencies = [ "aprender-build-sha", "aprender-train", @@ -1832,7 +1832,7 @@ dependencies = [ [[package]] name = "aprender-train-wasm" -version = "0.70.3" +version = "0.71.0" dependencies = [ "proptest", "serde", @@ -1843,7 +1843,7 @@ dependencies = [ [[package]] name = "aprender-tsp" -version = "0.70.3" +version = "0.71.0" dependencies = [ "aprender", "assert_cmd", @@ -1858,7 +1858,7 @@ dependencies = [ [[package]] name = "aprender-update" -version = "0.70.3" +version = "0.71.0" dependencies = [ "flate2", "semver", @@ -1872,7 +1872,7 @@ dependencies = [ [[package]] name = "aprender-verify" -version = "0.70.3" +version = "0.71.0" dependencies = [ "chrono", "criterion 0.5.1", @@ -1884,7 +1884,7 @@ dependencies = [ [[package]] name = "aprender-verify-ml" -version = "0.70.3" +version = "0.71.0" dependencies = [ "aprender-build-sha", "aprender-compute", @@ -1915,7 +1915,7 @@ dependencies = [ [[package]] name = "aprender-viz" -version = "0.70.3" +version = "0.71.0" dependencies = [ "approx", "aprender-common", @@ -1944,7 +1944,7 @@ dependencies = [ [[package]] name = "aprender-zram" -version = "0.70.3" +version = "0.71.0" dependencies = [ "aprender-zram-adaptive", "aprender-zram-core", @@ -1954,7 +1954,7 @@ dependencies = [ [[package]] name = "aprender-zram-adaptive" -version = "0.70.3" +version = "0.71.0" dependencies = [ "aprender-zram-core", "proptest", @@ -1963,7 +1963,7 @@ dependencies = [ [[package]] name = "aprender-zram-cli" -version = "0.70.3" +version = "0.71.0" dependencies = [ "aprender-build-sha", "aprender-zram-adaptive", @@ -1978,7 +1978,7 @@ dependencies = [ [[package]] name = "aprender-zram-core" -version = "0.70.3" +version = "0.71.0" dependencies = [ "aprender-gpu", "criterion 0.7.0", @@ -1990,7 +1990,7 @@ dependencies = [ [[package]] name = "aprender-zram-generator" -version = "0.70.3" +version = "0.71.0" dependencies = [ "aprender-build-sha", "aprender-zram-core", diff --git a/Cargo.toml b/Cargo.toml index d904cce360..bc1e617da3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -162,7 +162,7 @@ exclude = [ resolver = "2" [workspace.package] -version = "0.70.3" +version = "0.71.0" edition = "2021" license = "MIT" repository = "https://github.com/paiml/aprender" @@ -244,11 +244,11 @@ tokenizers = { version = "0.23.1", default-features = false, features = ["fancy- # aprender-rand (lib name `trueno_rand`) is a workspace MEMBER but had no # [workspace.dependencies] entry, so every consumer had to re-declare the path. # It is the counter-based Philox RNG behind the SetFit encoder's dropout masks. -aprender-rand = { path = "crates/aprender-rand", version = "0.70.3" } +aprender-rand = { path = "crates/aprender-rand", version = "0.71.0" } # aprender-core (lib name `aprender`): a consumer that hand-pins it falls behind the # workspace version, and the rc stamp then cannot resolve it (aprender-decide pinned # 0.70.2 under a 0.70.3 workspace). Inherit this entry instead. -aprender-core = { path = "crates/aprender-core", version = "0.70.3" } +aprender-core = { path = "crates/aprender-core", version = "0.71.0" } # Chat templates minijinja = { version = "2.14", features = ["loader", "serde"] } @@ -258,9 +258,9 @@ minijinja = { version = "2.14", features = ["loader", "serde"] } # `crates/aprender-contracts` (lib name `provable_contracts`). Declaring it from # crates.io compiled a SECOND copy of the contract engine alongside the in-tree # one — exactly the duplicate-crate + publish-cycle the consolidation removed. -provable-contracts = { path = "crates/aprender-contracts", version = "0.70.3", package = "aprender-contracts" } -provable-contracts-macros = { path = "crates/aprender-contracts-macros", version = "0.70.3", package = "aprender-contracts-macros" } -aprender-contracts-macros = { path = "crates/aprender-contracts-macros", version = "0.70.3" } +provable-contracts = { path = "crates/aprender-contracts", version = "0.71.0", package = "aprender-contracts" } +provable-contracts-macros = { path = "crates/aprender-contracts-macros", version = "0.71.0", package = "aprender-contracts-macros" } +aprender-contracts-macros = { path = "crates/aprender-contracts-macros", version = "0.71.0" } # YAML serde_yaml = "0.9" @@ -272,69 +272,69 @@ hf-hub = { version = "0.4", default-features = false, features = ["ureq"] } hf-xet = "1.5.1" # Workspace-internal crates (paths within the monorepo) -aprender-compute = { path = "crates/aprender-compute", version = "0.70.3" } -aprender-gpu = { path = "crates/aprender-gpu", version = "0.70.3" } -aprender-quant = { path = "crates/aprender-quant", version = "0.70.3" } -aprender-serve = { path = "crates/aprender-serve", version = "0.70.3" } -realizar = { path = "crates/aprender-serve", version = "0.70.3", package = "aprender-serve" } -alimentar = { path = "crates/aprender-data", version = "0.70.3", package = "aprender-data" } -pacha = { path = "crates/aprender-registry", version = "0.70.3", package = "aprender-registry" } +aprender-compute = { path = "crates/aprender-compute", version = "0.71.0" } +aprender-gpu = { path = "crates/aprender-gpu", version = "0.71.0" } +aprender-quant = { path = "crates/aprender-quant", version = "0.71.0" } +aprender-serve = { path = "crates/aprender-serve", version = "0.71.0" } +realizar = { path = "crates/aprender-serve", version = "0.71.0", package = "aprender-serve" } +alimentar = { path = "crates/aprender-data", version = "0.71.0", package = "aprender-data" } +pacha = { path = "crates/aprender-registry", version = "0.71.0", package = "aprender-registry" } # APR-MONO self-containment: legacy sibling lib-name aliases -> in-monorepo crates (2026-06-12) -trueno-quant = { path = "crates/aprender-quant", version = "0.70.3", package = "aprender-quant" } -trueno-db = { path = "crates/aprender-db", version = "0.70.3", package = "aprender-db" } -trueno-viz = { path = "crates/aprender-viz", version = "0.70.3", package = "aprender-viz" } -trueno-rag = { path = "crates/aprender-rag", version = "0.70.3", package = "aprender-rag" } -trueno-cuda-edge = { path = "crates/aprender-cuda-edge", version = "0.70.3", package = "aprender-cuda-edge" } -trueno-sparse = { path = "crates/aprender-sparse", version = "0.70.3", package = "aprender-sparse" } -trueno-solve = { path = "crates/aprender-solve", version = "0.70.3", package = "aprender-solve" } -renacer-core = { path = "crates/aprender-profile-core", version = "0.70.3", package = "aprender-profile-core" } -repartir = { path = "crates/aprender-distribute", version = "0.70.3", package = "aprender-distribute" } -simular = { path = "crates/aprender-simulate", version = "0.70.3", package = "aprender-simulate" } -probar = { path = "crates/aprender-test-lib", version = "0.70.3", package = "aprender-test-lib" } +trueno-quant = { path = "crates/aprender-quant", version = "0.71.0", package = "aprender-quant" } +trueno-db = { path = "crates/aprender-db", version = "0.71.0", package = "aprender-db" } +trueno-viz = { path = "crates/aprender-viz", version = "0.71.0", package = "aprender-viz" } +trueno-rag = { path = "crates/aprender-rag", version = "0.71.0", package = "aprender-rag" } +trueno-cuda-edge = { path = "crates/aprender-cuda-edge", version = "0.71.0", package = "aprender-cuda-edge" } +trueno-sparse = { path = "crates/aprender-sparse", version = "0.71.0", package = "aprender-sparse" } +trueno-solve = { path = "crates/aprender-solve", version = "0.71.0", package = "aprender-solve" } +renacer-core = { path = "crates/aprender-profile-core", version = "0.71.0", package = "aprender-profile-core" } +repartir = { path = "crates/aprender-distribute", version = "0.71.0", package = "aprender-distribute" } +simular = { path = "crates/aprender-simulate", version = "0.71.0", package = "aprender-simulate" } +probar = { path = "crates/aprender-test-lib", version = "0.71.0", package = "aprender-test-lib" } # English key for the same crate, named after the command it serves (`apr test`). # The dep KEY, not the package name, is what becomes the extern crate name when # the two differ — the lib target is still `jugar_probar`, so keying it by its # package name would have imported Spanish anyway. Renaming that lib target is # 196 call sites in 17 crates and gets its own PR; this key is how new code # reads English today. -apr-test = { path = "crates/aprender-test-lib", version = "0.70.3", package = "aprender-test-lib", default-features = false, features = ["llm"] } +apr-test = { path = "crates/aprender-test-lib", version = "0.71.0", package = "aprender-test-lib", default-features = false, features = ["llm"] } # `jugar-probar` is the crates.io/lib name of the same in-tree crate as `probar` # above. Both aliases resolve to ONE package; consumers that spelled it # `jugar-probar = "0.4"|"0.5"|"1.0"` were building three extra copies. -jugar-probar = { path = "crates/aprender-test-lib", version = "0.70.3", package = "aprender-test-lib" } +jugar-probar = { path = "crates/aprender-test-lib", version = "0.71.0", package = "aprender-test-lib" } # NOTE: the matching `jugar-probar-derive` alias already exists further down this # same table (it points at crates/aprender-test-derive). crates/aprender-test-lib # was pinning `jugar-probar-derive = "=1.0.3"` from crates.io regardless — the # alias was sitting there unused. -aprender-train = { path = "crates/aprender-train", version = "0.70.3" } -entrenar = { path = "crates/aprender-train", version = "0.70.3", package = "aprender-train" } -aprender-train-common = { path = "crates/aprender-train-common", version = "0.70.3" } -aprender-train-distill = { path = "crates/aprender-train-distill", version = "0.70.3" } -aprender-orchestrate = { path = "crates/aprender-orchestrate", version = "0.70.3" } -aprender-contracts = { path = "crates/aprender-contracts", version = "0.70.3" } -aprender-build-sha = { path = "crates/aprender-build-sha", version = "0.70.3" } -aprender-profile = { path = "crates/aprender-profile", version = "0.70.3" } -aprender-profile-core = { path = "crates/aprender-profile-core", version = "0.70.3" } -aprender-zram-core = { path = "crates/aprender-zram-core", version = "0.70.3" } -aprender-zram-adaptive = { path = "crates/aprender-zram-adaptive", version = "0.70.3" } -aprender-zram-cli = { path = "crates/aprender-zram-cli", version = "0.70.3" } -aprender-present-core = { path = "crates/aprender-present-core", version = "0.70.3" } -aprender-present-terminal = { path = "crates/aprender-present-terminal", version = "0.70.3" } -aprender-present-widgets = { path = "crates/aprender-present-widgets", version = "0.70.3" } -aprender-present-layout = { path = "crates/aprender-present-layout", version = "0.70.3" } -aprender-present-yaml = { path = "crates/aprender-present-yaml", version = "0.70.3" } -aprender-present-test = { path = "crates/aprender-present-test", version = "0.70.3" } -aprender-db = { path = "crates/aprender-db", version = "0.70.3" } -aprender-graph = { path = "crates/aprender-graph", version = "0.70.3" } -aprender-rag = { path = "crates/aprender-rag", version = "0.70.3" } -aprender-rag-cli = { path = "crates/aprender-rag-cli", version = "0.70.3" } -aprender-qa-cli = { path = "crates/aprender-qa-cli", version = "0.70.3" } -aprender-cgp = { path = "crates/aprender-cgp", version = "0.70.3" } -aprender-contrastive-data = { path = "crates/aprender-contrastive-data", version = "0.70.3" } -aprender-contracts-cli = { path = "crates/aprender-contracts-cli", version = "0.70.3" } -aprender-update = { path = "crates/aprender-update", version = "0.70.3" } -aprender-data = { path = "crates/aprender-data", version = "0.70.3" } -aprender-ptx-debug = { path = "crates/aprender-ptx-debug", version = "0.70.3" } +aprender-train = { path = "crates/aprender-train", version = "0.71.0" } +entrenar = { path = "crates/aprender-train", version = "0.71.0", package = "aprender-train" } +aprender-train-common = { path = "crates/aprender-train-common", version = "0.71.0" } +aprender-train-distill = { path = "crates/aprender-train-distill", version = "0.71.0" } +aprender-orchestrate = { path = "crates/aprender-orchestrate", version = "0.71.0" } +aprender-contracts = { path = "crates/aprender-contracts", version = "0.71.0" } +aprender-build-sha = { path = "crates/aprender-build-sha", version = "0.71.0" } +aprender-profile = { path = "crates/aprender-profile", version = "0.71.0" } +aprender-profile-core = { path = "crates/aprender-profile-core", version = "0.71.0" } +aprender-zram-core = { path = "crates/aprender-zram-core", version = "0.71.0" } +aprender-zram-adaptive = { path = "crates/aprender-zram-adaptive", version = "0.71.0" } +aprender-zram-cli = { path = "crates/aprender-zram-cli", version = "0.71.0" } +aprender-present-core = { path = "crates/aprender-present-core", version = "0.71.0" } +aprender-present-terminal = { path = "crates/aprender-present-terminal", version = "0.71.0" } +aprender-present-widgets = { path = "crates/aprender-present-widgets", version = "0.71.0" } +aprender-present-layout = { path = "crates/aprender-present-layout", version = "0.71.0" } +aprender-present-yaml = { path = "crates/aprender-present-yaml", version = "0.71.0" } +aprender-present-test = { path = "crates/aprender-present-test", version = "0.71.0" } +aprender-db = { path = "crates/aprender-db", version = "0.71.0" } +aprender-graph = { path = "crates/aprender-graph", version = "0.71.0" } +aprender-rag = { path = "crates/aprender-rag", version = "0.71.0" } +aprender-rag-cli = { path = "crates/aprender-rag-cli", version = "0.71.0" } +aprender-qa-cli = { path = "crates/aprender-qa-cli", version = "0.71.0" } +aprender-cgp = { path = "crates/aprender-cgp", version = "0.71.0" } +aprender-contrastive-data = { path = "crates/aprender-contrastive-data", version = "0.71.0" } +aprender-contracts-cli = { path = "crates/aprender-contracts-cli", version = "0.71.0" } +aprender-update = { path = "crates/aprender-update", version = "0.71.0" } +aprender-data = { path = "crates/aprender-data", version = "0.71.0" } +aprender-ptx-debug = { path = "crates/aprender-ptx-debug", version = "0.71.0" } # Additional external deps needed by sub-crates serde_yaml_ng = "0.10" @@ -393,25 +393,25 @@ wasmtime = "48.0.3" bollard = "0.17" # Old crate name aliases (workspace = true refs in migrated sub-crates) -trueno-zram-core = { path = "crates/aprender-zram-core", version = "0.70.3", package = "aprender-zram-core" } -trueno-zram-adaptive = { path = "crates/aprender-zram-adaptive", version = "0.70.3", package = "aprender-zram-adaptive" } -trueno = { path = "crates/aprender-compute", version = "0.70.3", package = "aprender-compute" } -presentar-core = { path = "crates/aprender-present-core", version = "0.70.3", package = "aprender-present-core" } -presentar-terminal = { path = "crates/aprender-present-terminal", version = "0.70.3", package = "aprender-present-terminal" } -presentar-widgets = { path = "crates/aprender-present-widgets", version = "0.70.3", package = "aprender-present-widgets" } -presentar-layout = { path = "crates/aprender-present-layout", version = "0.70.3", package = "aprender-present-layout" } -presentar-yaml = { path = "crates/aprender-present-yaml", version = "0.70.3", package = "aprender-present-yaml" } -presentar-test = { path = "crates/aprender-present-test", version = "0.70.3", package = "aprender-present-test" } -presentar-test-macros = { path = "crates/aprender-present-test-macros", version = "0.70.3", package = "aprender-present-test-macros" } -jugar-probar-derive = { path = "crates/aprender-test-derive", version = "0.70.3", package = "aprender-test-derive" } -batuta-common = { path = "crates/aprender-common", version = "0.70.3", package = "aprender-common" } +trueno-zram-core = { path = "crates/aprender-zram-core", version = "0.71.0", package = "aprender-zram-core" } +trueno-zram-adaptive = { path = "crates/aprender-zram-adaptive", version = "0.71.0", package = "aprender-zram-adaptive" } +trueno = { path = "crates/aprender-compute", version = "0.71.0", package = "aprender-compute" } +presentar-core = { path = "crates/aprender-present-core", version = "0.71.0", package = "aprender-present-core" } +presentar-terminal = { path = "crates/aprender-present-terminal", version = "0.71.0", package = "aprender-present-terminal" } +presentar-widgets = { path = "crates/aprender-present-widgets", version = "0.71.0", package = "aprender-present-widgets" } +presentar-layout = { path = "crates/aprender-present-layout", version = "0.71.0", package = "aprender-present-layout" } +presentar-yaml = { path = "crates/aprender-present-yaml", version = "0.71.0", package = "aprender-present-yaml" } +presentar-test = { path = "crates/aprender-present-test", version = "0.71.0", package = "aprender-present-test" } +presentar-test-macros = { path = "crates/aprender-present-test-macros", version = "0.71.0", package = "aprender-present-test-macros" } +jugar-probar-derive = { path = "crates/aprender-test-derive", version = "0.71.0", package = "aprender-test-derive" } +batuta-common = { path = "crates/aprender-common", version = "0.71.0", package = "aprender-common" } # APR-MONO sibling aliases — MUST use workspace refs, NEVER crates.io versions. # Hard-pinning these to crates.io creates a diamond: sub-crates pull older # aprender versions → multiple `aprender` crates in Cargo.lock = MUDA. -renacer = { path = "crates/aprender-profile", version = "0.70.3", package = "aprender-profile" } -entrenar-lora = { path = "crates/aprender-train-lora", version = "0.70.3", package = "aprender-train-lora" } -batuta = { path = "crates/aprender-orchestrate", version = "0.70.3", package = "aprender-orchestrate" } -trueno-graph = { path = "crates/aprender-graph", version = "0.70.3", package = "aprender-graph" } +renacer = { path = "crates/aprender-profile", version = "0.71.0", package = "aprender-profile" } +entrenar-lora = { path = "crates/aprender-train-lora", version = "0.71.0", package = "aprender-train-lora" } +batuta = { path = "crates/aprender-orchestrate", version = "0.71.0", package = "aprender-orchestrate" } +trueno-graph = { path = "crates/aprender-graph", version = "0.71.0", package = "aprender-graph" } [workspace.lints.rust] # Safety @@ -593,7 +593,7 @@ semicolon_if_nothing_returned = "allow" # Style preference [package] name = "aprender" -version = "0.70.3" +version = "0.71.0" edition = "2021" rust-version = "1.91" authors = ["Noah Gift "] @@ -757,9 +757,9 @@ required-features = ["cli"] # (e.g. apr-cookbook, downstream apps) don't pull apr-cli's heavy transitive # dep graph (renacer/profile/inference/zram/visualization). `cargo install # aprender` still works because `cli` is in default features. See GH-1599. -apr-cli = { path = "crates/apr-cli", version = "0.70.3", default-features = true, optional = true } +apr-cli = { path = "crates/apr-cli", version = "0.71.0", default-features = true, optional = true } # Core ML library (lib name = "aprender") -aprender_ml = { path = "crates/aprender-core", version = "0.70.3", package = "aprender-core" } +aprender_ml = { path = "crates/aprender-core", version = "0.71.0", package = "aprender-core" } [features] default = ["cli"] diff --git a/crates/apr-cli/Cargo.toml b/crates/apr-cli/Cargo.toml index 4ad4615bf5..68b0ea9713 100644 --- a/crates/apr-cli/Cargo.toml +++ b/crates/apr-cli/Cargo.toml @@ -130,12 +130,12 @@ libc = "0.2" # performed. A surface and its capability must not be able to drift. apr-test = { workspace = true } # Core aprender library -aprender = { path = "../aprender-core", version = "0.70.3", package = "aprender-core", default-features = true, features = ["format-compression"] } +aprender = { path = "../aprender-core", version = "0.71.0", package = "aprender-core", default-features = true, features = ["format-compression"] } # Provable-contracts engine — `apr beat-run` parses + evaluates BeatBenchmark contracts (PMAT-741) aprender-contracts = { workspace = true } # MCP (Model Context Protocol) server — `apr mcp` subcommand -aprender-mcp = { path = "../aprender-mcp", version = "0.70.3", features = ["apr-tools"] } # apr mcp serves apr's own tools (PMAT-3954) +aprender-mcp = { path = "../aprender-mcp", version = "0.71.0", features = ["apr-tools"] } # apr mcp serves apr's own tools (PMAT-3954) async-stream = "0.3" provable-contracts-macros = { workspace = true } @@ -146,7 +146,7 @@ batuta-common = { workspace = true, features = ["cli-roles"] } # Non-optional since claude-code-parity-apr M150 (outcome-parity Phase 3 # requires apr code in default builds). GH-344: path dep via # .cargo/config.toml.dev-overrides; CI uses checkout+symlink. -batuta = { version = "0.70.3", path = "../aprender-orchestrate", package = "aprender-orchestrate", default-features = false, features = ["agents", "agents-inference", "agents-mcp", "rag"] } +batuta = { version = "0.71.0", path = "../aprender-orchestrate", package = "aprender-orchestrate", default-features = false, features = ["agents", "agents-inference", "agents-mcp", "rag"] } # Inference engine (workspace path dep — enables cuda/gpu feature forwarding) # GH-573: shim (0.9.1) didn't forward cuda feature → 0.7 tok/s instead of 273 tok/s @@ -242,7 +242,7 @@ pacha = { workspace = true } # published-crate dependency cycle the APR-MONO consolidation removed, and # (b) was never actually linked, so every `trueno_explain::` path in # commands/ptx_explain.rs failed to resolve. -trueno-explain = { path = "../aprender-explain", version = "0.70.3", package = "aprender-explain", optional = true } +trueno-explain = { path = "../aprender-explain", version = "0.71.0", package = "aprender-explain", optional = true } # ML tuning: LoRA/QLoRA configuration and memory planning (GH-176, PMAT-184) entrenar-lora = { workspace = true, optional = true } diff --git a/crates/aprender-bench-compute/Cargo.toml b/crates/aprender-bench-compute/Cargo.toml index ae5f4cc512..2670a87bee 100644 --- a/crates/aprender-bench-compute/Cargo.toml +++ b/crates/aprender-bench-compute/Cargo.toml @@ -11,7 +11,7 @@ disallowed_methods = "allow" [dependencies] # The implementation under test -aprender = { path = "../aprender-core", version = "0.70.3", package = "aprender-core", features = ["format-quantize"] } +aprender = { path = "../aprender-core", version = "0.71.0", package = "aprender-core", features = ["format-quantize"] } # Reference implementation for A/B comparison # Pure Rust (no BLAS) to keep comparison fair — both are pure Rust SIMD diff --git a/crates/aprender-bench-tokenizer/Cargo.toml b/crates/aprender-bench-tokenizer/Cargo.toml index d57a83514b..6d409c4f83 100644 --- a/crates/aprender-bench-tokenizer/Cargo.toml +++ b/crates/aprender-bench-tokenizer/Cargo.toml @@ -11,7 +11,7 @@ disallowed_methods = "allow" [dependencies] # The implementation under test -aprender = { path = "../aprender-core", version = "0.70.3", package = "aprender-core" } +aprender = { path = "../aprender-core", version = "0.71.0", package = "aprender-core" } # HuggingFace reference for A/B comparison # Regular dep (not dev-dep) because [[bench]] harness=false binaries can't see dev-deps diff --git a/crates/aprender-cbtop/Cargo.toml b/crates/aprender-cbtop/Cargo.toml index eede8c1182..3430db0f85 100644 --- a/crates/aprender-cbtop/Cargo.toml +++ b/crates/aprender-cbtop/Cargo.toml @@ -32,7 +32,7 @@ presentar-terminal = { workspace = true } # Compute backends trueno = { workspace = true } -trueno-gpu = { version = "0.70.3", path = "../aprender-gpu", package = "aprender-gpu", optional = true } +trueno-gpu = { version = "0.71.0", path = "../aprender-gpu", package = "aprender-gpu", optional = true } # Terminal handling crossterm = "0.28" diff --git a/crates/aprender-cgp/Cargo.toml b/crates/aprender-cgp/Cargo.toml index 8ca0a16515..1d69773d34 100644 --- a/crates/aprender-cgp/Cargo.toml +++ b/crates/aprender-cgp/Cargo.toml @@ -36,7 +36,7 @@ num_cpus = "1.17" comfy-table = "7" colored = "3" # GPU profiling: direct cuBLAS measurement (OWN THE STACK) -trueno-gpu = { path = "../aprender-gpu", version = "0.70.3", features = ["cuda"], optional = true, package = "aprender-gpu" } +trueno-gpu = { path = "../aprender-gpu", version = "0.71.0", features = ["cuda"], optional = true, package = "aprender-gpu" } [features] default = [] diff --git a/crates/aprender-compute/Cargo.toml b/crates/aprender-compute/Cargo.toml index b4b5f39515..f4bee6047f 100644 --- a/crates/aprender-compute/Cargo.toml +++ b/crates/aprender-compute/Cargo.toml @@ -41,11 +41,11 @@ futures-intrusive = { version = "0.5", optional = true } wasm-bindgen-futures = { version = "0.4", optional = true } wasm-bindgen = { version = "0.2", optional = true } # Native CUDA monitoring via trueno-gpu (TRUENO-SPEC-010) -trueno-gpu = { version = "0.70.3", path = "../aprender-gpu", optional = true, package = "aprender-gpu" } +trueno-gpu = { version = "0.71.0", path = "../aprender-gpu", optional = true, package = "aprender-gpu" } # PMAT-336: Provable contracts compile-time enforcement provable-contracts-macros = { workspace = true } # P1a: Sovereign GEMM microkernel codegen (compile-time shape specialization) -trueno-gemm-codegen = { version = "0.70.3", path = "../aprender-gemm-codegen", package = "aprender-gemm-codegen" } +trueno-gemm-codegen = { version = "0.71.0", path = "../aprender-gemm-codegen", package = "aprender-gemm-codegen" } # Tracing/profiling support (renacer integration) tracing = { version = "0.1", optional = true } tracing-subscriber = { version = "0.3", features = ["env-filter", "fmt"], optional = true } diff --git a/crates/aprender-contracts-cli/Cargo.toml b/crates/aprender-contracts-cli/Cargo.toml index b473d77e2d..ddf13b7a19 100644 --- a/crates/aprender-contracts-cli/Cargo.toml +++ b/crates/aprender-contracts-cli/Cargo.toml @@ -18,7 +18,7 @@ name = "pv-sat" path = "src/bin/pv-sat/main.rs" [dependencies] -aprender-contracts = { path = "../aprender-contracts", version = "0.70.3" } +aprender-contracts = { path = "../aprender-contracts", version = "0.71.0" } # #3745 S1: `--model` is declared through the shared role type `ModelPath`, so # `apr surface` reads `apr pv verify-structure --model` as a model argument. batuta-common = { workspace = true, features = ["cli-roles"] } diff --git a/crates/aprender-contracts/Cargo.toml b/crates/aprender-contracts/Cargo.toml index 42e836f0d0..2f77b55308 100644 --- a/crates/aprender-contracts/Cargo.toml +++ b/crates/aprender-contracts/Cargo.toml @@ -26,7 +26,7 @@ kani = [] name = "provable_contracts" [dependencies] -provable-contracts-macros = { path = "../aprender-contracts-macros", version = "0.70.3", package = "aprender-contracts-macros" } +provable-contracts-macros = { path = "../aprender-contracts-macros", version = "0.71.0", package = "aprender-contracts-macros" } regex = "1" sha2 = { workspace = true } blake3 = "1" diff --git a/crates/aprender-core/Cargo.toml b/crates/aprender-core/Cargo.toml index e0888fa54e..94d37a7a31 100644 --- a/crates/aprender-core/Cargo.toml +++ b/crates/aprender-core/Cargo.toml @@ -135,7 +135,7 @@ trueno-quant = { workspace = true } # APR-2231: sovereign `.apr` container leaf. aprender-core From-wraps its # AprFormatError into AprenderError and re-exports it (no API break). Leaf has # no dependency back on core, so this introduces no publish cycle. -apr-format = { path = "../apr-format", version = "0.70" } +apr-format = { path = "../apr-format", version = "0.71" } # NOTE: trueno-rag (=aprender-rag) removed (APR-MONO self-containment): aprender-rag # depends on aprender-core + aprender-serve, so core→rag closed a core→rag→serve→core diff --git a/crates/aprender-cuda-edge/Cargo.toml b/crates/aprender-cuda-edge/Cargo.toml index f2c93a0a52..c0ce852965 100644 --- a/crates/aprender-cuda-edge/Cargo.toml +++ b/crates/aprender-cuda-edge/Cargo.toml @@ -15,7 +15,7 @@ name = "trueno_cuda_edge" [dependencies] thiserror = "2.0" serde = { version = "1.0", features = ["derive"] } -trueno-gpu = { version = "0.70.3", path = "../aprender-gpu", package = "aprender-gpu", optional = true } +trueno-gpu = { version = "0.71.0", path = "../aprender-gpu", package = "aprender-gpu", optional = true } [dev-dependencies] proptest = "1.6" diff --git a/crates/aprender-explain/Cargo.toml b/crates/aprender-explain/Cargo.toml index 55ba1df834..e43ee5440f 100644 --- a/crates/aprender-explain/Cargo.toml +++ b/crates/aprender-explain/Cargo.toml @@ -25,7 +25,7 @@ presentar-terminal = { workspace = true } crossterm = "0.28" # Internal dependency for PTX generation -trueno-gpu = { version = "0.70.3", path = "../aprender-gpu", package = "aprender-gpu", features = ["cuda"] } +trueno-gpu = { version = "0.71.0", path = "../aprender-gpu", package = "aprender-gpu", features = ["cuda"] } [build-dependencies] aprender-build-sha = { workspace = true } # #4219: APR_GIT_SHA for --version diff --git a/crates/aprender-graph/Cargo.toml b/crates/aprender-graph/Cargo.toml index 54b6601687..0717b4e018 100644 --- a/crates/aprender-graph/Cargo.toml +++ b/crates/aprender-graph/Cargo.toml @@ -28,7 +28,7 @@ anyhow = "1" thiserror = "2" # Phase 2+ dependencies -aprender = { path = "../aprender-core", version = "0.70.3", package = "aprender-core" } # Sparse matrix ops, PageRank +aprender = { path = "../aprender-core", version = "0.71.0", package = "aprender-core" } # Sparse matrix ops, PageRank trueno = { workspace = true } # Phase 3: GPU acceleration (optional - requires wgpu-capable hardware) diff --git a/crates/aprender-monte-carlo/Cargo.toml b/crates/aprender-monte-carlo/Cargo.toml index ba0c0f7b8f..e109f5c607 100644 --- a/crates/aprender-monte-carlo/Cargo.toml +++ b/crates/aprender-monte-carlo/Cargo.toml @@ -18,7 +18,7 @@ path = "src/lib.rs" name = "aprender-monte-carlo" [dependencies] -aprender = { path = "../../", version = "0.70.3" } +aprender = { path = "../../", version = "0.71.0" } clap = { version = "4", features = ["derive"] } rand = "0.9" rand_chacha = "0.9" diff --git a/crates/aprender-orchestrate/Cargo.toml b/crates/aprender-orchestrate/Cargo.toml index 417a9e599d..a242eaa297 100644 --- a/crates/aprender-orchestrate/Cargo.toml +++ b/crates/aprender-orchestrate/Cargo.toml @@ -132,7 +132,7 @@ pacha = { workspace = true, optional = true } realizar = { workspace = true, optional = true } # ML algorithms and training (native-only) -aprender = { path = "../aprender-core", version = "0.70.3", package = "aprender-core", optional = true } +aprender = { path = "../aprender-core", version = "0.71.0", package = "aprender-core", optional = true } entrenar = { workspace = true, optional = true } alimentar = { workspace = true, optional = true } diff --git a/crates/aprender-profile/Cargo.toml b/crates/aprender-profile/Cargo.toml index 2796a26d54..b9fa405068 100644 --- a/crates/aprender-profile/Cargo.toml +++ b/crates/aprender-profile/Cargo.toml @@ -74,7 +74,7 @@ trueno = { workspace = true } # Machine learning for anomaly detection (Sprint 23) # GH-581: Use path dep to avoid trueno version skew with local workspace -aprender = { path = "../aprender-core", version = "0.70.3", package = "aprender-core" } +aprender = { path = "../aprender-core", version = "0.71.0", package = "aprender-core" } # Lock-free data structures for ring buffer (Sprint 40) crossbeam = { version = "0.8", default-features = false, features = ["crossbeam-channel", "crossbeam-queue", "std"] } @@ -112,7 +112,7 @@ crossterm = "0.28" # SIMD-accelerated visualization primitives (sister project) # NOTE: `monitor` feature removed upstream (#1979); this crate uses only viz primitives. -trueno-viz = { version = "0.70.3", path = "../aprender-viz", package = "aprender-viz", default-features = false } +trueno-viz = { version = "0.71.0", path = "../aprender-viz", package = "aprender-viz", default-features = false } # OpenTelemetry for distributed tracing (Sprint 30) opentelemetry = { version = "0.31.0", optional = true } diff --git a/crates/aprender-qa-cli/Cargo.toml b/crates/aprender-qa-cli/Cargo.toml index 3304f825e1..3040345664 100644 --- a/crates/aprender-qa-cli/Cargo.toml +++ b/crates/aprender-qa-cli/Cargo.toml @@ -23,10 +23,10 @@ serde_json = { workspace = true } serde_yaml = { workspace = true } clap = { workspace = true } chrono = { workspace = true } -aprender-qa-gen = { version = "0.70.3", path = "../aprender-qa-gen" } -aprender-qa-runner = { version = "0.70.3", path = "../aprender-qa-runner" } -aprender-qa-report = { version = "0.70.3", path = "../aprender-qa-report" } -aprender-qa-certify = { version = "0.70.3", path = "../aprender-qa-certify" } +aprender-qa-gen = { version = "0.71.0", path = "../aprender-qa-gen" } +aprender-qa-runner = { version = "0.71.0", path = "../aprender-qa-runner" } +aprender-qa-report = { version = "0.71.0", path = "../aprender-qa-report" } +aprender-qa-certify = { version = "0.71.0", path = "../aprender-qa-certify" } ctrlc = "3.4" safetensors = { workspace = true } colored = "2.2" diff --git a/crates/aprender-qa-report/Cargo.toml b/crates/aprender-qa-report/Cargo.toml index da7d7cf491..d9e3f0e482 100644 --- a/crates/aprender-qa-report/Cargo.toml +++ b/crates/aprender-qa-report/Cargo.toml @@ -18,8 +18,8 @@ anyhow = { workspace = true } chrono = { workspace = true } colored = "2.2" csv = "1.3" -aprender-qa-gen = { version = "0.70.3", path = "../aprender-qa-gen" } -aprender-qa-runner = { version = "0.70.3", path = "../aprender-qa-runner" } +aprender-qa-gen = { version = "0.71.0", path = "../aprender-qa-gen" } +aprender-qa-runner = { version = "0.71.0", path = "../aprender-qa-runner" } [dev-dependencies] proptest = { workspace = true } diff --git a/crates/aprender-qa-runner/Cargo.toml b/crates/aprender-qa-runner/Cargo.toml index d68cde1569..ebe63bf531 100644 --- a/crates/aprender-qa-runner/Cargo.toml +++ b/crates/aprender-qa-runner/Cargo.toml @@ -22,14 +22,14 @@ anyhow = { workspace = true } tokio = { workspace = true } chrono = { workspace = true, features = ["serde"] } tracing = { workspace = true } -aprender-qa-gen = { version = "0.70.3", path = "../aprender-qa-gen" } +aprender-qa-gen = { version = "0.71.0", path = "../aprender-qa-gen" } hostname = "0.4" rayon = "1.10" sha2 = "0.10" regex = { workspace = true } num_cpus = "1.16" safetensors = { workspace = true } -jugar-probar = { version = "0.70.3", path = "../aprender-test-lib", package = "aprender-test-lib", features = ["llm-types"] } +jugar-probar = { version = "0.71.0", path = "../aprender-test-lib", package = "aprender-test-lib", features = ["llm-types"] } # Test fixtures dependencies (optional) tempfile = { version = "3", optional = true } half = { version = "2.4", optional = true } diff --git a/crates/aprender-rag-cli/Cargo.toml b/crates/aprender-rag-cli/Cargo.toml index 270548bed4..fc81f0e7a5 100644 --- a/crates/aprender-rag-cli/Cargo.toml +++ b/crates/aprender-rag-cli/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "aprender-rag-cli" -version = "0.70.3" +version = "0.71.0" edition = "2021" authors = ["Pragmatic AI Labs"] description = "CLI for Trueno-RAG pipeline" diff --git a/crates/aprender-rag/Cargo.toml b/crates/aprender-rag/Cargo.toml index c52a233be4..f7d2e537c9 100644 --- a/crates/aprender-rag/Cargo.toml +++ b/crates/aprender-rag/Cargo.toml @@ -63,7 +63,7 @@ fastembed = { version = "5", optional = true } # NVIDIA Embed Nemotron 8B via realizar GGUF infrastructure (GH-3) # Large model (~4-32GB depending on quantization), requires GGUF file -realizar = { version = "0.70.3", path = "../aprender-serve", package = "aprender-serve", default-features = false, optional = true } +realizar = { version = "0.71.0", path = "../aprender-serve", package = "aprender-serve", default-features = false, optional = true } # Eval: LLM-as-judge via Anthropic API (PMAT-015) diff --git a/crates/aprender-registry/Cargo.toml b/crates/aprender-registry/Cargo.toml index 774e69f5df..7e11f8d431 100644 --- a/crates/aprender-registry/Cargo.toml +++ b/crates/aprender-registry/Cargo.toml @@ -42,7 +42,7 @@ zstd = { version = "0.13", optional = true } clap = { version = "4", features = ["derive"], optional = true } # Ecosystem integration -aprender = { path = "../aprender-core", version = "0.70.3", package = "aprender-core", optional = true } +aprender = { path = "../aprender-core", version = "0.71.0", package = "aprender-core", optional = true } alimentar = { workspace = true, optional = true } # HTTP client (optional) diff --git a/crates/aprender-serve/Cargo.toml b/crates/aprender-serve/Cargo.toml index 863925584c..6d80fe7e9e 100644 --- a/crates/aprender-serve/Cargo.toml +++ b/crates/aprender-serve/Cargo.toml @@ -51,7 +51,7 @@ trueno = { workspace = true, features = ["gpu"] } unicode-properties = { workspace = true } # CUDA PTX generation and runtime for NVIDIA GPUs (optional, pure Rust, no LLVM/nvcc) # v0.4.12: BatchedQ6K, MultiWarp, RopeNeox, FlashDecoding kernels -trueno-gpu = { version = "0.70.3", path = "../aprender-gpu", package = "aprender-gpu", optional = true, features = ["cuda"] } # APR-MONO single-source-of-truth +trueno-gpu = { version = "0.71.0", path = "../aprender-gpu", package = "aprender-gpu", optional = true, features = ["cuda"] } # APR-MONO single-source-of-truth # KV cache storage (for attention caching) trueno-db = { workspace = true, optional = true } # K-quantization formats (Q4_K, Q5_K, Q6_K) - Toyota Way: ONE source of truth @@ -65,7 +65,7 @@ renacer-core = { workspace = true } # ML algorithms and .apr model format (optional for aprender model serving) # v0.24.1: chat templates, GGUF tokenizer preservation (local may be newer) -aprender = { path = "../aprender-core", version = "0.70.3", package = "aprender-core", optional = true } +aprender = { path = "../aprender-core", version = "0.71.0", package = "aprender-core", optional = true } # Data loading library (optional for data pipeline examples) alimentar = { workspace = true, optional = true } diff --git a/crates/aprender-shell/Cargo.toml b/crates/aprender-shell/Cargo.toml index 60e9c62258..e89a05c27e 100644 --- a/crates/aprender-shell/Cargo.toml +++ b/crates/aprender-shell/Cargo.toml @@ -24,7 +24,7 @@ harness = false [dependencies] # Use path for local dev, crates.io for release -aprender = { path = "../aprender-core", version = "0.70.3", package = "aprender-core", features = ["format-encryption", "format-compression", "format-homomorphic"] } +aprender = { path = "../aprender-core", version = "0.71.0", package = "aprender-core", features = ["format-encryption", "format-compression", "format-homomorphic"] } clap = { version = "4", features = ["derive"] } dirs = "5" serde = { version = "1", features = ["derive"] } diff --git a/crates/aprender-simulate/Cargo.toml b/crates/aprender-simulate/Cargo.toml index 1ecd9130cc..f7dd0f6c5b 100644 --- a/crates/aprender-simulate/Cargo.toml +++ b/crates/aprender-simulate/Cargo.toml @@ -59,7 +59,7 @@ crossterm = { version = "0.28", optional = true } # ComputeBlocks from presentar-terminal (SIMD-optimized TUI widgets) # Path deps to avoid jugar-probar prod dep chain (CB-081) presentar-terminal = { workspace = true, optional = true } -presentar-core = { version = "0.70.3", path = "../aprender-present-core", package = "aprender-present-core", default-features = false, optional = true } +presentar-core = { version = "0.71.0", path = "../aprender-present-core", package = "aprender-present-core", default-features = false, optional = true } # WASM (optional) wasm-bindgen = { version = "0.2", optional = true } diff --git a/crates/aprender-test-cli/Cargo.toml b/crates/aprender-test-cli/Cargo.toml index d11ddf4e8e..5c1bf94b47 100644 --- a/crates/aprender-test-cli/Cargo.toml +++ b/crates/aprender-test-cli/Cargo.toml @@ -35,7 +35,7 @@ path = "src/main.rs" [dependencies] # Core library (was: jugar-probar) — dep key preserves Rust identifier -jugar-probar = { path = "../aprender-test-lib", version = "0.70.3", package = "aprender-test-lib" } +jugar-probar = { path = "../aprender-test-lib", version = "0.71.0", package = "aprender-test-lib" } # CLI framework clap = { workspace = true } diff --git a/crates/aprender-train-bench/Cargo.toml b/crates/aprender-train-bench/Cargo.toml index 6cf9a9e95e..1513b6f14b 100644 --- a/crates/aprender-train-bench/Cargo.toml +++ b/crates/aprender-train-bench/Cargo.toml @@ -20,7 +20,7 @@ workspace = true [dependencies] entrenar = { workspace = true } -entrenar-common = { version = "0.70.3", path = "../aprender-train-common", package = "aprender-train-common" } +entrenar-common = { version = "0.71.0", path = "../aprender-train-common", package = "aprender-train-common" } clap = { version = "4.5", features = ["derive"] } serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" diff --git a/crates/aprender-train-distill/Cargo.toml b/crates/aprender-train-distill/Cargo.toml index 9266525ad1..a166a97bf5 100644 --- a/crates/aprender-train-distill/Cargo.toml +++ b/crates/aprender-train-distill/Cargo.toml @@ -33,7 +33,7 @@ shard-batch-source = [] [dependencies] entrenar = { workspace = true } -entrenar-common = { version = "0.70.3", path = "../aprender-train-common", package = "aprender-train-common" } +entrenar-common = { version = "0.71.0", path = "../aprender-train-common", package = "aprender-train-common" } clap = { version = "4.5", features = ["derive"] } serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" diff --git a/crates/aprender-train-inspect/Cargo.toml b/crates/aprender-train-inspect/Cargo.toml index 6ae0577c29..66144a3a45 100644 --- a/crates/aprender-train-inspect/Cargo.toml +++ b/crates/aprender-train-inspect/Cargo.toml @@ -20,7 +20,7 @@ workspace = true [dependencies] entrenar = { workspace = true } -entrenar-common = { version = "0.70.3", path = "../aprender-train-common", package = "aprender-train-common" } +entrenar-common = { version = "0.71.0", path = "../aprender-train-common", package = "aprender-train-common" } clap = { version = "4.5", features = ["derive"] } serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" diff --git a/crates/aprender-train-lora/Cargo.toml b/crates/aprender-train-lora/Cargo.toml index 92e0a9c6c8..ab58fd6dcb 100644 --- a/crates/aprender-train-lora/Cargo.toml +++ b/crates/aprender-train-lora/Cargo.toml @@ -20,7 +20,7 @@ workspace = true [dependencies] entrenar = { workspace = true } -entrenar-common = { version = "0.70.3", path = "../aprender-train-common", package = "aprender-train-common" } +entrenar-common = { version = "0.71.0", path = "../aprender-train-common", package = "aprender-train-common" } clap = { version = "4.5", features = ["derive"] } serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" diff --git a/crates/aprender-train-shell/Cargo.toml b/crates/aprender-train-shell/Cargo.toml index 63caba08b3..4f617a064d 100644 --- a/crates/aprender-train-shell/Cargo.toml +++ b/crates/aprender-train-shell/Cargo.toml @@ -20,7 +20,7 @@ workspace = true [dependencies] entrenar = { workspace = true } -entrenar-common = { version = "0.70.3", path = "../aprender-train-common", package = "aprender-train-common" } +entrenar-common = { version = "0.71.0", path = "../aprender-train-common", package = "aprender-train-common" } clap = { version = "4.5", features = ["derive"] } rustyline = "14" serde = { version = "1.0", features = ["derive"] } diff --git a/crates/aprender-train/Cargo.toml b/crates/aprender-train/Cargo.toml index 336353e85d..18a6ae4d78 100644 --- a/crates/aprender-train/Cargo.toml +++ b/crates/aprender-train/Cargo.toml @@ -80,9 +80,9 @@ provable-contracts-macros = { workspace = true } # Core PAIML stack ndarray = "0.16" trueno = { workspace = true, features = ["parallel"] } -trueno-gpu = { version = "0.70.3", path = "../aprender-gpu", package = "aprender-gpu", optional = true } # CUDA compute (APR-MONO single-source-of-truth) +trueno-gpu = { version = "0.71.0", path = "../aprender-gpu", package = "aprender-gpu", optional = true } # CUDA compute (APR-MONO single-source-of-truth) realizar = { workspace = true, optional = true } -aprender = { path = "../aprender-core", version = "0.70.3", package = "aprender-core" } # ML interpret module + .apr format + tokenizers +aprender = { path = "../aprender-core", version = "0.71.0", package = "aprender-core" } # ML interpret module + .apr format + tokenizers trueno-viz = { workspace = true, features = ["terminal"] } batuta-common = { workspace = true, optional = true } # WithDimensions trait for `viz` feature (APR-MONO §S #1978) presentar-terminal = { workspace = true, optional = true } diff --git a/crates/aprender-tsp/Cargo.toml b/crates/aprender-tsp/Cargo.toml index 5d713a4e76..daad82e1a0 100644 --- a/crates/aprender-tsp/Cargo.toml +++ b/crates/aprender-tsp/Cargo.toml @@ -18,7 +18,7 @@ path = "src/lib.rs" name = "aprender-tsp" [dependencies] -aprender = { path = "../../", version = "0.70.3" } +aprender = { path = "../../", version = "0.71.0" } clap = { version = "4", features = ["derive"] } rand = "0.9" crc32fast = "1.4" diff --git a/crates/aprender-verify-ml/Cargo.toml b/crates/aprender-verify-ml/Cargo.toml index 494b5f9799..d5d62258ef 100644 --- a/crates/aprender-verify-ml/Cargo.toml +++ b/crates/aprender-verify-ml/Cargo.toml @@ -56,7 +56,7 @@ uuid = { version = "1.11", features = ["v4", "serde"] } trueno = { workspace = true } # Machine learning for bug prediction (optional) -aprender = { path = "../aprender-core", version = "0.70.3", package = "aprender-core", optional = true } +aprender = { path = "../aprender-core", version = "0.71.0", package = "aprender-core", optional = true } # LLM fine-tuning integration (optional) entrenar = { workspace = true, optional = true } diff --git a/crates/aprender-viz/Cargo.toml b/crates/aprender-viz/Cargo.toml index cb046e9df2..8aba030023 100644 --- a/crates/aprender-viz/Cargo.toml +++ b/crates/aprender-viz/Cargo.toml @@ -14,7 +14,7 @@ categories = ["visualization", "graphics", "science", "wasm"] [dependencies] # Core SIMD/GPU acceleration -trueno = { version = "0.70.3", path = "../aprender-compute", package = "aprender-compute", default-features = false } +trueno = { version = "0.71.0", path = "../aprender-compute", package = "aprender-compute", default-features = false } # Shared Batuta stack utilities (display traits, formatting) batuta-common = { workspace = true } @@ -52,10 +52,10 @@ kurbo = { version = "0.13", optional = true } resvg = { version = "0.45", optional = true, default-features = false } # Optional: ML integration -aprender = { path = "../aprender-core", version = "0.70.3", package = "aprender-core", optional = true } +aprender = { path = "../aprender-core", version = "0.71.0", package = "aprender-core", optional = true } # Optional: Graph integration -trueno-graph = { version = "0.70.3", path = "../aprender-graph", package = "aprender-graph", default-features = false, optional = true } +trueno-graph = { version = "0.71.0", path = "../aprender-graph", package = "aprender-graph", default-features = false, optional = true } # Optional: Database integration trueno-db = { workspace = true, optional = true } diff --git a/crates/facades/Cargo.lock b/crates/facades/Cargo.lock index b0fceb27f5..18a4b441c2 100644 --- a/crates/facades/Cargo.lock +++ b/crates/facades/Cargo.lock @@ -13,7 +13,7 @@ dependencies = [ [[package]] name = "aprender-contracts" -version = "0.70.3" +version = "0.71.0" dependencies = [ "aprender-contracts-macros", "blake3", @@ -29,7 +29,7 @@ dependencies = [ [[package]] name = "aprender-contracts-macros" -version = "0.70.3" +version = "0.71.0" dependencies = [ "proc-macro2", "quote", diff --git a/crates/facades/provable-contracts-macros/Cargo.toml b/crates/facades/provable-contracts-macros/Cargo.toml index 2cbdf2ae15..aa240a83a2 100644 --- a/crates/facades/provable-contracts-macros/Cargo.toml +++ b/crates/facades/provable-contracts-macros/Cargo.toml @@ -30,7 +30,7 @@ path = "src/lib.rs" # facade resolves. `scripts/check_facade_compat.sh` fails if this version ever # drifts from the aprender workspace version — these manifests are excluded # from the root workspace, so `cargo set-version` does NOT reach them. -upstream = { path = "../../aprender-contracts-macros", version = "0.70.3", package = "aprender-contracts-macros" } +upstream = { path = "../../aprender-contracts-macros", version = "0.71.0", package = "aprender-contracts-macros" } [features] default = [] diff --git a/crates/facades/provable-contracts/Cargo.toml b/crates/facades/provable-contracts/Cargo.toml index 19102f8f5a..e5f3df59cb 100644 --- a/crates/facades/provable-contracts/Cargo.toml +++ b/crates/facades/provable-contracts/Cargo.toml @@ -31,7 +31,7 @@ path = "src/lib.rs" # facade resolves. `scripts/check_facade_compat.sh` fails if this version ever # drifts from the aprender workspace version — these manifests are excluded # from the root workspace, so `cargo set-version` does NOT reach them. -upstream = { path = "../../aprender-contracts", version = "0.70.3", package = "aprender-contracts" } +upstream = { path = "../../aprender-contracts", version = "0.71.0", package = "aprender-contracts" } [dev-dependencies] # Used by three of the vendored 0.3.1 example programs. They were regular diff --git a/evidence/crux/0.71.0/prompt-certification-inventory.json b/evidence/crux/0.71.0/prompt-certification-inventory.json new file mode 100644 index 0000000000..93705629cf --- /dev/null +++ b/evidence/crux/0.71.0/prompt-certification-inventory.json @@ -0,0 +1,13 @@ +[ + {"model": "Qwen3.5-2B", + "source": {"repo": "Qwen/Qwen3.5-2B", "revision": "15852e8c16360a2fea060d615a32b45270f8a8fc"}, + "bf16_gguf": "dc11be2ca4519954a3e4b013ef59497b5522209987d67891851113eda89619f1", + "quants": {"Q4_K_M": "aaf42c8b7c3cab2bf3d69c355048d4a0ee9973d48f16c731c0520ee914699223"}, + "thinking": ["on", "off"]}, + {"model": "Qwen3.5-4B", + "source": {"repo": "Qwen/Qwen3.5-4B", "revision": "851bf6e806efd8d0a36b00ddf55e13ccb7b8cd0a"}, + "bf16_gguf": "9e6e2841a75f503ccb330831832fd7861266e187e0dbf149a954219ccb8c197a", + "quants": {"Q4_K_M": "00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4", + "UD-Q4_K_XL": "b252c5610a42ca82d20fe2a12813e9d069eed89292907e26c783eeb0bc961bc7"}, + "thinking": ["on", "off"]} +] diff --git a/evidence/crux/0.71.0/prompt-certification.json b/evidence/crux/0.71.0/prompt-certification.json new file mode 100644 index 0000000000..0191ce6dab --- /dev/null +++ b/evidence/crux/0.71.0/prompt-certification.json @@ -0,0 +1,8906 @@ +{ + "schema": "crux-prompt-certification/v1", + "prompts": "scripts/crux_inference_prompts.v2.json", + "prompts_sha256": "cea80df5e785213667e27697fe3d139b99dcba95cfc6e970d76ac005d50fc806", + "apr_commit": "812dbe3eb5a713f076f2e7f313c7ec6b14027a55", + "inventory_sha256": "b8e4c4626b20b7764041f5b5fe6d8a98166a952a64b3a844207b796b388778e0", + "manifests": { + "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl": "f56747953b33bb2e4adac4bac12be902224a202b2a07c35ede3e47121d178728", + "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl": "6a5657345a6a61ee95ca7a8d7237d7f9748ec8e6d652bdd95be042a586a4e45f" + }, + "admitted": { + "Qwen3.5-2B/Q4_K_M": [], + "Qwen3.5-4B/Q4_K_M": [ + "ctl-2plus2", + "ctl-code-add", + "fact-capital-japan", + "fact-capital-france", + "golden-paris", + "nonascii-water-kanji", + "arith-17x23", + "arith-precedence", + "arith-word-apples", + "hard-not-prime", + "hard-reverse-word", + "hard-count-letters", + "recall-arith-chain", + "code-count-vowels", + "long-primes-10", + "long-countdown-15" + ], + "Qwen3.5-4B/UD-Q4_K_XL": [ + "ctl-2plus2", + "ctl-code-add", + "fact-capital-japan", + "fact-capital-france", + "golden-paris", + "nonascii-water-kanji", + "arith-17x23", + "arith-precedence", + "arith-word-apples", + "hard-not-prime", + "hard-count-letters", + "recall-arith-chain", + "long-primes-10", + "long-countdown-15" + ] + }, + "admitted_by_sha": { + "aaf42c8b7c3cab2bf3d69c355048d4a0ee9973d48f16c731c0520ee914699223": [], + "00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4": [ + "ctl-2plus2", + "ctl-code-add", + "fact-capital-japan", + "fact-capital-france", + "golden-paris", + "nonascii-water-kanji", + "arith-17x23", + "arith-precedence", + "arith-word-apples", + "hard-not-prime", + "hard-reverse-word", + "hard-count-letters", + "recall-arith-chain", + "code-count-vowels", + "long-primes-10", + "long-countdown-15" + ], + "b252c5610a42ca82d20fe2a12813e9d069eed89292907e26c783eeb0bc961bc7": [ + "ctl-2plus2", + "ctl-code-add", + "fact-capital-japan", + "fact-capital-france", + "golden-paris", + "nonascii-water-kanji", + "arith-17x23", + "arith-precedence", + "arith-word-apples", + "hard-not-prime", + "hard-count-letters", + "recall-arith-chain", + "long-primes-10", + "long-countdown-15" + ] + }, + "admitted_by_sha_thinking": { + "aaf42c8b7c3cab2bf3d69c355048d4a0ee9973d48f16c731c0520ee914699223": { + "on": [], + "off": [ + "ctl-2plus2", + "ctl-recall-7", + "ctl-code-add", + "fact-capital-japan", + "fact-capital-france", + "golden-paris", + "arith-17x23", + "arith-precedence", + "arith-word-apples", + "hard-add-4digit", + "hard-not-prime", + "hard-count-letters", + "recall-cats", + "recall-name", + "recall-arith-chain", + "code-is-palindrome", + "code-fib", + "code-count-vowels", + "long-primes-10", + "long-countdown-15" + ] + }, + "00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4": { + "on": [ + "ctl-2plus2", + "ctl-code-add", + "fact-capital-japan", + "fact-capital-france", + "golden-paris", + "nonascii-water-kanji", + "arith-17x23", + "arith-precedence", + "arith-word-apples", + "hard-not-prime", + "hard-reverse-word", + "hard-count-letters", + "recall-arith-chain", + "code-count-vowels", + "long-primes-10", + "long-countdown-15" + ], + "off": [ + "ctl-2plus2", + "ctl-recall-7", + "ctl-code-add", + "fact-capital-japan", + "fact-capital-france", + "golden-paris", + "nonascii-water-kanji", + "arith-17x23", + "arith-precedence", + "arith-word-apples", + "hard-add-4digit", + "hard-not-prime", + "hard-reverse-word", + "hard-count-letters", + "recall-cats", + "recall-name", + "recall-arith-chain", + "code-is-palindrome", + "code-fib", + "code-count-vowels", + "long-primes-10", + "long-countdown-15" + ] + }, + "b252c5610a42ca82d20fe2a12813e9d069eed89292907e26c783eeb0bc961bc7": { + "on": [ + "ctl-2plus2", + "ctl-code-add", + "fact-capital-japan", + "fact-capital-france", + "golden-paris", + "nonascii-water-kanji", + "arith-17x23", + "arith-precedence", + "arith-word-apples", + "hard-not-prime", + "hard-count-letters", + "recall-arith-chain", + "long-primes-10", + "long-countdown-15" + ], + "off": [ + "ctl-2plus2", + "ctl-recall-7", + "ctl-code-add", + "fact-capital-japan", + "fact-capital-france", + "golden-paris", + "nonascii-water-kanji", + "arith-17x23", + "arith-precedence", + "arith-word-apples", + "hard-add-4digit", + "hard-not-prime", + "hard-reverse-word", + "hard-count-letters", + "recall-cats", + "recall-name", + "recall-arith-chain", + "code-is-palindrome", + "code-fib", + "code-count-vowels", + "long-primes-10", + "long-countdown-15" + ] + } + }, + "rejected": { + "Qwen3.5-2B/Q4_K_M": { + "ctl-2plus2": "ggml@bf16 llama.cpp serve run thinking=on on gx10: unclosed think (budget exhausted)", + "ctl-recall-7": "ggml@bf16 llama.cpp serve run thinking=on on gx10: unclosed think (budget exhausted)", + "ctl-code-add": "hf@bf16 thinking=on: no row", + "fact-capital-japan": "ggml@bf16 llama.cpp serve run thinking=on on gx10: unclosed think (budget exhausted)", + "fact-capital-france": "hf@bf16 thinking=on: no row", + "golden-paris": "ggml@bf16 thinking=on: no row", + "fact-water-formula": "ggml@bf16 llama.cpp serve run thinking=on on gx10: unclosed think (budget exhausted)", + "nonascii-water-kanji": "ggml@bf16 thinking=on: no row", + "arith-17x23": "ggml@bf16 llama.cpp serve run thinking=on on gx10: unclosed think (budget exhausted)", + "arith-precedence": "hf@bf16 thinking=on: no row", + "arith-word-apples": "hf@bf16 thinking=on: no row", + "hard-add-4digit": "hf@bf16 thinking=on: no row", + "hard-not-prime": "ggml@bf16 thinking=on: no row", + "hard-reverse-word": "ggml@bf16 thinking=on: no row", + "hard-count-letters": "ggml@bf16 thinking=on: no row", + "recall-cats": "ggml@bf16 thinking=on: no row", + "recall-name": "ggml@bf16 thinking=on: no row", + "recall-arith-chain": "ggml@bf16 thinking=on: no row", + "code-is-palindrome": "ggml@bf16 thinking=on: no row", + "code-fib": "ggml@bf16 thinking=on: no row", + "code-count-vowels": "ggml@bf16 thinking=on: no row", + "long-primes-10": "ggml@bf16 thinking=on: no row", + "long-countdown-15": "ggml@bf16 thinking=on: no row" + }, + "Qwen3.5-4B/Q4_K_M": { + "ctl-recall-7": "ggml@quant llama.cpp serve run thinking=on on gx10: unclosed think (budget exhausted)", + "fact-water-formula": "ggml@bf16 llama.cpp serve run thinking=on on gx10: unclosed think (budget exhausted)", + "hard-add-4digit": "ggml@quant llama.cpp serve run thinking=on on gx10: unclosed think (budget exhausted)", + "recall-cats": "ggml@bf16 llama.cpp serve run thinking=on on gx10: unclosed think (budget exhausted)", + "recall-name": "ggml@quant llama.cpp serve run thinking=on on gx10: unclosed think (budget exhausted)", + "code-is-palindrome": "vllm@bf16 vllm chat thinking=on on gx10: tests_failed: AssertionError", + "code-fib": "ggml@bf16 llama.cpp serve run thinking=on on gx10: unclosed think (budget exhausted)" + }, + "Qwen3.5-4B/UD-Q4_K_XL": { + "ctl-recall-7": "ggml@quant llama.cpp serve run thinking=on on gx10: unclosed think (budget exhausted)", + "fact-water-formula": "ggml@bf16 llama.cpp serve run thinking=on on gx10: unclosed think (budget exhausted)", + "hard-add-4digit": "ggml@quant llama.cpp serve run thinking=on on gx10: unclosed think (budget exhausted)", + "hard-reverse-word": "ggml@quant llama.cpp serve run thinking=on on gx10: mismatch", + "recall-cats": "ggml@bf16 llama.cpp serve run thinking=on on gx10: unclosed think (budget exhausted)", + "recall-name": "ggml@quant llama.cpp serve run thinking=on on gx10: unclosed think (budget exhausted)", + "code-is-palindrome": "vllm@bf16 vllm chat thinking=on on gx10: tests_failed: AssertionError", + "code-fib": "ggml@bf16 llama.cpp serve run thinking=on on gx10: unclosed think (budget exhausted)", + "code-count-vowels": "ggml@quant llama.cpp serve run thinking=on on gx10: unclosed think (budget exhausted)" + } + }, + "uncontrolled": [ + "Qwen3.5-2B/Q4_K_M" + ], + "uncontrolled_detail": [ + { + "model": "Qwen3.5-2B/Q4_K_M", + "sha256": "aaf42c8b7c3cab2bf3d69c355048d4a0ee9973d48f16c731c0520ee914699223", + "thinking": "on", + "verbs": [ + "chat", + "code", + "run", + "serve run", + "serve stream" + ] + } + ], + "think_closure": { + "Qwen3.5-2B/Q4_K_M|ctl-2plus2": { + "ggml@bf16:llama.cpp": "unclosed" + }, + "Qwen3.5-2B/Q4_K_M|ctl-recall-7": { + "ggml@bf16:llama.cpp": "unclosed" + }, + "Qwen3.5-2B/Q4_K_M|ctl-code-add": { + "ggml@bf16:llama.cpp": "closed" + }, + "Qwen3.5-2B/Q4_K_M|fact-capital-japan": { + "ggml@bf16:llama.cpp": "unclosed" + }, + "Qwen3.5-2B/Q4_K_M|fact-capital-france": { + "ggml@bf16:llama.cpp": "closed" + }, + "Qwen3.5-2B/Q4_K_M|fact-water-formula": { + "ggml@bf16:llama.cpp": "unclosed" + }, + "Qwen3.5-2B/Q4_K_M|arith-17x23": { + "ggml@bf16:llama.cpp": "unclosed" + }, + "Qwen3.5-2B/Q4_K_M|arith-precedence": { + "ggml@bf16:llama.cpp": "closed" + }, + "Qwen3.5-2B/Q4_K_M|arith-word-apples": { + "ggml@bf16:llama.cpp": "closed" + }, + "Qwen3.5-2B/Q4_K_M|hard-add-4digit": { + "ggml@bf16:llama.cpp": "closed" + }, + "Qwen3.5-4B/Q4_K_M|ctl-2plus2": { + "ggml@bf16:llama.cpp": "closed", + "hf@bf16:hf": "closed", + "vllm@bf16:vllm": "closed", + "ggml@quant:llama.cpp": "closed" + }, + "Qwen3.5-4B/Q4_K_M|ctl-recall-7": { + "ggml@bf16:llama.cpp": "closed", + "hf@bf16:hf": "closed", + "vllm@bf16:vllm": "closed", + "ggml@quant:llama.cpp": "unclosed" + }, + "Qwen3.5-4B/Q4_K_M|ctl-code-add": { + "ggml@bf16:llama.cpp": "closed", + "hf@bf16:hf": "closed", + "vllm@bf16:vllm": "closed", + "ggml@quant:llama.cpp": "closed" + }, + "Qwen3.5-4B/Q4_K_M|fact-capital-japan": { + "ggml@bf16:llama.cpp": "closed", + "hf@bf16:hf": "closed", + "vllm@bf16:vllm": "closed", + "ggml@quant:llama.cpp": "closed" + }, + "Qwen3.5-4B/Q4_K_M|fact-capital-france": { + "ggml@bf16:llama.cpp": "closed", + "hf@bf16:hf": "closed", + "vllm@bf16:vllm": "closed", + "ggml@quant:llama.cpp": "closed" + }, + "Qwen3.5-4B/Q4_K_M|golden-paris": { + "ggml@bf16:llama.cpp": "closed", + "hf@bf16:hf": "closed", + "vllm@bf16:vllm": "closed", + "ggml@quant:llama.cpp": "closed" + }, + "Qwen3.5-4B/Q4_K_M|fact-water-formula": { + "ggml@bf16:llama.cpp": "unclosed", + "hf@bf16:hf": "unclosed", + "vllm@bf16:vllm": "unclosed", + "ggml@quant:llama.cpp": "closed" + }, + "Qwen3.5-4B/Q4_K_M|nonascii-water-kanji": { + "ggml@bf16:llama.cpp": "closed", + "hf@bf16:hf": "closed", + "vllm@bf16:vllm": "closed", + "ggml@quant:llama.cpp": "closed" + }, + "Qwen3.5-4B/Q4_K_M|arith-17x23": { + "ggml@bf16:llama.cpp": "closed", + "hf@bf16:hf": "closed", + "vllm@bf16:vllm": "closed", + "ggml@quant:llama.cpp": "closed" + }, + "Qwen3.5-4B/Q4_K_M|arith-precedence": { + "ggml@bf16:llama.cpp": "closed", + "hf@bf16:hf": "closed", + "vllm@bf16:vllm": "closed", + "ggml@quant:llama.cpp": "closed" + }, + "Qwen3.5-4B/Q4_K_M|arith-word-apples": { + "ggml@bf16:llama.cpp": "closed", + "hf@bf16:hf": "closed", + "vllm@bf16:vllm": "closed", + "ggml@quant:llama.cpp": "closed" + }, + "Qwen3.5-4B/Q4_K_M|hard-add-4digit": { + "ggml@bf16:llama.cpp": "closed", + "hf@bf16:hf": "closed", + "vllm@bf16:vllm": "closed", + "ggml@quant:llama.cpp": "unclosed" + }, + "Qwen3.5-4B/Q4_K_M|hard-not-prime": { + "ggml@bf16:llama.cpp": "closed", + "hf@bf16:hf": "closed", + "vllm@bf16:vllm": "closed", + "ggml@quant:llama.cpp": "closed" + }, + "Qwen3.5-4B/Q4_K_M|hard-reverse-word": { + "ggml@bf16:llama.cpp": "closed", + "hf@bf16:hf": "closed", + "vllm@bf16:vllm": "closed", + "ggml@quant:llama.cpp": "closed" + }, + "Qwen3.5-4B/Q4_K_M|hard-count-letters": { + "ggml@bf16:llama.cpp": "closed", + "hf@bf16:hf": "closed", + "vllm@bf16:vllm": "closed", + "ggml@quant:llama.cpp": "closed" + }, + "Qwen3.5-4B/Q4_K_M|recall-cats": { + "ggml@bf16:llama.cpp": "unclosed", + "hf@bf16:hf": "closed", + "vllm@bf16:vllm": "closed", + "ggml@quant:llama.cpp": "unclosed" + }, + "Qwen3.5-4B/Q4_K_M|recall-name": { + "ggml@bf16:llama.cpp": "closed", + "hf@bf16:hf": "closed", + "vllm@bf16:vllm": "closed", + "ggml@quant:llama.cpp": "unclosed" + }, + "Qwen3.5-4B/Q4_K_M|recall-arith-chain": { + "ggml@bf16:llama.cpp": "closed", + "hf@bf16:hf": "closed", + "vllm@bf16:vllm": "closed", + "ggml@quant:llama.cpp": "closed" + }, + "Qwen3.5-4B/Q4_K_M|code-is-palindrome": { + "ggml@bf16:llama.cpp": "closed", + "hf@bf16:hf": "closed", + "vllm@bf16:vllm": "closed", + "ggml@quant:llama.cpp": "unclosed" + }, + "Qwen3.5-4B/Q4_K_M|code-fib": { + "ggml@bf16:llama.cpp": "unclosed", + "hf@bf16:hf": "unclosed", + "vllm@bf16:vllm": "unclosed", + "ggml@quant:llama.cpp": "unclosed" + }, + "Qwen3.5-4B/Q4_K_M|code-count-vowels": { + "ggml@bf16:llama.cpp": "closed", + "hf@bf16:hf": "closed", + "vllm@bf16:vllm": "closed", + "ggml@quant:llama.cpp": "closed" + }, + "Qwen3.5-4B/Q4_K_M|long-primes-10": { + "ggml@bf16:llama.cpp": "closed", + "hf@bf16:hf": "closed", + "vllm@bf16:vllm": "closed", + "ggml@quant:llama.cpp": "closed" + }, + "Qwen3.5-4B/Q4_K_M|long-countdown-15": { + "ggml@bf16:llama.cpp": "closed", + "hf@bf16:hf": "closed", + "vllm@bf16:vllm": "closed", + "ggml@quant:llama.cpp": "closed" + }, + "Qwen3.5-4B/UD-Q4_K_XL|ctl-2plus2": { + "ggml@bf16:llama.cpp": "closed", + "hf@bf16:hf": "closed", + "vllm@bf16:vllm": "closed", + "ggml@quant:llama.cpp": "closed" + }, + "Qwen3.5-4B/UD-Q4_K_XL|ctl-recall-7": { + "ggml@bf16:llama.cpp": "closed", + "hf@bf16:hf": "closed", + "vllm@bf16:vllm": "closed", + "ggml@quant:llama.cpp": "unclosed" + }, + "Qwen3.5-4B/UD-Q4_K_XL|ctl-code-add": { + "ggml@bf16:llama.cpp": "closed", + "hf@bf16:hf": "closed", + "vllm@bf16:vllm": "closed", + "ggml@quant:llama.cpp": "closed" + }, + "Qwen3.5-4B/UD-Q4_K_XL|fact-capital-japan": { + "ggml@bf16:llama.cpp": "closed", + "hf@bf16:hf": "closed", + "vllm@bf16:vllm": "closed", + "ggml@quant:llama.cpp": "closed" + }, + "Qwen3.5-4B/UD-Q4_K_XL|fact-capital-france": { + "ggml@bf16:llama.cpp": "closed", + "hf@bf16:hf": "closed", + "vllm@bf16:vllm": "closed", + "ggml@quant:llama.cpp": "closed" + }, + "Qwen3.5-4B/UD-Q4_K_XL|golden-paris": { + "ggml@bf16:llama.cpp": "closed", + "hf@bf16:hf": "closed", + "vllm@bf16:vllm": "closed", + "ggml@quant:llama.cpp": "closed" + }, + "Qwen3.5-4B/UD-Q4_K_XL|fact-water-formula": { + "ggml@bf16:llama.cpp": "unclosed", + "hf@bf16:hf": "unclosed", + "vllm@bf16:vllm": "unclosed", + "ggml@quant:llama.cpp": "unclosed" + }, + "Qwen3.5-4B/UD-Q4_K_XL|nonascii-water-kanji": { + "ggml@bf16:llama.cpp": "closed", + "hf@bf16:hf": "closed", + "vllm@bf16:vllm": "closed", + "ggml@quant:llama.cpp": "closed" + }, + "Qwen3.5-4B/UD-Q4_K_XL|arith-17x23": { + "ggml@bf16:llama.cpp": "closed", + "hf@bf16:hf": "closed", + "vllm@bf16:vllm": "closed", + "ggml@quant:llama.cpp": "closed" + }, + "Qwen3.5-4B/UD-Q4_K_XL|arith-precedence": { + "ggml@bf16:llama.cpp": "closed", + "hf@bf16:hf": "closed", + "vllm@bf16:vllm": "closed", + "ggml@quant:llama.cpp": "closed" + }, + "Qwen3.5-4B/UD-Q4_K_XL|arith-word-apples": { + "ggml@bf16:llama.cpp": "closed", + "hf@bf16:hf": "closed", + "vllm@bf16:vllm": "closed", + "ggml@quant:llama.cpp": "closed" + }, + "Qwen3.5-4B/UD-Q4_K_XL|hard-add-4digit": { + "ggml@bf16:llama.cpp": "closed", + "hf@bf16:hf": "closed", + "vllm@bf16:vllm": "closed", + "ggml@quant:llama.cpp": "unclosed" + }, + "Qwen3.5-4B/UD-Q4_K_XL|hard-not-prime": { + "ggml@bf16:llama.cpp": "closed", + "hf@bf16:hf": "closed", + "vllm@bf16:vllm": "closed", + "ggml@quant:llama.cpp": "closed" + }, + "Qwen3.5-4B/UD-Q4_K_XL|hard-reverse-word": { + "ggml@bf16:llama.cpp": "closed", + "hf@bf16:hf": "closed", + "vllm@bf16:vllm": "closed", + "ggml@quant:llama.cpp": "closed" + }, + "Qwen3.5-4B/UD-Q4_K_XL|hard-count-letters": { + "ggml@bf16:llama.cpp": "closed", + "hf@bf16:hf": "closed", + "vllm@bf16:vllm": "closed", + "ggml@quant:llama.cpp": "closed" + }, + "Qwen3.5-4B/UD-Q4_K_XL|recall-cats": { + "ggml@bf16:llama.cpp": "unclosed", + "hf@bf16:hf": "closed", + "vllm@bf16:vllm": "closed", + "ggml@quant:llama.cpp": "unclosed" + }, + "Qwen3.5-4B/UD-Q4_K_XL|recall-name": { + "ggml@bf16:llama.cpp": "closed", + "hf@bf16:hf": "closed", + "vllm@bf16:vllm": "closed", + "ggml@quant:llama.cpp": "unclosed" + }, + "Qwen3.5-4B/UD-Q4_K_XL|recall-arith-chain": { + "ggml@bf16:llama.cpp": "closed", + "hf@bf16:hf": "closed", + "vllm@bf16:vllm": "closed", + "ggml@quant:llama.cpp": "closed" + }, + "Qwen3.5-4B/UD-Q4_K_XL|code-is-palindrome": { + "ggml@bf16:llama.cpp": "closed", + "hf@bf16:hf": "closed", + "vllm@bf16:vllm": "closed", + "ggml@quant:llama.cpp": "unclosed" + }, + "Qwen3.5-4B/UD-Q4_K_XL|code-fib": { + "ggml@bf16:llama.cpp": "unclosed", + "hf@bf16:hf": "unclosed", + "vllm@bf16:vllm": "unclosed", + "ggml@quant:llama.cpp": "unclosed" + }, + "Qwen3.5-4B/UD-Q4_K_XL|code-count-vowels": { + "ggml@bf16:llama.cpp": "closed", + "hf@bf16:hf": "closed", + "vllm@bf16:vllm": "closed", + "ggml@quant:llama.cpp": "unclosed" + }, + "Qwen3.5-4B/UD-Q4_K_XL|long-primes-10": { + "ggml@bf16:llama.cpp": "closed", + "hf@bf16:hf": "closed", + "vllm@bf16:vllm": "closed", + "ggml@quant:llama.cpp": "closed" + }, + "Qwen3.5-4B/UD-Q4_K_XL|long-countdown-15": { + "ggml@bf16:llama.cpp": "closed", + "hf@bf16:hf": "closed", + "vllm@bf16:vllm": "closed", + "ggml@quant:llama.cpp": "closed" + } + }, + "cells": [ + { + "leg": "ggml@bf16", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": false, + "why": "unclosed think (budget exhausted)", + "extracted": null, + "think": "unclosed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L1", + "prompt_id": "ctl-2plus2", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": false, + "why": "unclosed think (budget exhausted)", + "extracted": null, + "think": "unclosed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L263", + "prompt_id": "ctl-2plus2", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "ctl-2plus2", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "ctl-2plus2", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "ctl-2plus2", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "4", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L11", + "prompt_id": "ctl-2plus2", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "4", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L1", + "prompt_id": "ctl-2plus2", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "4", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L23", + "prompt_id": "ctl-2plus2", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "4", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L33", + "prompt_id": "ctl-2plus2", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": false, + "why": "unclosed think (budget exhausted)", + "extracted": null, + "think": "unclosed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L2", + "prompt_id": "ctl-recall-7", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": false, + "why": "unclosed think (budget exhausted)", + "extracted": null, + "think": "unclosed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L264", + "prompt_id": "ctl-recall-7", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "ctl-recall-7", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "ctl-recall-7", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "ctl-recall-7", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "7", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L12", + "prompt_id": "ctl-recall-7", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "7", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L2", + "prompt_id": "ctl-recall-7", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "7", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L24", + "prompt_id": "ctl-recall-7", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "7", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L34", + "prompt_id": "ctl-recall-7", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "def add(a, b):\n return a + b\n", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L3", + "prompt_id": "ctl-code-add", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "ctl-code-add", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "ctl-code-add", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "ctl-code-add", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "def add(a, b):\n return a + b\n", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L13", + "prompt_id": "ctl-code-add", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "def add(a, b):\n return a + b\n", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L3", + "prompt_id": "ctl-code-add", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "def add(a, b):\n return a + b\n", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L25", + "prompt_id": "ctl-code-add", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "def add(a, b):\n return a + b\n", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L35", + "prompt_id": "ctl-code-add", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": false, + "why": "unclosed think (budget exhausted)", + "extracted": null, + "think": "unclosed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L4", + "prompt_id": "fact-capital-japan", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "fact-capital-japan", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "fact-capital-japan", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "fact-capital-japan", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "Tokyo", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L14", + "prompt_id": "fact-capital-japan", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "Tokyo", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L4", + "prompt_id": "fact-capital-japan", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "Tokyo", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L26", + "prompt_id": "fact-capital-japan", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "Tokyo", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L36", + "prompt_id": "fact-capital-japan", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "Paris", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L5", + "prompt_id": "fact-capital-france", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "fact-capital-france", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "fact-capital-france", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "fact-capital-france", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "Paris", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L15", + "prompt_id": "fact-capital-france", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "Paris", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L5", + "prompt_id": "fact-capital-france", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "Paris", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L27", + "prompt_id": "fact-capital-france", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "Paris", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L37", + "prompt_id": "fact-capital-france", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "golden-paris", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "golden-paris", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "golden-paris", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "golden-paris", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "Paris", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L16", + "prompt_id": "golden-paris", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "Paris", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L6", + "prompt_id": "golden-paris", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "Paris", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L28", + "prompt_id": "golden-paris", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "Paris", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L38", + "prompt_id": "golden-paris", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": false, + "why": "unclosed think (budget exhausted)", + "extracted": null, + "think": "unclosed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L6", + "prompt_id": "fact-water-formula", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "fact-water-formula", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "fact-water-formula", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "fact-water-formula", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": false, + "why": "mismatch", + "extracted": "H_2O", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L17", + "prompt_id": "fact-water-formula", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": false, + "why": "mismatch", + "extracted": "H_2O", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L7", + "prompt_id": "fact-water-formula", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "lambda", + "correct": false, + "why": "mismatch", + "extracted": "H_2O", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L29", + "prompt_id": "fact-water-formula", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": false, + "why": "mismatch", + "extracted": "H₂O", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L39", + "prompt_id": "fact-water-formula", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "nonascii-water-kanji", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "nonascii-water-kanji", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "nonascii-water-kanji", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "nonascii-water-kanji", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "nonascii-water-kanji", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "nonascii-water-kanji", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "nonascii-water-kanji", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "nonascii-water-kanji", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": false, + "why": "unclosed think (budget exhausted)", + "extracted": null, + "think": "unclosed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L7", + "prompt_id": "arith-17x23", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "arith-17x23", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "arith-17x23", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "arith-17x23", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "391", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L18", + "prompt_id": "arith-17x23", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "391", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L8", + "prompt_id": "arith-17x23", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "391", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L30", + "prompt_id": "arith-17x23", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "391", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L40", + "prompt_id": "arith-17x23", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "23", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L8", + "prompt_id": "arith-precedence", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "arith-precedence", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "arith-precedence", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "arith-precedence", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "23", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L19", + "prompt_id": "arith-precedence", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "23", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L9", + "prompt_id": "arith-precedence", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "23", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L31", + "prompt_id": "arith-precedence", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "23", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L41", + "prompt_id": "arith-precedence", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "56", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L9", + "prompt_id": "arith-word-apples", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "arith-word-apples", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "arith-word-apples", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "arith-word-apples", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "56", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L20", + "prompt_id": "arith-word-apples", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "56", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L10", + "prompt_id": "arith-word-apples", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "56", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L32", + "prompt_id": "arith-word-apples", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "56", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L42", + "prompt_id": "arith-word-apples", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "8613", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L10", + "prompt_id": "hard-add-4digit", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "hard-add-4digit", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "hard-add-4digit", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "hard-add-4digit", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "8613", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L21", + "prompt_id": "hard-add-4digit", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "8613", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L11", + "prompt_id": "hard-add-4digit", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "8613", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L33", + "prompt_id": "hard-add-4digit", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "8613", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L43", + "prompt_id": "hard-add-4digit", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "hard-not-prime", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "hard-not-prime", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "hard-not-prime", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "hard-not-prime", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "15", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L22", + "prompt_id": "hard-not-prime", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "15", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L12", + "prompt_id": "hard-not-prime", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "15", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L34", + "prompt_id": "hard-not-prime", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "15", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L44", + "prompt_id": "hard-not-prime", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "hard-reverse-word", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "hard-reverse-word", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "hard-reverse-word", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "hard-reverse-word", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": false, + "why": "mismatch", + "extracted": "tnipal", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L23", + "prompt_id": "hard-reverse-word", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": false, + "why": "mismatch", + "extracted": "tnipal", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L13", + "prompt_id": "hard-reverse-word", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "lambda", + "correct": false, + "why": "mismatch", + "extracted": "tnipal", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L35", + "prompt_id": "hard-reverse-word", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": false, + "why": "mismatch", + "extracted": "tenap", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L45", + "prompt_id": "hard-reverse-word", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "hard-count-letters", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "hard-count-letters", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "hard-count-letters", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "hard-count-letters", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "3", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L24", + "prompt_id": "hard-count-letters", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "3", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L14", + "prompt_id": "hard-count-letters", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "3", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L36", + "prompt_id": "hard-count-letters", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "3", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L46", + "prompt_id": "hard-count-letters", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "recall-cats", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "recall-cats", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "recall-cats", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "recall-cats", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "5", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L25", + "prompt_id": "recall-cats", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "5", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L15", + "prompt_id": "recall-cats", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "5", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L37", + "prompt_id": "recall-cats", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "5", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L47", + "prompt_id": "recall-cats", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "recall-name", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "recall-name", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "recall-name", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "recall-name", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "Ada", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L26", + "prompt_id": "recall-name", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "Ada", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L16", + "prompt_id": "recall-name", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "Ada", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L38", + "prompt_id": "recall-name", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "Ada", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L48", + "prompt_id": "recall-name", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "recall-arith-chain", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "recall-arith-chain", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "recall-arith-chain", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "recall-arith-chain", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "7", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L27", + "prompt_id": "recall-arith-chain", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "7", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L17", + "prompt_id": "recall-arith-chain", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "7", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L39", + "prompt_id": "recall-arith-chain", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "7", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L49", + "prompt_id": "recall-arith-chain", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "code-is-palindrome", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "code-is-palindrome", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "code-is-palindrome", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "code-is-palindrome", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "def is_palindrome(s):\n s = s.lower()\n return s == s[::-1]\n", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L28", + "prompt_id": "code-is-palindrome", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "def is_palindrome(s):\n s = s.lower()\n return s == s[::-1]\n", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L18", + "prompt_id": "code-is-palindrome", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "def is_palindrome(s):\n s = s.lower()\n return s == s[::-1]\n", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L40", + "prompt_id": "code-is-palindrome", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "def is_palindrome(s):\n s = s.lower()\n return s == s[::-1]\n", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L50", + "prompt_id": "code-is-palindrome", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "code-fib", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "code-fib", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "code-fib", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "code-fib", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "def fib(n):\n if n < 0:\n raise ValueError(\"n must be a non-negative integer\")\n if n == 0:\n return 0\n if n == 1:\n return 1\n a, b = 0, 1\n for _ in range(2, n + 1):\n a, b = b, a + b\n return b\n", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L29", + "prompt_id": "code-fib", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "def fib(n):\n if n < 0:\n raise ValueError(\"n must be a non-negative integer\")\n if n == 0:\n return 0\n if n == 1:\n return 1\n a, b = 0, 1\n for _ in range(2, n + 1):\n a, b = b, a + b\n return b\n", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L19", + "prompt_id": "code-fib", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "def fib(n):\n if n < 0:\n raise ValueError(\"n must be a non-negative integer\")\n if n == 0:\n return 0\n if n == 1:\n return 1\n a, b = 0, 1\n for _ in range(2, n + 1):\n a, b = b, a + b\n return b\n", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L41", + "prompt_id": "code-fib", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "def fib(n):\n if n < 0:\n raise ValueError(\"n must be a non-negative integer\")\n if n == 0:\n return 0\n if n == 1:\n return 1\n a, b = 0, 1\n for _ in range(2, n + 1):\n a, b = b, a + b\n return b\n", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L51", + "prompt_id": "code-fib", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "code-count-vowels", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "code-count-vowels", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "code-count-vowels", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "code-count-vowels", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "def count_vowels(s):\n vowels = set('aeiouAEIOU')\n return sum(1 for char in s if char.lower() in vowels)\n", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L30", + "prompt_id": "code-count-vowels", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "def count_vowels(s):\n vowels = set('aeiouAEIOU')\n return sum(1 for char in s if char.lower() in vowels)\n", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L20", + "prompt_id": "code-count-vowels", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "def count_vowels(s):\n vowels = set('aeiouAEIOU')\n return sum(1 for char in s if char.lower() in vowels)\n", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L42", + "prompt_id": "code-count-vowels", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "def count_vowels(s):\n vowels = set('aeiouAEIOU')\n return sum(1 for char in s if char in vowels)\n", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L52", + "prompt_id": "code-count-vowels", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "long-primes-10", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "long-primes-10", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "long-primes-10", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "long-primes-10", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": [ + "2", + "3", + "5", + "7", + "11", + "13", + "17", + "19", + "23", + "29" + ], + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L31", + "prompt_id": "long-primes-10", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": [ + "2", + "3", + "5", + "7", + "11", + "13", + "17", + "19", + "23", + "29" + ], + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L21", + "prompt_id": "long-primes-10", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": [ + "2", + "3", + "5", + "7", + "11", + "13", + "17", + "19", + "23", + "29" + ], + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L43", + "prompt_id": "long-primes-10", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": [ + "2", + "3", + "5", + "7", + "11", + "13", + "17", + "19", + "23", + "29" + ], + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L53", + "prompt_id": "long-primes-10", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "long-countdown-15", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "long-countdown-15", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "long-countdown-15", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "long-countdown-15", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": [ + "15", + "14", + "13", + "12", + "11", + "10", + "9", + "8", + "7", + "6", + "5", + "4", + "3", + "2", + "1" + ], + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L32", + "prompt_id": "long-countdown-15", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": [ + "15", + "14", + "13", + "12", + "11", + "10", + "9", + "8", + "7", + "6", + "5", + "4", + "3", + "2", + "1" + ], + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L22", + "prompt_id": "long-countdown-15", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": [ + "15", + "14", + "13", + "12", + "11", + "10", + "9", + "8", + "7", + "6", + "5", + "4", + "3", + "2", + "1" + ], + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L44", + "prompt_id": "long-countdown-15", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": [ + "15", + "14", + "13", + "12", + "11", + "10", + "9", + "8", + "7", + "6", + "5", + "4", + "3", + "2", + "1" + ], + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L54", + "prompt_id": "long-countdown-15", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "4", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L121", + "prompt_id": "ctl-2plus2", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "engine": "hf", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "4", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L187", + "prompt_id": "ctl-2plus2", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "4", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L235", + "prompt_id": "ctl-2plus2", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "4", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L143", + "prompt_id": "ctl-2plus2", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "4", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L55", + "prompt_id": "ctl-2plus2", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "4", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L45", + "prompt_id": "ctl-2plus2", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "4", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L210", + "prompt_id": "ctl-2plus2", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "4", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L77", + "prompt_id": "ctl-2plus2", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "7", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L122", + "prompt_id": "ctl-recall-7", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "engine": "hf", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "7", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L188", + "prompt_id": "ctl-recall-7", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "7", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L236", + "prompt_id": "ctl-recall-7", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": false, + "why": "unclosed think (budget exhausted)", + "extracted": null, + "think": "unclosed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L144", + "prompt_id": "ctl-recall-7", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "7", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L56", + "prompt_id": "ctl-recall-7", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "7", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L46", + "prompt_id": "ctl-recall-7", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "7", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L211", + "prompt_id": "ctl-recall-7", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "7", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L78", + "prompt_id": "ctl-recall-7", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "def add(a, b):\n return a + b\n", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L127", + "prompt_id": "ctl-code-add", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "engine": "hf", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "def add(a, b):\n return a + b\n", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L189", + "prompt_id": "ctl-code-add", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "def add(a, b):\n return a + b\n", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L237", + "prompt_id": "ctl-code-add", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "def add(a, b):\n return a + b\n", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L145", + "prompt_id": "ctl-code-add", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "def add(a, b):\n return a + b\n", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L57", + "prompt_id": "ctl-code-add", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "def add(a, b):\n return a + b\n", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L47", + "prompt_id": "ctl-code-add", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "def add(a, b):\n return a + b\n", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L212", + "prompt_id": "ctl-code-add", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "def add(a, b):\n return a + b\n", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L79", + "prompt_id": "ctl-code-add", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "Tokyo", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L123", + "prompt_id": "fact-capital-japan", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "engine": "hf", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "Tokyo", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L190", + "prompt_id": "fact-capital-japan", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "Tokyo", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L238", + "prompt_id": "fact-capital-japan", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "Tokyo", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L146", + "prompt_id": "fact-capital-japan", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "Tokyo", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L58", + "prompt_id": "fact-capital-japan", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "Tokyo", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L48", + "prompt_id": "fact-capital-japan", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "Tokyo", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L213", + "prompt_id": "fact-capital-japan", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "Tokyo", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L80", + "prompt_id": "fact-capital-japan", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "Paris", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L124", + "prompt_id": "fact-capital-france", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "engine": "hf", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "Paris", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L191", + "prompt_id": "fact-capital-france", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "Paris", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L239", + "prompt_id": "fact-capital-france", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "Paris", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L147", + "prompt_id": "fact-capital-france", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "Paris", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L59", + "prompt_id": "fact-capital-france", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "Paris", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L49", + "prompt_id": "fact-capital-france", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "Paris", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L214", + "prompt_id": "fact-capital-france", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "Paris", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L81", + "prompt_id": "fact-capital-france", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "Paris", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L128", + "prompt_id": "golden-paris", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "engine": "hf", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "Paris", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L192", + "prompt_id": "golden-paris", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "Paris", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L240", + "prompt_id": "golden-paris", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "Paris", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L148", + "prompt_id": "golden-paris", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "Paris", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L60", + "prompt_id": "golden-paris", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "Paris", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L50", + "prompt_id": "golden-paris", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "Paris", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L215", + "prompt_id": "golden-paris", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "Paris", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L82", + "prompt_id": "golden-paris", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": false, + "why": "unclosed think (budget exhausted)", + "extracted": null, + "think": "unclosed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L125", + "prompt_id": "fact-water-formula", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "engine": "hf", + "verb": "chat", + "host": "gx10", + "correct": false, + "why": "unclosed think (budget exhausted)", + "extracted": null, + "think": "unclosed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L193", + "prompt_id": "fact-water-formula", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": false, + "why": "unclosed think (budget exhausted)", + "extracted": null, + "think": "unclosed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L241", + "prompt_id": "fact-water-formula", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": false, + "why": "mismatch", + "extracted": "H_2O", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L149", + "prompt_id": "fact-water-formula", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": false, + "why": "mismatch", + "extracted": "H₂O", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L61", + "prompt_id": "fact-water-formula", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": false, + "why": "mismatch", + "extracted": "H₂O", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L51", + "prompt_id": "fact-water-formula", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": false, + "why": "mismatch", + "extracted": "H₂O", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L216", + "prompt_id": "fact-water-formula", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": false, + "why": "mismatch", + "extracted": "H2O", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L83", + "prompt_id": "fact-water-formula", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "水", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L233", + "prompt_id": "nonascii-water-kanji", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "engine": "hf", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "水", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L194", + "prompt_id": "nonascii-water-kanji", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "水", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L242", + "prompt_id": "nonascii-water-kanji", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "水", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L258", + "prompt_id": "nonascii-water-kanji", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "水", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L234", + "prompt_id": "nonascii-water-kanji", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "水", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L262", + "prompt_id": "nonascii-water-kanji", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "水", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L217", + "prompt_id": "nonascii-water-kanji", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "水", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L259", + "prompt_id": "nonascii-water-kanji", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "391", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L126", + "prompt_id": "arith-17x23", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "engine": "hf", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "391", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L195", + "prompt_id": "arith-17x23", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "391", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L243", + "prompt_id": "arith-17x23", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "391", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L150", + "prompt_id": "arith-17x23", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "391", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L62", + "prompt_id": "arith-17x23", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "391", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L52", + "prompt_id": "arith-17x23", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "391", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L218", + "prompt_id": "arith-17x23", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "391", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L84", + "prompt_id": "arith-17x23", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "23", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L129", + "prompt_id": "arith-precedence", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "engine": "hf", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "23", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L196", + "prompt_id": "arith-precedence", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "23", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L244", + "prompt_id": "arith-precedence", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "23", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L151", + "prompt_id": "arith-precedence", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "23", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L63", + "prompt_id": "arith-precedence", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "23", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L53", + "prompt_id": "arith-precedence", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "23", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L219", + "prompt_id": "arith-precedence", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "23", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L85", + "prompt_id": "arith-precedence", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "56", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L130", + "prompt_id": "arith-word-apples", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "engine": "hf", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "56", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L197", + "prompt_id": "arith-word-apples", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "56", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L245", + "prompt_id": "arith-word-apples", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "56", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L152", + "prompt_id": "arith-word-apples", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "56", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L64", + "prompt_id": "arith-word-apples", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "56", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L54", + "prompt_id": "arith-word-apples", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "56", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L220", + "prompt_id": "arith-word-apples", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "56", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L86", + "prompt_id": "arith-word-apples", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "8613", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L131", + "prompt_id": "hard-add-4digit", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "engine": "hf", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "8613", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L198", + "prompt_id": "hard-add-4digit", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "8613", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L246", + "prompt_id": "hard-add-4digit", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": false, + "why": "unclosed think (budget exhausted)", + "extracted": null, + "think": "unclosed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L153", + "prompt_id": "hard-add-4digit", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "8613", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L65", + "prompt_id": "hard-add-4digit", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "8613", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L55", + "prompt_id": "hard-add-4digit", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "8613", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L221", + "prompt_id": "hard-add-4digit", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "8613", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L87", + "prompt_id": "hard-add-4digit", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "15", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L132", + "prompt_id": "hard-not-prime", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "engine": "hf", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "15", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L199", + "prompt_id": "hard-not-prime", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "15", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L247", + "prompt_id": "hard-not-prime", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "15", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L154", + "prompt_id": "hard-not-prime", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "15", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L66", + "prompt_id": "hard-not-prime", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "15", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L56", + "prompt_id": "hard-not-prime", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "15", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L222", + "prompt_id": "hard-not-prime", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "15", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L88", + "prompt_id": "hard-not-prime", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "tenalp", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L133", + "prompt_id": "hard-reverse-word", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "engine": "hf", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "tenalp", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L200", + "prompt_id": "hard-reverse-word", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "tenalp", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L248", + "prompt_id": "hard-reverse-word", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "tenalp", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L155", + "prompt_id": "hard-reverse-word", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "tenalp", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L67", + "prompt_id": "hard-reverse-word", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "tenalp", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L57", + "prompt_id": "hard-reverse-word", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "tenalp", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L223", + "prompt_id": "hard-reverse-word", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "tenalp", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L89", + "prompt_id": "hard-reverse-word", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "3", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L134", + "prompt_id": "hard-count-letters", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "engine": "hf", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "3", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L201", + "prompt_id": "hard-count-letters", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "3", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L249", + "prompt_id": "hard-count-letters", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "3", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L156", + "prompt_id": "hard-count-letters", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "3", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L68", + "prompt_id": "hard-count-letters", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "3", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L58", + "prompt_id": "hard-count-letters", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "3", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L224", + "prompt_id": "hard-count-letters", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "3", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L90", + "prompt_id": "hard-count-letters", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": false, + "why": "unclosed think (budget exhausted)", + "extracted": null, + "think": "unclosed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L135", + "prompt_id": "recall-cats", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "engine": "hf", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "5", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L202", + "prompt_id": "recall-cats", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "5", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L250", + "prompt_id": "recall-cats", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": false, + "why": "unclosed think (budget exhausted)", + "extracted": null, + "think": "unclosed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L157", + "prompt_id": "recall-cats", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "5", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L69", + "prompt_id": "recall-cats", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "5", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L59", + "prompt_id": "recall-cats", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "5", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L225", + "prompt_id": "recall-cats", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "5", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L91", + "prompt_id": "recall-cats", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "Ada", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L136", + "prompt_id": "recall-name", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "engine": "hf", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "Ada", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L203", + "prompt_id": "recall-name", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "Ada", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L251", + "prompt_id": "recall-name", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": false, + "why": "unclosed think (budget exhausted)", + "extracted": null, + "think": "unclosed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L158", + "prompt_id": "recall-name", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "Ada", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L70", + "prompt_id": "recall-name", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "Ada", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L60", + "prompt_id": "recall-name", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "Ada", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L226", + "prompt_id": "recall-name", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "Ada", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L92", + "prompt_id": "recall-name", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "7", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L137", + "prompt_id": "recall-arith-chain", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "engine": "hf", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "7", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L204", + "prompt_id": "recall-arith-chain", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "7", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L252", + "prompt_id": "recall-arith-chain", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "7", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L159", + "prompt_id": "recall-arith-chain", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "7", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L71", + "prompt_id": "recall-arith-chain", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "7", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L61", + "prompt_id": "recall-arith-chain", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "7", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L227", + "prompt_id": "recall-arith-chain", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "7", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L93", + "prompt_id": "recall-arith-chain", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "def is_palindrome(s):\n s = s.lower()\n return s == s[::-1]\n", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L138", + "prompt_id": "code-is-palindrome", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "engine": "hf", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "def is_palindrome(s):\n return s.lower() == s[::-1].lower()\n", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L205", + "prompt_id": "code-is-palindrome", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": false, + "why": "tests_failed: AssertionError", + "extracted": "def is_palindrome(s):\n return s.lower() == s[::-1]\n", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L253", + "prompt_id": "code-is-palindrome", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": false, + "why": "unclosed think (budget exhausted)", + "extracted": null, + "think": "unclosed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L160", + "prompt_id": "code-is-palindrome", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "def is_palindrome(s):\n cleaned = ''.join(char.lower() for char in s if char.isalnum())\n return cleaned == cleaned[::-1]\n", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L72", + "prompt_id": "code-is-palindrome", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "def is_palindrome(s):\n cleaned = ''.join(char.lower() for char in s if char.isalnum())\n return cleaned == cleaned[::-1]\n", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L62", + "prompt_id": "code-is-palindrome", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "def is_palindrome(s):\n cleaned = ''.join(char.lower() for char in s if char.isalnum())\n return cleaned == cleaned[::-1]\n", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L228", + "prompt_id": "code-is-palindrome", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "def is_palindrome(s):\n cleaned = ''.join(char.lower() for char in s if char.isalnum())\n return cleaned == cleaned[::-1]\n", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L94", + "prompt_id": "code-is-palindrome", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": false, + "why": "unclosed think (budget exhausted)", + "extracted": null, + "think": "unclosed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L139", + "prompt_id": "code-fib", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "engine": "hf", + "verb": "chat", + "host": "gx10", + "correct": false, + "why": "unclosed think (budget exhausted)", + "extracted": null, + "think": "unclosed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L206", + "prompt_id": "code-fib", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": false, + "why": "unclosed think (budget exhausted)", + "extracted": null, + "think": "unclosed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L254", + "prompt_id": "code-fib", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": false, + "why": "unclosed think (budget exhausted)", + "extracted": null, + "think": "unclosed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L161", + "prompt_id": "code-fib", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "def fib(n):\n if n == 0:\n return 0\n elif n == 1:\n return 1\n else:\n a, b = 0, 1\n for _ in range(2, n + 1):\n a, b = b, a + b\n return b\n", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L73", + "prompt_id": "code-fib", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "def fib(n):\n if n == 0:\n return 0\n elif n == 1:\n return 1\n a, b = 0, 1\n for _ in range(2, n + 1):\n a, b = b, a + b\n return b\n", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L63", + "prompt_id": "code-fib", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "def fib(n):\n if n == 0:\n return 0\n elif n == 1:\n return 1\n a, b = 0, 1\n for _ in range(2, n + 1):\n a, b = b, a + b\n return b\n", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L229", + "prompt_id": "code-fib", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "def fib(n):\n if n == 0:\n return 0\n elif n == 1:\n return 1\n else:\n a, b = 0, 1\n for _ in range(2, n + 1):\n a, b = b, a + b\n return b\n", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L95", + "prompt_id": "code-fib", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "def count_vowels(s):\n count = 0\n for char in s:\n if char.lower() in 'aeiou':\n count += 1\n return count\n", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L140", + "prompt_id": "code-count-vowels", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "engine": "hf", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "def count_vowels(s):\n count = 0\n vowels = \"aeiou\"\n for char in s:\n if char.lower() in vowels:\n count += 1\n return count\n", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L207", + "prompt_id": "code-count-vowels", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "def count_vowels(s):\n return sum(1 for char in s if char.lower() in 'aeiou')\n", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L255", + "prompt_id": "code-count-vowels", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "def count_vowels(s):\n vowels = \"aeiou\"\n count = 0\n for char in s:\n if char.lower() in vowels:\n count += 1\n return count\n", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L162", + "prompt_id": "code-count-vowels", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "def count_vowels(s):\n vowels = set('aeiouAEIOU')\n return sum(1 for char in s if char in vowels)\n", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L74", + "prompt_id": "code-count-vowels", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "def count_vowels(s):\n vowels = set('aeiouAEIOU')\n return sum(1 for char in s if char in vowels)\n", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L64", + "prompt_id": "code-count-vowels", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "def count_vowels(s):\n vowels = set('aeiouAEIOU')\n return sum(1 for char in s if char in vowels)\n", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L230", + "prompt_id": "code-count-vowels", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "def count_vowels(s):\n vowels = \"aeiouAEIOU\"\n count = 0\n for char in s:\n if char in vowels:\n count += 1\n return count\n", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L96", + "prompt_id": "code-count-vowels", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": [ + "2", + "3", + "5", + "7", + "11", + "13", + "17", + "19", + "23", + "29" + ], + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L141", + "prompt_id": "long-primes-10", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "engine": "hf", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": [ + "2", + "3", + "5", + "7", + "11", + "13", + "17", + "19", + "23", + "29" + ], + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L208", + "prompt_id": "long-primes-10", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": [ + "2", + "3", + "5", + "7", + "11", + "13", + "17", + "19", + "23", + "29" + ], + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L256", + "prompt_id": "long-primes-10", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": [ + "2", + "3", + "5", + "7", + "11", + "13", + "17", + "19", + "23", + "29" + ], + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L163", + "prompt_id": "long-primes-10", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": [ + "2", + "3", + "5", + "7", + "11", + "13", + "17", + "19", + "23", + "29" + ], + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L75", + "prompt_id": "long-primes-10", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": [ + "2", + "3", + "5", + "7", + "11", + "13", + "17", + "19", + "23", + "29" + ], + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L65", + "prompt_id": "long-primes-10", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": [ + "2", + "3", + "5", + "7", + "11", + "13", + "17", + "19", + "23", + "29" + ], + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L231", + "prompt_id": "long-primes-10", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": [ + "2", + "3", + "5", + "7", + "11", + "13", + "17", + "19", + "23", + "29" + ], + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L97", + "prompt_id": "long-primes-10", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": [ + "15", + "14", + "13", + "12", + "11", + "10", + "9", + "8", + "7", + "6", + "5", + "4", + "3", + "2", + "1" + ], + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L142", + "prompt_id": "long-countdown-15", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "engine": "hf", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": [ + "15", + "14", + "13", + "12", + "11", + "10", + "9", + "8", + "7", + "6", + "5", + "4", + "3", + "2", + "1" + ], + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L209", + "prompt_id": "long-countdown-15", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": [ + "15", + "14", + "13", + "12", + "11", + "10", + "9", + "8", + "7", + "6", + "5", + "4", + "3", + "2", + "1" + ], + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L257", + "prompt_id": "long-countdown-15", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": [ + "15", + "14", + "13", + "12", + "11", + "10", + "9", + "8", + "7", + "6", + "5", + "4", + "3", + "2", + "1" + ], + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L164", + "prompt_id": "long-countdown-15", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": [ + "15", + "14", + "13", + "12", + "11", + "10", + "9", + "8", + "7", + "6", + "5", + "4", + "3", + "2", + "1" + ], + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L76", + "prompt_id": "long-countdown-15", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": [ + "15", + "14", + "13", + "12", + "11", + "10", + "9", + "8", + "7", + "6", + "5", + "4", + "3", + "2", + "1" + ], + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L66", + "prompt_id": "long-countdown-15", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": [ + "15", + "14", + "13", + "12", + "11", + "10", + "9", + "8", + "7", + "6", + "5", + "4", + "3", + "2", + "1" + ], + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L232", + "prompt_id": "long-countdown-15", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": [ + "15", + "14", + "13", + "12", + "11", + "10", + "9", + "8", + "7", + "6", + "5", + "4", + "3", + "2", + "1" + ], + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L98", + "prompt_id": "long-countdown-15", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "4", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L121", + "prompt_id": "ctl-2plus2", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "engine": "hf", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "4", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L187", + "prompt_id": "ctl-2plus2", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "4", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L235", + "prompt_id": "ctl-2plus2", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "4", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L165", + "prompt_id": "ctl-2plus2", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "4", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L55", + "prompt_id": "ctl-2plus2", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "4", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L45", + "prompt_id": "ctl-2plus2", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "4", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L210", + "prompt_id": "ctl-2plus2", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "4", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L99", + "prompt_id": "ctl-2plus2", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "7", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L122", + "prompt_id": "ctl-recall-7", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "engine": "hf", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "7", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L188", + "prompt_id": "ctl-recall-7", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "7", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L236", + "prompt_id": "ctl-recall-7", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": false, + "why": "unclosed think (budget exhausted)", + "extracted": null, + "think": "unclosed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L166", + "prompt_id": "ctl-recall-7", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "7", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L56", + "prompt_id": "ctl-recall-7", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "7", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L46", + "prompt_id": "ctl-recall-7", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "7", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L211", + "prompt_id": "ctl-recall-7", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "7", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L100", + "prompt_id": "ctl-recall-7", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "def add(a, b):\n return a + b\n", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L127", + "prompt_id": "ctl-code-add", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "engine": "hf", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "def add(a, b):\n return a + b\n", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L189", + "prompt_id": "ctl-code-add", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "def add(a, b):\n return a + b\n", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L237", + "prompt_id": "ctl-code-add", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "def add(a, b):\n return a + b\n", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L167", + "prompt_id": "ctl-code-add", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "def add(a, b):\n return a + b\n", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L57", + "prompt_id": "ctl-code-add", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "def add(a, b):\n return a + b\n", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L47", + "prompt_id": "ctl-code-add", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "def add(a, b):\n return a + b\n", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L212", + "prompt_id": "ctl-code-add", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "def add(a, b):\n return a + b\n", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L101", + "prompt_id": "ctl-code-add", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "Tokyo", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L123", + "prompt_id": "fact-capital-japan", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "engine": "hf", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "Tokyo", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L190", + "prompt_id": "fact-capital-japan", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "Tokyo", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L238", + "prompt_id": "fact-capital-japan", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "Tokyo", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L168", + "prompt_id": "fact-capital-japan", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "Tokyo", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L58", + "prompt_id": "fact-capital-japan", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "Tokyo", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L48", + "prompt_id": "fact-capital-japan", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "Tokyo", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L213", + "prompt_id": "fact-capital-japan", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "Tokyo", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L102", + "prompt_id": "fact-capital-japan", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "Paris", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L124", + "prompt_id": "fact-capital-france", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "engine": "hf", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "Paris", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L191", + "prompt_id": "fact-capital-france", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "Paris", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L239", + "prompt_id": "fact-capital-france", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "Paris", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L169", + "prompt_id": "fact-capital-france", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "Paris", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L59", + "prompt_id": "fact-capital-france", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "Paris", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L49", + "prompt_id": "fact-capital-france", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "Paris", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L214", + "prompt_id": "fact-capital-france", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "Paris", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L103", + "prompt_id": "fact-capital-france", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "Paris", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L128", + "prompt_id": "golden-paris", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "engine": "hf", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "Paris", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L192", + "prompt_id": "golden-paris", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "Paris", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L240", + "prompt_id": "golden-paris", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "Paris", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L170", + "prompt_id": "golden-paris", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "Paris", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L60", + "prompt_id": "golden-paris", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "Paris", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L50", + "prompt_id": "golden-paris", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "Paris", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L215", + "prompt_id": "golden-paris", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "Paris", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L104", + "prompt_id": "golden-paris", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": false, + "why": "unclosed think (budget exhausted)", + "extracted": null, + "think": "unclosed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L125", + "prompt_id": "fact-water-formula", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "engine": "hf", + "verb": "chat", + "host": "gx10", + "correct": false, + "why": "unclosed think (budget exhausted)", + "extracted": null, + "think": "unclosed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L193", + "prompt_id": "fact-water-formula", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": false, + "why": "unclosed think (budget exhausted)", + "extracted": null, + "think": "unclosed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L241", + "prompt_id": "fact-water-formula", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": false, + "why": "unclosed think (budget exhausted)", + "extracted": null, + "think": "unclosed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L171", + "prompt_id": "fact-water-formula", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": false, + "why": "mismatch", + "extracted": "H₂O", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L61", + "prompt_id": "fact-water-formula", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": false, + "why": "mismatch", + "extracted": "H₂O", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L51", + "prompt_id": "fact-water-formula", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": false, + "why": "mismatch", + "extracted": "H₂O", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L216", + "prompt_id": "fact-water-formula", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": false, + "why": "mismatch", + "extracted": "H₂O", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L105", + "prompt_id": "fact-water-formula", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "水", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L233", + "prompt_id": "nonascii-water-kanji", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "engine": "hf", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "水", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L194", + "prompt_id": "nonascii-water-kanji", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "水", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L242", + "prompt_id": "nonascii-water-kanji", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "水", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L260", + "prompt_id": "nonascii-water-kanji", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "水", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L234", + "prompt_id": "nonascii-water-kanji", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "水", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L262", + "prompt_id": "nonascii-water-kanji", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "水", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L217", + "prompt_id": "nonascii-water-kanji", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "水", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L261", + "prompt_id": "nonascii-water-kanji", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "391", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L126", + "prompt_id": "arith-17x23", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "engine": "hf", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "391", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L195", + "prompt_id": "arith-17x23", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "391", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L243", + "prompt_id": "arith-17x23", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "391", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L172", + "prompt_id": "arith-17x23", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "391", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L62", + "prompt_id": "arith-17x23", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "391", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L52", + "prompt_id": "arith-17x23", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "391", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L218", + "prompt_id": "arith-17x23", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "391", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L106", + "prompt_id": "arith-17x23", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "23", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L129", + "prompt_id": "arith-precedence", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "engine": "hf", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "23", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L196", + "prompt_id": "arith-precedence", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "23", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L244", + "prompt_id": "arith-precedence", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "23", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L173", + "prompt_id": "arith-precedence", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "23", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L63", + "prompt_id": "arith-precedence", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "23", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L53", + "prompt_id": "arith-precedence", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "23", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L219", + "prompt_id": "arith-precedence", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "23", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L107", + "prompt_id": "arith-precedence", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "56", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L130", + "prompt_id": "arith-word-apples", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "engine": "hf", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "56", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L197", + "prompt_id": "arith-word-apples", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "56", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L245", + "prompt_id": "arith-word-apples", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "56", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L174", + "prompt_id": "arith-word-apples", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "56", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L64", + "prompt_id": "arith-word-apples", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "56", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L54", + "prompt_id": "arith-word-apples", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "56", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L220", + "prompt_id": "arith-word-apples", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "56", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L108", + "prompt_id": "arith-word-apples", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "8613", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L131", + "prompt_id": "hard-add-4digit", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "engine": "hf", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "8613", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L198", + "prompt_id": "hard-add-4digit", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "8613", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L246", + "prompt_id": "hard-add-4digit", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": false, + "why": "unclosed think (budget exhausted)", + "extracted": null, + "think": "unclosed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L175", + "prompt_id": "hard-add-4digit", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "8613", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L65", + "prompt_id": "hard-add-4digit", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "8613", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L55", + "prompt_id": "hard-add-4digit", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "8613", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L221", + "prompt_id": "hard-add-4digit", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "8613", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L109", + "prompt_id": "hard-add-4digit", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "15", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L132", + "prompt_id": "hard-not-prime", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "engine": "hf", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "15", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L199", + "prompt_id": "hard-not-prime", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "15", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L247", + "prompt_id": "hard-not-prime", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "15", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L176", + "prompt_id": "hard-not-prime", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "15", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L66", + "prompt_id": "hard-not-prime", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "15", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L56", + "prompt_id": "hard-not-prime", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "15", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L222", + "prompt_id": "hard-not-prime", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "15", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L110", + "prompt_id": "hard-not-prime", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "tenalp", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L133", + "prompt_id": "hard-reverse-word", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "engine": "hf", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "tenalp", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L200", + "prompt_id": "hard-reverse-word", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "tenalp", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L248", + "prompt_id": "hard-reverse-word", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": false, + "why": "mismatch", + "extracted": "teanlap", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L177", + "prompt_id": "hard-reverse-word", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "tenalp", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L67", + "prompt_id": "hard-reverse-word", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "tenalp", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L57", + "prompt_id": "hard-reverse-word", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "tenalp", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L223", + "prompt_id": "hard-reverse-word", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "tenalp", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L111", + "prompt_id": "hard-reverse-word", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "3", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L134", + "prompt_id": "hard-count-letters", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "engine": "hf", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "3", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L201", + "prompt_id": "hard-count-letters", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "3", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L249", + "prompt_id": "hard-count-letters", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "3", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L178", + "prompt_id": "hard-count-letters", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "3", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L68", + "prompt_id": "hard-count-letters", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "3", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L58", + "prompt_id": "hard-count-letters", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "3", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L224", + "prompt_id": "hard-count-letters", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "3", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L112", + "prompt_id": "hard-count-letters", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": false, + "why": "unclosed think (budget exhausted)", + "extracted": null, + "think": "unclosed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L135", + "prompt_id": "recall-cats", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "engine": "hf", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "5", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L202", + "prompt_id": "recall-cats", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "5", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L250", + "prompt_id": "recall-cats", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": false, + "why": "unclosed think (budget exhausted)", + "extracted": null, + "think": "unclosed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L179", + "prompt_id": "recall-cats", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "5", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L69", + "prompt_id": "recall-cats", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "5", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L59", + "prompt_id": "recall-cats", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "5", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L225", + "prompt_id": "recall-cats", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "5", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L113", + "prompt_id": "recall-cats", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "Ada", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L136", + "prompt_id": "recall-name", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "engine": "hf", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "Ada", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L203", + "prompt_id": "recall-name", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "Ada", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L251", + "prompt_id": "recall-name", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": false, + "why": "unclosed think (budget exhausted)", + "extracted": null, + "think": "unclosed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L180", + "prompt_id": "recall-name", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "Ada", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L70", + "prompt_id": "recall-name", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "Ada", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L60", + "prompt_id": "recall-name", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "Ada", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L226", + "prompt_id": "recall-name", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "Ada", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L114", + "prompt_id": "recall-name", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "7", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L137", + "prompt_id": "recall-arith-chain", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "engine": "hf", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "7", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L204", + "prompt_id": "recall-arith-chain", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "7", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L252", + "prompt_id": "recall-arith-chain", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "7", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L181", + "prompt_id": "recall-arith-chain", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "7", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L71", + "prompt_id": "recall-arith-chain", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "7", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L61", + "prompt_id": "recall-arith-chain", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "7", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L227", + "prompt_id": "recall-arith-chain", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "7", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L115", + "prompt_id": "recall-arith-chain", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "def is_palindrome(s):\n s = s.lower()\n return s == s[::-1]\n", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L138", + "prompt_id": "code-is-palindrome", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "engine": "hf", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "def is_palindrome(s):\n return s.lower() == s[::-1].lower()\n", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L205", + "prompt_id": "code-is-palindrome", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": false, + "why": "tests_failed: AssertionError", + "extracted": "def is_palindrome(s):\n return s.lower() == s[::-1]\n", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L253", + "prompt_id": "code-is-palindrome", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": false, + "why": "unclosed think (budget exhausted)", + "extracted": null, + "think": "unclosed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L182", + "prompt_id": "code-is-palindrome", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "def is_palindrome(s):\n cleaned = ''.join(char.lower() for char in s if char.isalnum())\n return cleaned == cleaned[::-1]\n", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L72", + "prompt_id": "code-is-palindrome", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "def is_palindrome(s):\n cleaned = ''.join(char.lower() for char in s if char.isalnum())\n return cleaned == cleaned[::-1]\n", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L62", + "prompt_id": "code-is-palindrome", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "def is_palindrome(s):\n cleaned = ''.join(char.lower() for char in s if char.isalnum())\n return cleaned == cleaned[::-1]\n", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L228", + "prompt_id": "code-is-palindrome", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "def is_palindrome(s):\n cleaned = ''.join(char.lower() for char in s if char.isalnum())\n return cleaned == cleaned[::-1]\n", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L116", + "prompt_id": "code-is-palindrome", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": false, + "why": "unclosed think (budget exhausted)", + "extracted": null, + "think": "unclosed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L139", + "prompt_id": "code-fib", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "engine": "hf", + "verb": "chat", + "host": "gx10", + "correct": false, + "why": "unclosed think (budget exhausted)", + "extracted": null, + "think": "unclosed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L206", + "prompt_id": "code-fib", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": false, + "why": "unclosed think (budget exhausted)", + "extracted": null, + "think": "unclosed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L254", + "prompt_id": "code-fib", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": false, + "why": "unclosed think (budget exhausted)", + "extracted": null, + "think": "unclosed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L183", + "prompt_id": "code-fib", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "def fib(n):\n if n == 0:\n return 0\n elif n == 1:\n return 1\n else:\n a, b = 0, 1\n for _ in range(2, n + 1):\n a, b = b, a + b\n return b\n", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L73", + "prompt_id": "code-fib", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "def fib(n):\n if n == 0:\n return 0\n elif n == 1:\n return 1\n a, b = 0, 1\n for _ in range(2, n + 1):\n a, b = b, a + b\n return b\n", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L63", + "prompt_id": "code-fib", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "def fib(n):\n if n == 0:\n return 0\n elif n == 1:\n return 1\n a, b = 0, 1\n for _ in range(2, n + 1):\n a, b = b, a + b\n return b\n", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L229", + "prompt_id": "code-fib", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "def fib(n):\n if n == 0:\n return 0\n elif n == 1:\n return 1\n else:\n a, b = 0, 1\n for _ in range(2, n + 1):\n a, b = b, a + b\n return b\n", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L117", + "prompt_id": "code-fib", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "def count_vowels(s):\n count = 0\n for char in s:\n if char.lower() in 'aeiou':\n count += 1\n return count\n", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L140", + "prompt_id": "code-count-vowels", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "engine": "hf", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "def count_vowels(s):\n count = 0\n vowels = \"aeiou\"\n for char in s:\n if char.lower() in vowels:\n count += 1\n return count\n", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L207", + "prompt_id": "code-count-vowels", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "def count_vowels(s):\n return sum(1 for char in s if char.lower() in 'aeiou')\n", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L255", + "prompt_id": "code-count-vowels", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": false, + "why": "unclosed think (budget exhausted)", + "extracted": null, + "think": "unclosed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L184", + "prompt_id": "code-count-vowels", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "def count_vowels(s):\n vowels = set('aeiouAEIOU')\n return sum(1 for char in s if char in vowels)\n", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L74", + "prompt_id": "code-count-vowels", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "def count_vowels(s):\n vowels = set('aeiouAEIOU')\n return sum(1 for char in s if char in vowels)\n", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L64", + "prompt_id": "code-count-vowels", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "def count_vowels(s):\n vowels = set('aeiouAEIOU')\n return sum(1 for char in s if char in vowels)\n", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L230", + "prompt_id": "code-count-vowels", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "def count_vowels(s):\n vowels = \"aeiouAEIOU\"\n count = 0\n for char in s:\n if char in vowels:\n count += 1\n return count\n", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L118", + "prompt_id": "code-count-vowels", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": [ + "2", + "3", + "5", + "7", + "11", + "13", + "17", + "19", + "23", + "29" + ], + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L141", + "prompt_id": "long-primes-10", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "engine": "hf", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": [ + "2", + "3", + "5", + "7", + "11", + "13", + "17", + "19", + "23", + "29" + ], + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L208", + "prompt_id": "long-primes-10", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": [ + "2", + "3", + "5", + "7", + "11", + "13", + "17", + "19", + "23", + "29" + ], + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L256", + "prompt_id": "long-primes-10", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": [ + "2", + "3", + "5", + "7", + "11", + "13", + "17", + "19", + "23", + "29" + ], + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L185", + "prompt_id": "long-primes-10", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": [ + "2", + "3", + "5", + "7", + "11", + "13", + "17", + "19", + "23", + "29" + ], + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L75", + "prompt_id": "long-primes-10", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": [ + "2", + "3", + "5", + "7", + "11", + "13", + "17", + "19", + "23", + "29" + ], + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L65", + "prompt_id": "long-primes-10", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": [ + "2", + "3", + "5", + "7", + "11", + "13", + "17", + "19", + "23", + "29" + ], + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L231", + "prompt_id": "long-primes-10", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": [ + "2", + "3", + "5", + "7", + "11", + "13", + "17", + "19", + "23", + "29" + ], + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L119", + "prompt_id": "long-primes-10", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": [ + "15", + "14", + "13", + "12", + "11", + "10", + "9", + "8", + "7", + "6", + "5", + "4", + "3", + "2", + "1" + ], + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L142", + "prompt_id": "long-countdown-15", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "engine": "hf", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": [ + "15", + "14", + "13", + "12", + "11", + "10", + "9", + "8", + "7", + "6", + "5", + "4", + "3", + "2", + "1" + ], + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L209", + "prompt_id": "long-countdown-15", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": [ + "15", + "14", + "13", + "12", + "11", + "10", + "9", + "8", + "7", + "6", + "5", + "4", + "3", + "2", + "1" + ], + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L257", + "prompt_id": "long-countdown-15", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": [ + "15", + "14", + "13", + "12", + "11", + "10", + "9", + "8", + "7", + "6", + "5", + "4", + "3", + "2", + "1" + ], + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L186", + "prompt_id": "long-countdown-15", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": [ + "15", + "14", + "13", + "12", + "11", + "10", + "9", + "8", + "7", + "6", + "5", + "4", + "3", + "2", + "1" + ], + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L76", + "prompt_id": "long-countdown-15", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": [ + "15", + "14", + "13", + "12", + "11", + "10", + "9", + "8", + "7", + "6", + "5", + "4", + "3", + "2", + "1" + ], + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L66", + "prompt_id": "long-countdown-15", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": [ + "15", + "14", + "13", + "12", + "11", + "10", + "9", + "8", + "7", + "6", + "5", + "4", + "3", + "2", + "1" + ], + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L232", + "prompt_id": "long-countdown-15", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": [ + "15", + "14", + "13", + "12", + "11", + "10", + "9", + "8", + "7", + "6", + "5", + "4", + "3", + "2", + "1" + ], + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L120", + "prompt_id": "long-countdown-15", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + } + ] +}