From 70426faf472c420ccafa726df614033c2849f9a2 Mon Sep 17 00:00:00 2001 From: Noah Gift Date: Mon, 12 Oct 2026 08:42:13 +0200 Subject: [PATCH] release: 0.71.1 bump-version.sh 0.71.1 (every workspace, facades included; --check green), the CHANGELOG [0.71.1] section drafted from the 5 PRs merged since v0.71.0, and the CRUX prompt certification carried from 0.71.0 to evidence/crux/0.71.1 with apr_commit set to this bump's parent, as prepare_bump.sh carries it. Cut by hand at the parent: prepare_bump.sh cuts only at a T-2 GO commit, and T-2 has never produced a GO. Closes #5183 Refs: #3998 Agent: aprender-cd Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 34 + Cargo.lock | 168 +- Cargo.toml | 148 +- crates/apr-cli/Cargo.toml | 8 +- crates/aprender-bench-compute/Cargo.toml | 2 +- crates/aprender-bench-tokenizer/Cargo.toml | 2 +- crates/aprender-cbtop/Cargo.toml | 2 +- crates/aprender-cgp/Cargo.toml | 2 +- crates/aprender-compute/Cargo.toml | 4 +- crates/aprender-contracts-cli/Cargo.toml | 2 +- crates/aprender-contracts/Cargo.toml | 2 +- crates/aprender-cuda-edge/Cargo.toml | 2 +- crates/aprender-explain/Cargo.toml | 2 +- crates/aprender-graph/Cargo.toml | 2 +- crates/aprender-monte-carlo/Cargo.toml | 2 +- crates/aprender-orchestrate/Cargo.toml | 2 +- crates/aprender-profile/Cargo.toml | 4 +- crates/aprender-qa-cli/Cargo.toml | 8 +- crates/aprender-qa-report/Cargo.toml | 4 +- crates/aprender-qa-runner/Cargo.toml | 4 +- crates/aprender-rag-cli/Cargo.toml | 2 +- crates/aprender-rag/Cargo.toml | 2 +- crates/aprender-registry/Cargo.toml | 2 +- crates/aprender-serve/Cargo.toml | 4 +- crates/aprender-shell/Cargo.toml | 2 +- crates/aprender-simulate/Cargo.toml | 2 +- crates/aprender-test-cli/Cargo.toml | 2 +- crates/aprender-train-bench/Cargo.toml | 2 +- crates/aprender-train-distill/Cargo.toml | 2 +- crates/aprender-train-inspect/Cargo.toml | 2 +- crates/aprender-train-lora/Cargo.toml | 2 +- crates/aprender-train-shell/Cargo.toml | 2 +- crates/aprender-train/Cargo.toml | 4 +- crates/aprender-tsp/Cargo.toml | 2 +- crates/aprender-verify-ml/Cargo.toml | 2 +- crates/aprender-viz/Cargo.toml | 6 +- crates/facades/Cargo.lock | 4 +- .../provable-contracts-macros/Cargo.toml | 2 +- crates/facades/provable-contracts/Cargo.toml | 2 +- .../prompt-certification-inventory.json | 13 + .../crux/0.71.1/prompt-certification.json | 8906 +++++++++++++++++ 41 files changed, 9162 insertions(+), 209 deletions(-) create mode 100644 evidence/crux/0.71.1/prompt-certification-inventory.json create mode 100644 evidence/crux/0.71.1/prompt-certification.json diff --git a/CHANGELOG.md b/CHANGELOG.md index aaa1400518..e7f92a603d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,40 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [0.71.1] - 2026-10-12 + +A patch release on 0.71.0: the published aprender-core crate compiles again, a tag push runs the full workspace, the CLI-tools bin has its surface row, the CRUX judge's two new definitions are back at grade B, and under the standing release policy the pre-publish dogfood (R5) leaves the release path the way readiness (R8) did, with CRUX smoke (R7) the release gate. + +### Fixed + +- fix(core): f32_bits_tests reads its ECE fixture at run time, so the published crate compiles (#5179) +- fix(ci): a tag push runs the full workspace and pins its first parent as the base (Fixes #5172) (#5178) +- fix(ledger): aprender-ci-tools shipped with no surface row, so the bin gate read 29/30 (#5170) (#5175) +- fix(crux-judge): scan_once and main back at grade B — CB-200 616 again (#5173) (#5180) +- fix(release): R5 never went green — not run under the standing policy, R8's form (#5174) (#5177) + +### Known issues +- **Qwen3 MoE (Qwen3-30B-A3B, Qwen3-Coder-30B-A3B) on GB10 is not claimed for 0.70 (#3715); planned for 0.70.1.** + On GB10 (sm_121) every verb (run, chat, serve, code) times out at 600 s on the 20k-token rung. The 0.69.1 + pre-load refusal is gone in 0.70.0, so these models now load and run, but too slowly to be claimed. The + release gate does not owe the qwen3moe x GB10 cells (`cells.declaimed` in + `contracts/model-capability-ladder-v1.yaml`), and the ladder gate is RED if any rung or long-rung + representative still claims them. Withdrawn, not waived: lambda holds both files and still owes every + qwen3moe cell. +- **GB10 dense models: prompt processing unbatched; planned for 0.70.1 (#4590).** Widens the Qwen3 entry below to every dense model on GB10 (operator ruling 2026-09-28 + 16:15Z): the serial prefill is the sm_12x default for every non-hybrid arch, so dense Qwen2.5 on GB10 + is de-claimed too (`cells.declaimed`: qwen2 and qwen3 on gx10). Output is correct; only prefill speed + is affected. Withdrawn, not waived: lambda still owes every dense cell, gx10 still owes every Qwen3.5 cell. +- **Qwen3 on GB10 (sm_121, and the sm_12x family) is correct but very slow, and is not claimed for + 0.70 (#4590).** Dense Qwen3 on sm_12x prefills one token at a time, the sm_12x default, because the + generic batched prefill corrupts the KV cache there (FALSIFY-CPU-GPU-009). Measured on 0.70.0 + (8d021f61e) at an 8k think-on prompt: Qwen3-1.7B 967 s, Qwen3-8B 2163 s, against 63-80 s for + Qwen3.5-9B on the same host (Qwen3.5 has its own batched prefill, #3596). `apr` prints a + `[KNOWN-ISSUE #4590]` warning when it loads Qwen3 on sm_12x. The release gate does not owe the + Qwen3 x GB10 cells (`cells.declaimed` in `contracts/model-capability-ladder-v1.yaml`); every other + host still owes every Qwen3 cell. Planned for 0.70.1: native sm_121 SASS, then a re-test of batched + prefill on GB10. + ## [0.71.0] - 2026-10-11 The 0.71.0 train: release checks move into code with a fixed stage budget, checks that cannot block a merge or a release run nightly instead of on the PR path, a nightly release rehearsal and nightly train judge one commit end to end, the CI helpers and the CRUX judge are ported to Rust, the Laya ModernBERT encoder lands, and Qwen3.5 hybrid GGUF export loads in llama.cpp. From this release on, a release ships on CRUX smoke; larger model rows run nightly and each red row is a ticket. diff --git a/Cargo.lock b/Cargo.lock index 12091a73b2..8de785cc78 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -248,7 +248,7 @@ dependencies = [ [[package]] name = "apr-cli" -version = "0.71.0" +version = "0.71.1" dependencies = [ "anyhow", "aprender-build-sha", @@ -325,7 +325,7 @@ dependencies = [ [[package]] name = "apr-format" -version = "0.71.0" +version = "0.71.1" dependencies = [ "bincode", "cargo_metadata", @@ -343,7 +343,7 @@ dependencies = [ [[package]] name = "aprender" -version = "0.71.0" +version = "0.71.1" dependencies = [ "apr-cli", "aprender-core", @@ -351,7 +351,7 @@ dependencies = [ [[package]] name = "aprender-bench-compute" -version = "0.71.0" +version = "0.71.1" dependencies = [ "aprender-core", "criterion 0.7.0", @@ -360,7 +360,7 @@ dependencies = [ [[package]] name = "aprender-bench-tokenizer" -version = "0.71.0" +version = "0.71.1" dependencies = [ "aprender-core", "criterion 0.7.0", @@ -369,14 +369,14 @@ dependencies = [ [[package]] name = "aprender-build-sha" -version = "0.71.0" +version = "0.71.1" dependencies = [ "tempfile", ] [[package]] name = "aprender-cbtop" -version = "0.71.0" +version = "0.71.1" dependencies = [ "anyhow", "aprender-build-sha", @@ -402,7 +402,7 @@ dependencies = [ [[package]] name = "aprender-cgp" -version = "0.71.0" +version = "0.71.1" dependencies = [ "anyhow", "aprender-build-sha", @@ -423,7 +423,7 @@ dependencies = [ [[package]] name = "aprender-ci-tools" -version = "0.71.0" +version = "0.71.1" dependencies = [ "aprender-build-sha", "clap", @@ -436,7 +436,7 @@ dependencies = [ [[package]] name = "aprender-common" -version = "0.71.0" +version = "0.71.1" dependencies = [ "clap", "lz4_flex 0.11.6", @@ -445,7 +445,7 @@ dependencies = [ [[package]] name = "aprender-compute" -version = "0.71.0" +version = "0.71.1" dependencies = [ "anyhow", "aprender-contracts", @@ -495,7 +495,7 @@ dependencies = [ [[package]] name = "aprender-compute-xtask" -version = "0.71.0" +version = "0.71.1" dependencies = [ "anyhow", "aprender-build-sha", @@ -510,7 +510,7 @@ dependencies = [ [[package]] name = "aprender-contracts" -version = "0.71.0" +version = "0.71.1" dependencies = [ "aprender-contracts-macros", "blake3", @@ -530,7 +530,7 @@ dependencies = [ [[package]] name = "aprender-contracts-cli" -version = "0.71.0" +version = "0.71.1" dependencies = [ "aprender-build-sha", "aprender-common", @@ -546,7 +546,7 @@ dependencies = [ [[package]] name = "aprender-contracts-macros" -version = "0.71.0" +version = "0.71.1" dependencies = [ "proc-macro2", "quote", @@ -556,7 +556,7 @@ dependencies = [ [[package]] name = "aprender-contrastive-data" -version = "0.71.0" +version = "0.71.1" dependencies = [ "aprender-contracts-macros", "aprender-rand", @@ -571,7 +571,7 @@ dependencies = [ [[package]] name = "aprender-core" -version = "0.71.0" +version = "0.71.1" dependencies = [ "aes-gcm", "alsa", @@ -625,7 +625,7 @@ dependencies = [ [[package]] name = "aprender-cuda-edge" -version = "0.71.0" +version = "0.71.1" dependencies = [ "aprender-gpu", "proptest", @@ -636,7 +636,7 @@ dependencies = [ [[package]] name = "aprender-cupti" -version = "0.71.0" +version = "0.71.1" dependencies = [ "bindgen", "bitflags 2.13.0", @@ -646,7 +646,7 @@ dependencies = [ [[package]] name = "aprender-data" -version = "0.71.0" +version = "0.71.1" dependencies = [ "aes-gcm", "aprender-build-sha", @@ -699,7 +699,7 @@ dependencies = [ [[package]] name = "aprender-db" -version = "0.71.0" +version = "0.71.1" dependencies = [ "anyhow", "aprender-build-sha", @@ -746,7 +746,7 @@ dependencies = [ [[package]] name = "aprender-decide" -version = "0.71.0" +version = "0.71.1" dependencies = [ "aprender-contracts-macros", "aprender-core", @@ -765,7 +765,7 @@ dependencies = [ [[package]] name = "aprender-distribute" -version = "0.71.0" +version = "0.71.1" dependencies = [ "aprender-compute", "aprender-db", @@ -795,7 +795,7 @@ dependencies = [ [[package]] name = "aprender-explain" -version = "0.71.0" +version = "0.71.1" dependencies = [ "aprender-build-sha", "aprender-gpu", @@ -814,7 +814,7 @@ dependencies = [ [[package]] name = "aprender-fft" -version = "0.71.0" +version = "0.71.1" dependencies = [ "criterion 0.7.0", "proptest", @@ -823,7 +823,7 @@ dependencies = [ [[package]] name = "aprender-gemm-codegen" -version = "0.71.0" +version = "0.71.1" dependencies = [ "proc-macro2", "quote", @@ -832,7 +832,7 @@ dependencies = [ [[package]] name = "aprender-gpu" -version = "0.71.0" +version = "0.71.1" dependencies = [ "aprender-common", "aprender-simulate", @@ -850,7 +850,7 @@ dependencies = [ [[package]] name = "aprender-graph" -version = "0.71.0" +version = "0.71.1" dependencies = [ "anyhow", "aprender-compute", @@ -871,7 +871,7 @@ dependencies = [ [[package]] name = "aprender-image" -version = "0.71.0" +version = "0.71.1" dependencies = [ "criterion 0.7.0", "proptest", @@ -880,7 +880,7 @@ dependencies = [ [[package]] name = "aprender-mcp" -version = "0.71.0" +version = "0.71.1" dependencies = [ "inventory", "jsonschema", @@ -893,7 +893,7 @@ dependencies = [ [[package]] name = "aprender-monte-carlo" -version = "0.71.0" +version = "0.71.1" dependencies = [ "aprender", "assert_cmd", @@ -912,7 +912,7 @@ dependencies = [ [[package]] name = "aprender-orchestrate" -version = "0.71.0" +version = "0.71.1" dependencies = [ "anyhow", "aprender-build-sha", @@ -983,7 +983,7 @@ dependencies = [ [[package]] name = "aprender-present-cli" -version = "0.71.0" +version = "0.71.1" dependencies = [ "aprender-build-sha", "aprender-contracts", @@ -998,7 +998,7 @@ dependencies = [ [[package]] name = "aprender-present-core" -version = "0.71.0" +version = "0.71.1" dependencies = [ "aprender-contracts", "aprender-contracts-macros", @@ -1011,7 +1011,7 @@ dependencies = [ [[package]] name = "aprender-present-layout" -version = "0.71.0" +version = "0.71.1" dependencies = [ "aprender-present-core", "criterion 0.7.0", @@ -1022,7 +1022,7 @@ dependencies = [ [[package]] name = "aprender-present-lib" -version = "0.71.0" +version = "0.71.1" dependencies = [ "aprender-contracts", "aprender-present-core", @@ -1051,7 +1051,7 @@ dependencies = [ [[package]] name = "aprender-present-terminal" -version = "0.71.0" +version = "0.71.1" dependencies = [ "aprender-build-sha", "aprender-contracts", @@ -1073,7 +1073,7 @@ dependencies = [ [[package]] name = "aprender-present-test" -version = "0.71.0" +version = "0.71.1" dependencies = [ "aprender-present-core", "aprender-present-test-macros", @@ -1084,7 +1084,7 @@ dependencies = [ [[package]] name = "aprender-present-test-macros" -version = "0.71.0" +version = "0.71.1" dependencies = [ "proc-macro2", "quote", @@ -1093,7 +1093,7 @@ dependencies = [ [[package]] name = "aprender-present-widgets" -version = "0.71.0" +version = "0.71.1" dependencies = [ "aprender-present-core", "aprender-present-yaml", @@ -1105,7 +1105,7 @@ dependencies = [ [[package]] name = "aprender-present-yaml" -version = "0.71.0" +version = "0.71.1" dependencies = [ "aprender-present-core", "proptest", @@ -1115,7 +1115,7 @@ dependencies = [ [[package]] name = "aprender-profile" -version = "0.71.0" +version = "0.71.1" dependencies = [ "addr2line 0.25.1", "anyhow", @@ -1168,7 +1168,7 @@ dependencies = [ [[package]] name = "aprender-profile-core" -version = "0.71.0" +version = "0.71.1" dependencies = [ "hex", "serde", @@ -1178,7 +1178,7 @@ dependencies = [ [[package]] name = "aprender-ptx-debug" -version = "0.71.0" +version = "0.71.1" dependencies = [ "aprender-build-sha", "clap", @@ -1188,7 +1188,7 @@ dependencies = [ [[package]] name = "aprender-qa-certify" -version = "0.71.0" +version = "0.71.1" dependencies = [ "aprender-build-sha", "chrono", @@ -1200,7 +1200,7 @@ dependencies = [ [[package]] name = "aprender-qa-cli" -version = "0.71.0" +version = "0.71.1" dependencies = [ "aprender-build-sha", "aprender-qa-certify", @@ -1220,7 +1220,7 @@ dependencies = [ [[package]] name = "aprender-qa-gen" -version = "0.71.0" +version = "0.71.1" dependencies = [ "anyhow", "chrono", @@ -1237,7 +1237,7 @@ dependencies = [ [[package]] name = "aprender-qa-report" -version = "0.71.0" +version = "0.71.1" dependencies = [ "anyhow", "aprender-qa-gen", @@ -1255,7 +1255,7 @@ dependencies = [ [[package]] name = "aprender-qa-runner" -version = "0.71.0" +version = "0.71.1" dependencies = [ "anyhow", "aprender-qa-gen", @@ -1280,7 +1280,7 @@ dependencies = [ [[package]] name = "aprender-quant" -version = "0.71.0" +version = "0.71.1" dependencies = [ "half", "serde_json", @@ -1288,7 +1288,7 @@ dependencies = [ [[package]] name = "aprender-rag" -version = "0.71.0" +version = "0.71.1" dependencies = [ "aprender-common", "aprender-compute", @@ -1317,7 +1317,7 @@ dependencies = [ [[package]] name = "aprender-rag-cli" -version = "0.71.0" +version = "0.71.1" dependencies = [ "anyhow", "aprender-build-sha", @@ -1337,7 +1337,7 @@ dependencies = [ [[package]] name = "aprender-rand" -version = "0.71.0" +version = "0.71.1" dependencies = [ "criterion 0.7.0", "proptest", @@ -1346,7 +1346,7 @@ dependencies = [ [[package]] name = "aprender-registry" -version = "0.71.0" +version = "0.71.1" dependencies = [ "anyhow", "aprender-core", @@ -1376,7 +1376,7 @@ dependencies = [ [[package]] name = "aprender-review-experiment" -version = "0.71.0" +version = "0.71.1" dependencies = [ "serde", "serde_json", @@ -1386,7 +1386,7 @@ dependencies = [ [[package]] name = "aprender-serve" -version = "0.71.0" +version = "0.71.1" dependencies = [ "anyhow", "approx", @@ -1457,7 +1457,7 @@ dependencies = [ [[package]] name = "aprender-shell" -version = "0.71.0" +version = "0.71.1" dependencies = [ "aprender-core", "clap", @@ -1472,7 +1472,7 @@ dependencies = [ [[package]] name = "aprender-simulate" -version = "0.71.0" +version = "0.71.1" dependencies = [ "aprender-build-sha", "aprender-contracts", @@ -1514,7 +1514,7 @@ dependencies = [ [[package]] name = "aprender-solve" -version = "0.71.0" +version = "0.71.1" dependencies = [ "criterion 0.7.0", "half", @@ -1524,7 +1524,7 @@ dependencies = [ [[package]] name = "aprender-sparse" -version = "0.71.0" +version = "0.71.1" dependencies = [ "criterion 0.7.0", "proptest", @@ -1533,7 +1533,7 @@ dependencies = [ [[package]] name = "aprender-tensor" -version = "0.71.0" +version = "0.71.1" dependencies = [ "criterion 0.7.0", "proptest", @@ -1542,7 +1542,7 @@ dependencies = [ [[package]] name = "aprender-test-cli" -version = "0.71.0" +version = "0.71.1" dependencies = [ "aprender-build-sha", "aprender-test-lib", @@ -1571,7 +1571,7 @@ dependencies = [ [[package]] name = "aprender-test-derive" -version = "0.71.0" +version = "0.71.1" dependencies = [ "proc-macro2", "quote", @@ -1581,7 +1581,7 @@ dependencies = [ [[package]] name = "aprender-test-js-gen" -version = "0.71.0" +version = "0.71.1" dependencies = [ "blake3", "chrono", @@ -1597,7 +1597,7 @@ dependencies = [ [[package]] name = "aprender-test-lib" -version = "0.71.0" +version = "0.71.1" dependencies = [ "aprender-compute", "aprender-present-core", @@ -1643,7 +1643,7 @@ dependencies = [ [[package]] name = "aprender-test-showcase" -version = "0.71.0" +version = "0.71.1" dependencies = [ "aprender-present-terminal", "aprender-test-lib", @@ -1660,7 +1660,7 @@ dependencies = [ [[package]] name = "aprender-train" -version = "0.71.0" +version = "0.71.1" dependencies = [ "approx", "aprender-common", @@ -1729,7 +1729,7 @@ dependencies = [ [[package]] name = "aprender-train-bench" -version = "0.71.0" +version = "0.71.1" dependencies = [ "aprender-build-sha", "aprender-train", @@ -1747,7 +1747,7 @@ dependencies = [ [[package]] name = "aprender-train-common" -version = "0.71.0" +version = "0.71.1" dependencies = [ "aprender-viz", "clap", @@ -1759,7 +1759,7 @@ dependencies = [ [[package]] name = "aprender-train-distill" -version = "0.71.0" +version = "0.71.1" dependencies = [ "aprender-build-sha", "aprender-train", @@ -1781,7 +1781,7 @@ dependencies = [ [[package]] name = "aprender-train-inspect" -version = "0.71.0" +version = "0.71.1" dependencies = [ "aprender-build-sha", "aprender-train", @@ -1798,7 +1798,7 @@ dependencies = [ [[package]] name = "aprender-train-lora" -version = "0.71.0" +version = "0.71.1" dependencies = [ "aprender-build-sha", "aprender-train", @@ -1815,7 +1815,7 @@ dependencies = [ [[package]] name = "aprender-train-shell" -version = "0.71.0" +version = "0.71.1" dependencies = [ "aprender-build-sha", "aprender-train", @@ -1832,7 +1832,7 @@ dependencies = [ [[package]] name = "aprender-train-wasm" -version = "0.71.0" +version = "0.71.1" dependencies = [ "proptest", "serde", @@ -1843,7 +1843,7 @@ dependencies = [ [[package]] name = "aprender-tsp" -version = "0.71.0" +version = "0.71.1" dependencies = [ "aprender", "assert_cmd", @@ -1858,7 +1858,7 @@ dependencies = [ [[package]] name = "aprender-update" -version = "0.71.0" +version = "0.71.1" dependencies = [ "flate2", "semver", @@ -1872,7 +1872,7 @@ dependencies = [ [[package]] name = "aprender-verify" -version = "0.71.0" +version = "0.71.1" dependencies = [ "chrono", "criterion 0.5.1", @@ -1884,7 +1884,7 @@ dependencies = [ [[package]] name = "aprender-verify-ml" -version = "0.71.0" +version = "0.71.1" dependencies = [ "aprender-build-sha", "aprender-compute", @@ -1915,7 +1915,7 @@ dependencies = [ [[package]] name = "aprender-viz" -version = "0.71.0" +version = "0.71.1" dependencies = [ "approx", "aprender-common", @@ -1944,7 +1944,7 @@ dependencies = [ [[package]] name = "aprender-zram" -version = "0.71.0" +version = "0.71.1" dependencies = [ "aprender-zram-adaptive", "aprender-zram-core", @@ -1954,7 +1954,7 @@ dependencies = [ [[package]] name = "aprender-zram-adaptive" -version = "0.71.0" +version = "0.71.1" dependencies = [ "aprender-zram-core", "proptest", @@ -1963,7 +1963,7 @@ dependencies = [ [[package]] name = "aprender-zram-cli" -version = "0.71.0" +version = "0.71.1" dependencies = [ "aprender-build-sha", "aprender-zram-adaptive", @@ -1978,7 +1978,7 @@ dependencies = [ [[package]] name = "aprender-zram-core" -version = "0.71.0" +version = "0.71.1" dependencies = [ "aprender-gpu", "criterion 0.7.0", @@ -1990,7 +1990,7 @@ dependencies = [ [[package]] name = "aprender-zram-generator" -version = "0.71.0" +version = "0.71.1" dependencies = [ "aprender-build-sha", "aprender-zram-core", diff --git a/Cargo.toml b/Cargo.toml index bc1e617da3..3c0f789220 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -162,7 +162,7 @@ exclude = [ resolver = "2" [workspace.package] -version = "0.71.0" +version = "0.71.1" edition = "2021" license = "MIT" repository = "https://github.com/paiml/aprender" @@ -244,11 +244,11 @@ tokenizers = { version = "0.23.1", default-features = false, features = ["fancy- # aprender-rand (lib name `trueno_rand`) is a workspace MEMBER but had no # [workspace.dependencies] entry, so every consumer had to re-declare the path. # It is the counter-based Philox RNG behind the SetFit encoder's dropout masks. -aprender-rand = { path = "crates/aprender-rand", version = "0.71.0" } +aprender-rand = { path = "crates/aprender-rand", version = "0.71.1" } # aprender-core (lib name `aprender`): a consumer that hand-pins it falls behind the # workspace version, and the rc stamp then cannot resolve it (aprender-decide pinned # 0.70.2 under a 0.70.3 workspace). Inherit this entry instead. -aprender-core = { path = "crates/aprender-core", version = "0.71.0" } +aprender-core = { path = "crates/aprender-core", version = "0.71.1" } # Chat templates minijinja = { version = "2.14", features = ["loader", "serde"] } @@ -258,9 +258,9 @@ minijinja = { version = "2.14", features = ["loader", "serde"] } # `crates/aprender-contracts` (lib name `provable_contracts`). Declaring it from # crates.io compiled a SECOND copy of the contract engine alongside the in-tree # one — exactly the duplicate-crate + publish-cycle the consolidation removed. -provable-contracts = { path = "crates/aprender-contracts", version = "0.71.0", package = "aprender-contracts" } -provable-contracts-macros = { path = "crates/aprender-contracts-macros", version = "0.71.0", package = "aprender-contracts-macros" } -aprender-contracts-macros = { path = "crates/aprender-contracts-macros", version = "0.71.0" } +provable-contracts = { path = "crates/aprender-contracts", version = "0.71.1", package = "aprender-contracts" } +provable-contracts-macros = { path = "crates/aprender-contracts-macros", version = "0.71.1", package = "aprender-contracts-macros" } +aprender-contracts-macros = { path = "crates/aprender-contracts-macros", version = "0.71.1" } # YAML serde_yaml = "0.9" @@ -272,69 +272,69 @@ hf-hub = { version = "0.4", default-features = false, features = ["ureq"] } hf-xet = "1.5.1" # Workspace-internal crates (paths within the monorepo) -aprender-compute = { path = "crates/aprender-compute", version = "0.71.0" } -aprender-gpu = { path = "crates/aprender-gpu", version = "0.71.0" } -aprender-quant = { path = "crates/aprender-quant", version = "0.71.0" } -aprender-serve = { path = "crates/aprender-serve", version = "0.71.0" } -realizar = { path = "crates/aprender-serve", version = "0.71.0", package = "aprender-serve" } -alimentar = { path = "crates/aprender-data", version = "0.71.0", package = "aprender-data" } -pacha = { path = "crates/aprender-registry", version = "0.71.0", package = "aprender-registry" } +aprender-compute = { path = "crates/aprender-compute", version = "0.71.1" } +aprender-gpu = { path = "crates/aprender-gpu", version = "0.71.1" } +aprender-quant = { path = "crates/aprender-quant", version = "0.71.1" } +aprender-serve = { path = "crates/aprender-serve", version = "0.71.1" } +realizar = { path = "crates/aprender-serve", version = "0.71.1", package = "aprender-serve" } +alimentar = { path = "crates/aprender-data", version = "0.71.1", package = "aprender-data" } +pacha = { path = "crates/aprender-registry", version = "0.71.1", package = "aprender-registry" } # APR-MONO self-containment: legacy sibling lib-name aliases -> in-monorepo crates (2026-06-12) -trueno-quant = { path = "crates/aprender-quant", version = "0.71.0", package = "aprender-quant" } -trueno-db = { path = "crates/aprender-db", version = "0.71.0", package = "aprender-db" } -trueno-viz = { path = "crates/aprender-viz", version = "0.71.0", package = "aprender-viz" } -trueno-rag = { path = "crates/aprender-rag", version = "0.71.0", package = "aprender-rag" } -trueno-cuda-edge = { path = "crates/aprender-cuda-edge", version = "0.71.0", package = "aprender-cuda-edge" } -trueno-sparse = { path = "crates/aprender-sparse", version = "0.71.0", package = "aprender-sparse" } -trueno-solve = { path = "crates/aprender-solve", version = "0.71.0", package = "aprender-solve" } -renacer-core = { path = "crates/aprender-profile-core", version = "0.71.0", package = "aprender-profile-core" } -repartir = { path = "crates/aprender-distribute", version = "0.71.0", package = "aprender-distribute" } -simular = { path = "crates/aprender-simulate", version = "0.71.0", package = "aprender-simulate" } -probar = { path = "crates/aprender-test-lib", version = "0.71.0", package = "aprender-test-lib" } +trueno-quant = { path = "crates/aprender-quant", version = "0.71.1", package = "aprender-quant" } +trueno-db = { path = "crates/aprender-db", version = "0.71.1", package = "aprender-db" } +trueno-viz = { path = "crates/aprender-viz", version = "0.71.1", package = "aprender-viz" } +trueno-rag = { path = "crates/aprender-rag", version = "0.71.1", package = "aprender-rag" } +trueno-cuda-edge = { path = "crates/aprender-cuda-edge", version = "0.71.1", package = "aprender-cuda-edge" } +trueno-sparse = { path = "crates/aprender-sparse", version = "0.71.1", package = "aprender-sparse" } +trueno-solve = { path = "crates/aprender-solve", version = "0.71.1", package = "aprender-solve" } +renacer-core = { path = "crates/aprender-profile-core", version = "0.71.1", package = "aprender-profile-core" } +repartir = { path = "crates/aprender-distribute", version = "0.71.1", package = "aprender-distribute" } +simular = { path = "crates/aprender-simulate", version = "0.71.1", package = "aprender-simulate" } +probar = { path = "crates/aprender-test-lib", version = "0.71.1", package = "aprender-test-lib" } # English key for the same crate, named after the command it serves (`apr test`). # The dep KEY, not the package name, is what becomes the extern crate name when # the two differ — the lib target is still `jugar_probar`, so keying it by its # package name would have imported Spanish anyway. Renaming that lib target is # 196 call sites in 17 crates and gets its own PR; this key is how new code # reads English today. -apr-test = { path = "crates/aprender-test-lib", version = "0.71.0", package = "aprender-test-lib", default-features = false, features = ["llm"] } +apr-test = { path = "crates/aprender-test-lib", version = "0.71.1", package = "aprender-test-lib", default-features = false, features = ["llm"] } # `jugar-probar` is the crates.io/lib name of the same in-tree crate as `probar` # above. Both aliases resolve to ONE package; consumers that spelled it # `jugar-probar = "0.4"|"0.5"|"1.0"` were building three extra copies. -jugar-probar = { path = "crates/aprender-test-lib", version = "0.71.0", package = "aprender-test-lib" } +jugar-probar = { path = "crates/aprender-test-lib", version = "0.71.1", package = "aprender-test-lib" } # NOTE: the matching `jugar-probar-derive` alias already exists further down this # same table (it points at crates/aprender-test-derive). crates/aprender-test-lib # was pinning `jugar-probar-derive = "=1.0.3"` from crates.io regardless — the # alias was sitting there unused. -aprender-train = { path = "crates/aprender-train", version = "0.71.0" } -entrenar = { path = "crates/aprender-train", version = "0.71.0", package = "aprender-train" } -aprender-train-common = { path = "crates/aprender-train-common", version = "0.71.0" } -aprender-train-distill = { path = "crates/aprender-train-distill", version = "0.71.0" } -aprender-orchestrate = { path = "crates/aprender-orchestrate", version = "0.71.0" } -aprender-contracts = { path = "crates/aprender-contracts", version = "0.71.0" } -aprender-build-sha = { path = "crates/aprender-build-sha", version = "0.71.0" } -aprender-profile = { path = "crates/aprender-profile", version = "0.71.0" } -aprender-profile-core = { path = "crates/aprender-profile-core", version = "0.71.0" } -aprender-zram-core = { path = "crates/aprender-zram-core", version = "0.71.0" } -aprender-zram-adaptive = { path = "crates/aprender-zram-adaptive", version = "0.71.0" } -aprender-zram-cli = { path = "crates/aprender-zram-cli", version = "0.71.0" } -aprender-present-core = { path = "crates/aprender-present-core", version = "0.71.0" } -aprender-present-terminal = { path = "crates/aprender-present-terminal", version = "0.71.0" } -aprender-present-widgets = { path = "crates/aprender-present-widgets", version = "0.71.0" } -aprender-present-layout = { path = "crates/aprender-present-layout", version = "0.71.0" } -aprender-present-yaml = { path = "crates/aprender-present-yaml", version = "0.71.0" } -aprender-present-test = { path = "crates/aprender-present-test", version = "0.71.0" } -aprender-db = { path = "crates/aprender-db", version = "0.71.0" } -aprender-graph = { path = "crates/aprender-graph", version = "0.71.0" } -aprender-rag = { path = "crates/aprender-rag", version = "0.71.0" } -aprender-rag-cli = { path = "crates/aprender-rag-cli", version = "0.71.0" } -aprender-qa-cli = { path = "crates/aprender-qa-cli", version = "0.71.0" } -aprender-cgp = { path = "crates/aprender-cgp", version = "0.71.0" } -aprender-contrastive-data = { path = "crates/aprender-contrastive-data", version = "0.71.0" } -aprender-contracts-cli = { path = "crates/aprender-contracts-cli", version = "0.71.0" } -aprender-update = { path = "crates/aprender-update", version = "0.71.0" } -aprender-data = { path = "crates/aprender-data", version = "0.71.0" } -aprender-ptx-debug = { path = "crates/aprender-ptx-debug", version = "0.71.0" } +aprender-train = { path = "crates/aprender-train", version = "0.71.1" } +entrenar = { path = "crates/aprender-train", version = "0.71.1", package = "aprender-train" } +aprender-train-common = { path = "crates/aprender-train-common", version = "0.71.1" } +aprender-train-distill = { path = "crates/aprender-train-distill", version = "0.71.1" } +aprender-orchestrate = { path = "crates/aprender-orchestrate", version = "0.71.1" } +aprender-contracts = { path = "crates/aprender-contracts", version = "0.71.1" } +aprender-build-sha = { path = "crates/aprender-build-sha", version = "0.71.1" } +aprender-profile = { path = "crates/aprender-profile", version = "0.71.1" } +aprender-profile-core = { path = "crates/aprender-profile-core", version = "0.71.1" } +aprender-zram-core = { path = "crates/aprender-zram-core", version = "0.71.1" } +aprender-zram-adaptive = { path = "crates/aprender-zram-adaptive", version = "0.71.1" } +aprender-zram-cli = { path = "crates/aprender-zram-cli", version = "0.71.1" } +aprender-present-core = { path = "crates/aprender-present-core", version = "0.71.1" } +aprender-present-terminal = { path = "crates/aprender-present-terminal", version = "0.71.1" } +aprender-present-widgets = { path = "crates/aprender-present-widgets", version = "0.71.1" } +aprender-present-layout = { path = "crates/aprender-present-layout", version = "0.71.1" } +aprender-present-yaml = { path = "crates/aprender-present-yaml", version = "0.71.1" } +aprender-present-test = { path = "crates/aprender-present-test", version = "0.71.1" } +aprender-db = { path = "crates/aprender-db", version = "0.71.1" } +aprender-graph = { path = "crates/aprender-graph", version = "0.71.1" } +aprender-rag = { path = "crates/aprender-rag", version = "0.71.1" } +aprender-rag-cli = { path = "crates/aprender-rag-cli", version = "0.71.1" } +aprender-qa-cli = { path = "crates/aprender-qa-cli", version = "0.71.1" } +aprender-cgp = { path = "crates/aprender-cgp", version = "0.71.1" } +aprender-contrastive-data = { path = "crates/aprender-contrastive-data", version = "0.71.1" } +aprender-contracts-cli = { path = "crates/aprender-contracts-cli", version = "0.71.1" } +aprender-update = { path = "crates/aprender-update", version = "0.71.1" } +aprender-data = { path = "crates/aprender-data", version = "0.71.1" } +aprender-ptx-debug = { path = "crates/aprender-ptx-debug", version = "0.71.1" } # Additional external deps needed by sub-crates serde_yaml_ng = "0.10" @@ -393,25 +393,25 @@ wasmtime = "48.0.3" bollard = "0.17" # Old crate name aliases (workspace = true refs in migrated sub-crates) -trueno-zram-core = { path = "crates/aprender-zram-core", version = "0.71.0", package = "aprender-zram-core" } -trueno-zram-adaptive = { path = "crates/aprender-zram-adaptive", version = "0.71.0", package = "aprender-zram-adaptive" } -trueno = { path = "crates/aprender-compute", version = "0.71.0", package = "aprender-compute" } -presentar-core = { path = "crates/aprender-present-core", version = "0.71.0", package = "aprender-present-core" } -presentar-terminal = { path = "crates/aprender-present-terminal", version = "0.71.0", package = "aprender-present-terminal" } -presentar-widgets = { path = "crates/aprender-present-widgets", version = "0.71.0", package = "aprender-present-widgets" } -presentar-layout = { path = "crates/aprender-present-layout", version = "0.71.0", package = "aprender-present-layout" } -presentar-yaml = { path = "crates/aprender-present-yaml", version = "0.71.0", package = "aprender-present-yaml" } -presentar-test = { path = "crates/aprender-present-test", version = "0.71.0", package = "aprender-present-test" } -presentar-test-macros = { path = "crates/aprender-present-test-macros", version = "0.71.0", package = "aprender-present-test-macros" } -jugar-probar-derive = { path = "crates/aprender-test-derive", version = "0.71.0", package = "aprender-test-derive" } -batuta-common = { path = "crates/aprender-common", version = "0.71.0", package = "aprender-common" } +trueno-zram-core = { path = "crates/aprender-zram-core", version = "0.71.1", package = "aprender-zram-core" } +trueno-zram-adaptive = { path = "crates/aprender-zram-adaptive", version = "0.71.1", package = "aprender-zram-adaptive" } +trueno = { path = "crates/aprender-compute", version = "0.71.1", package = "aprender-compute" } +presentar-core = { path = "crates/aprender-present-core", version = "0.71.1", package = "aprender-present-core" } +presentar-terminal = { path = "crates/aprender-present-terminal", version = "0.71.1", package = "aprender-present-terminal" } +presentar-widgets = { path = "crates/aprender-present-widgets", version = "0.71.1", package = "aprender-present-widgets" } +presentar-layout = { path = "crates/aprender-present-layout", version = "0.71.1", package = "aprender-present-layout" } +presentar-yaml = { path = "crates/aprender-present-yaml", version = "0.71.1", package = "aprender-present-yaml" } +presentar-test = { path = "crates/aprender-present-test", version = "0.71.1", package = "aprender-present-test" } +presentar-test-macros = { path = "crates/aprender-present-test-macros", version = "0.71.1", package = "aprender-present-test-macros" } +jugar-probar-derive = { path = "crates/aprender-test-derive", version = "0.71.1", package = "aprender-test-derive" } +batuta-common = { path = "crates/aprender-common", version = "0.71.1", package = "aprender-common" } # APR-MONO sibling aliases — MUST use workspace refs, NEVER crates.io versions. # Hard-pinning these to crates.io creates a diamond: sub-crates pull older # aprender versions → multiple `aprender` crates in Cargo.lock = MUDA. -renacer = { path = "crates/aprender-profile", version = "0.71.0", package = "aprender-profile" } -entrenar-lora = { path = "crates/aprender-train-lora", version = "0.71.0", package = "aprender-train-lora" } -batuta = { path = "crates/aprender-orchestrate", version = "0.71.0", package = "aprender-orchestrate" } -trueno-graph = { path = "crates/aprender-graph", version = "0.71.0", package = "aprender-graph" } +renacer = { path = "crates/aprender-profile", version = "0.71.1", package = "aprender-profile" } +entrenar-lora = { path = "crates/aprender-train-lora", version = "0.71.1", package = "aprender-train-lora" } +batuta = { path = "crates/aprender-orchestrate", version = "0.71.1", package = "aprender-orchestrate" } +trueno-graph = { path = "crates/aprender-graph", version = "0.71.1", package = "aprender-graph" } [workspace.lints.rust] # Safety @@ -593,7 +593,7 @@ semicolon_if_nothing_returned = "allow" # Style preference [package] name = "aprender" -version = "0.71.0" +version = "0.71.1" edition = "2021" rust-version = "1.91" authors = ["Noah Gift "] @@ -757,9 +757,9 @@ required-features = ["cli"] # (e.g. apr-cookbook, downstream apps) don't pull apr-cli's heavy transitive # dep graph (renacer/profile/inference/zram/visualization). `cargo install # aprender` still works because `cli` is in default features. See GH-1599. -apr-cli = { path = "crates/apr-cli", version = "0.71.0", default-features = true, optional = true } +apr-cli = { path = "crates/apr-cli", version = "0.71.1", default-features = true, optional = true } # Core ML library (lib name = "aprender") -aprender_ml = { path = "crates/aprender-core", version = "0.71.0", package = "aprender-core" } +aprender_ml = { path = "crates/aprender-core", version = "0.71.1", package = "aprender-core" } [features] default = ["cli"] diff --git a/crates/apr-cli/Cargo.toml b/crates/apr-cli/Cargo.toml index 68b0ea9713..04de56d857 100644 --- a/crates/apr-cli/Cargo.toml +++ b/crates/apr-cli/Cargo.toml @@ -130,12 +130,12 @@ libc = "0.2" # performed. A surface and its capability must not be able to drift. apr-test = { workspace = true } # Core aprender library -aprender = { path = "../aprender-core", version = "0.71.0", package = "aprender-core", default-features = true, features = ["format-compression"] } +aprender = { path = "../aprender-core", version = "0.71.1", package = "aprender-core", default-features = true, features = ["format-compression"] } # Provable-contracts engine — `apr beat-run` parses + evaluates BeatBenchmark contracts (PMAT-741) aprender-contracts = { workspace = true } # MCP (Model Context Protocol) server — `apr mcp` subcommand -aprender-mcp = { path = "../aprender-mcp", version = "0.71.0", features = ["apr-tools"] } # apr mcp serves apr's own tools (PMAT-3954) +aprender-mcp = { path = "../aprender-mcp", version = "0.71.1", features = ["apr-tools"] } # apr mcp serves apr's own tools (PMAT-3954) async-stream = "0.3" provable-contracts-macros = { workspace = true } @@ -146,7 +146,7 @@ batuta-common = { workspace = true, features = ["cli-roles"] } # Non-optional since claude-code-parity-apr M150 (outcome-parity Phase 3 # requires apr code in default builds). GH-344: path dep via # .cargo/config.toml.dev-overrides; CI uses checkout+symlink. -batuta = { version = "0.71.0", path = "../aprender-orchestrate", package = "aprender-orchestrate", default-features = false, features = ["agents", "agents-inference", "agents-mcp", "rag"] } +batuta = { version = "0.71.1", path = "../aprender-orchestrate", package = "aprender-orchestrate", default-features = false, features = ["agents", "agents-inference", "agents-mcp", "rag"] } # Inference engine (workspace path dep — enables cuda/gpu feature forwarding) # GH-573: shim (0.9.1) didn't forward cuda feature → 0.7 tok/s instead of 273 tok/s @@ -242,7 +242,7 @@ pacha = { workspace = true } # published-crate dependency cycle the APR-MONO consolidation removed, and # (b) was never actually linked, so every `trueno_explain::` path in # commands/ptx_explain.rs failed to resolve. -trueno-explain = { path = "../aprender-explain", version = "0.71.0", package = "aprender-explain", optional = true } +trueno-explain = { path = "../aprender-explain", version = "0.71.1", package = "aprender-explain", optional = true } # ML tuning: LoRA/QLoRA configuration and memory planning (GH-176, PMAT-184) entrenar-lora = { workspace = true, optional = true } diff --git a/crates/aprender-bench-compute/Cargo.toml b/crates/aprender-bench-compute/Cargo.toml index 2670a87bee..97291587b3 100644 --- a/crates/aprender-bench-compute/Cargo.toml +++ b/crates/aprender-bench-compute/Cargo.toml @@ -11,7 +11,7 @@ disallowed_methods = "allow" [dependencies] # The implementation under test -aprender = { path = "../aprender-core", version = "0.71.0", package = "aprender-core", features = ["format-quantize"] } +aprender = { path = "../aprender-core", version = "0.71.1", package = "aprender-core", features = ["format-quantize"] } # Reference implementation for A/B comparison # Pure Rust (no BLAS) to keep comparison fair — both are pure Rust SIMD diff --git a/crates/aprender-bench-tokenizer/Cargo.toml b/crates/aprender-bench-tokenizer/Cargo.toml index 6d409c4f83..2b6d18758b 100644 --- a/crates/aprender-bench-tokenizer/Cargo.toml +++ b/crates/aprender-bench-tokenizer/Cargo.toml @@ -11,7 +11,7 @@ disallowed_methods = "allow" [dependencies] # The implementation under test -aprender = { path = "../aprender-core", version = "0.71.0", package = "aprender-core" } +aprender = { path = "../aprender-core", version = "0.71.1", package = "aprender-core" } # HuggingFace reference for A/B comparison # Regular dep (not dev-dep) because [[bench]] harness=false binaries can't see dev-deps diff --git a/crates/aprender-cbtop/Cargo.toml b/crates/aprender-cbtop/Cargo.toml index 3430db0f85..8cc9e39a01 100644 --- a/crates/aprender-cbtop/Cargo.toml +++ b/crates/aprender-cbtop/Cargo.toml @@ -32,7 +32,7 @@ presentar-terminal = { workspace = true } # Compute backends trueno = { workspace = true } -trueno-gpu = { version = "0.71.0", path = "../aprender-gpu", package = "aprender-gpu", optional = true } +trueno-gpu = { version = "0.71.1", path = "../aprender-gpu", package = "aprender-gpu", optional = true } # Terminal handling crossterm = "0.28" diff --git a/crates/aprender-cgp/Cargo.toml b/crates/aprender-cgp/Cargo.toml index 1d69773d34..5809939dc2 100644 --- a/crates/aprender-cgp/Cargo.toml +++ b/crates/aprender-cgp/Cargo.toml @@ -36,7 +36,7 @@ num_cpus = "1.17" comfy-table = "7" colored = "3" # GPU profiling: direct cuBLAS measurement (OWN THE STACK) -trueno-gpu = { path = "../aprender-gpu", version = "0.71.0", features = ["cuda"], optional = true, package = "aprender-gpu" } +trueno-gpu = { path = "../aprender-gpu", version = "0.71.1", features = ["cuda"], optional = true, package = "aprender-gpu" } [features] default = [] diff --git a/crates/aprender-compute/Cargo.toml b/crates/aprender-compute/Cargo.toml index f4bee6047f..3e88c16e25 100644 --- a/crates/aprender-compute/Cargo.toml +++ b/crates/aprender-compute/Cargo.toml @@ -41,11 +41,11 @@ futures-intrusive = { version = "0.5", optional = true } wasm-bindgen-futures = { version = "0.4", optional = true } wasm-bindgen = { version = "0.2", optional = true } # Native CUDA monitoring via trueno-gpu (TRUENO-SPEC-010) -trueno-gpu = { version = "0.71.0", path = "../aprender-gpu", optional = true, package = "aprender-gpu" } +trueno-gpu = { version = "0.71.1", path = "../aprender-gpu", optional = true, package = "aprender-gpu" } # PMAT-336: Provable contracts compile-time enforcement provable-contracts-macros = { workspace = true } # P1a: Sovereign GEMM microkernel codegen (compile-time shape specialization) -trueno-gemm-codegen = { version = "0.71.0", path = "../aprender-gemm-codegen", package = "aprender-gemm-codegen" } +trueno-gemm-codegen = { version = "0.71.1", path = "../aprender-gemm-codegen", package = "aprender-gemm-codegen" } # Tracing/profiling support (renacer integration) tracing = { version = "0.1", optional = true } tracing-subscriber = { version = "0.3", features = ["env-filter", "fmt"], optional = true } diff --git a/crates/aprender-contracts-cli/Cargo.toml b/crates/aprender-contracts-cli/Cargo.toml index ddf13b7a19..0df4e51a6e 100644 --- a/crates/aprender-contracts-cli/Cargo.toml +++ b/crates/aprender-contracts-cli/Cargo.toml @@ -18,7 +18,7 @@ name = "pv-sat" path = "src/bin/pv-sat/main.rs" [dependencies] -aprender-contracts = { path = "../aprender-contracts", version = "0.71.0" } +aprender-contracts = { path = "../aprender-contracts", version = "0.71.1" } # #3745 S1: `--model` is declared through the shared role type `ModelPath`, so # `apr surface` reads `apr pv verify-structure --model` as a model argument. batuta-common = { workspace = true, features = ["cli-roles"] } diff --git a/crates/aprender-contracts/Cargo.toml b/crates/aprender-contracts/Cargo.toml index 2f77b55308..43ee4f3b9c 100644 --- a/crates/aprender-contracts/Cargo.toml +++ b/crates/aprender-contracts/Cargo.toml @@ -26,7 +26,7 @@ kani = [] name = "provable_contracts" [dependencies] -provable-contracts-macros = { path = "../aprender-contracts-macros", version = "0.71.0", package = "aprender-contracts-macros" } +provable-contracts-macros = { path = "../aprender-contracts-macros", version = "0.71.1", package = "aprender-contracts-macros" } regex = "1" sha2 = { workspace = true } blake3 = "1" diff --git a/crates/aprender-cuda-edge/Cargo.toml b/crates/aprender-cuda-edge/Cargo.toml index c0ce852965..a060ec42ee 100644 --- a/crates/aprender-cuda-edge/Cargo.toml +++ b/crates/aprender-cuda-edge/Cargo.toml @@ -15,7 +15,7 @@ name = "trueno_cuda_edge" [dependencies] thiserror = "2.0" serde = { version = "1.0", features = ["derive"] } -trueno-gpu = { version = "0.71.0", path = "../aprender-gpu", package = "aprender-gpu", optional = true } +trueno-gpu = { version = "0.71.1", path = "../aprender-gpu", package = "aprender-gpu", optional = true } [dev-dependencies] proptest = "1.6" diff --git a/crates/aprender-explain/Cargo.toml b/crates/aprender-explain/Cargo.toml index e43ee5440f..f5893bed1d 100644 --- a/crates/aprender-explain/Cargo.toml +++ b/crates/aprender-explain/Cargo.toml @@ -25,7 +25,7 @@ presentar-terminal = { workspace = true } crossterm = "0.28" # Internal dependency for PTX generation -trueno-gpu = { version = "0.71.0", path = "../aprender-gpu", package = "aprender-gpu", features = ["cuda"] } +trueno-gpu = { version = "0.71.1", path = "../aprender-gpu", package = "aprender-gpu", features = ["cuda"] } [build-dependencies] aprender-build-sha = { workspace = true } # #4219: APR_GIT_SHA for --version diff --git a/crates/aprender-graph/Cargo.toml b/crates/aprender-graph/Cargo.toml index 0717b4e018..2d9671b1c8 100644 --- a/crates/aprender-graph/Cargo.toml +++ b/crates/aprender-graph/Cargo.toml @@ -28,7 +28,7 @@ anyhow = "1" thiserror = "2" # Phase 2+ dependencies -aprender = { path = "../aprender-core", version = "0.71.0", package = "aprender-core" } # Sparse matrix ops, PageRank +aprender = { path = "../aprender-core", version = "0.71.1", package = "aprender-core" } # Sparse matrix ops, PageRank trueno = { workspace = true } # Phase 3: GPU acceleration (optional - requires wgpu-capable hardware) diff --git a/crates/aprender-monte-carlo/Cargo.toml b/crates/aprender-monte-carlo/Cargo.toml index e109f5c607..3bf36078e1 100644 --- a/crates/aprender-monte-carlo/Cargo.toml +++ b/crates/aprender-monte-carlo/Cargo.toml @@ -18,7 +18,7 @@ path = "src/lib.rs" name = "aprender-monte-carlo" [dependencies] -aprender = { path = "../../", version = "0.71.0" } +aprender = { path = "../../", version = "0.71.1" } clap = { version = "4", features = ["derive"] } rand = "0.9" rand_chacha = "0.9" diff --git a/crates/aprender-orchestrate/Cargo.toml b/crates/aprender-orchestrate/Cargo.toml index a242eaa297..8aa0013e5e 100644 --- a/crates/aprender-orchestrate/Cargo.toml +++ b/crates/aprender-orchestrate/Cargo.toml @@ -132,7 +132,7 @@ pacha = { workspace = true, optional = true } realizar = { workspace = true, optional = true } # ML algorithms and training (native-only) -aprender = { path = "../aprender-core", version = "0.71.0", package = "aprender-core", optional = true } +aprender = { path = "../aprender-core", version = "0.71.1", package = "aprender-core", optional = true } entrenar = { workspace = true, optional = true } alimentar = { workspace = true, optional = true } diff --git a/crates/aprender-profile/Cargo.toml b/crates/aprender-profile/Cargo.toml index b9fa405068..e24ce798b6 100644 --- a/crates/aprender-profile/Cargo.toml +++ b/crates/aprender-profile/Cargo.toml @@ -74,7 +74,7 @@ trueno = { workspace = true } # Machine learning for anomaly detection (Sprint 23) # GH-581: Use path dep to avoid trueno version skew with local workspace -aprender = { path = "../aprender-core", version = "0.71.0", package = "aprender-core" } +aprender = { path = "../aprender-core", version = "0.71.1", package = "aprender-core" } # Lock-free data structures for ring buffer (Sprint 40) crossbeam = { version = "0.8", default-features = false, features = ["crossbeam-channel", "crossbeam-queue", "std"] } @@ -112,7 +112,7 @@ crossterm = "0.28" # SIMD-accelerated visualization primitives (sister project) # NOTE: `monitor` feature removed upstream (#1979); this crate uses only viz primitives. -trueno-viz = { version = "0.71.0", path = "../aprender-viz", package = "aprender-viz", default-features = false } +trueno-viz = { version = "0.71.1", path = "../aprender-viz", package = "aprender-viz", default-features = false } # OpenTelemetry for distributed tracing (Sprint 30) opentelemetry = { version = "0.31.0", optional = true } diff --git a/crates/aprender-qa-cli/Cargo.toml b/crates/aprender-qa-cli/Cargo.toml index 3040345664..e20e6f1814 100644 --- a/crates/aprender-qa-cli/Cargo.toml +++ b/crates/aprender-qa-cli/Cargo.toml @@ -23,10 +23,10 @@ serde_json = { workspace = true } serde_yaml = { workspace = true } clap = { workspace = true } chrono = { workspace = true } -aprender-qa-gen = { version = "0.71.0", path = "../aprender-qa-gen" } -aprender-qa-runner = { version = "0.71.0", path = "../aprender-qa-runner" } -aprender-qa-report = { version = "0.71.0", path = "../aprender-qa-report" } -aprender-qa-certify = { version = "0.71.0", path = "../aprender-qa-certify" } +aprender-qa-gen = { version = "0.71.1", path = "../aprender-qa-gen" } +aprender-qa-runner = { version = "0.71.1", path = "../aprender-qa-runner" } +aprender-qa-report = { version = "0.71.1", path = "../aprender-qa-report" } +aprender-qa-certify = { version = "0.71.1", path = "../aprender-qa-certify" } ctrlc = "3.4" safetensors = { workspace = true } colored = "2.2" diff --git a/crates/aprender-qa-report/Cargo.toml b/crates/aprender-qa-report/Cargo.toml index d9e3f0e482..6dbdcb688a 100644 --- a/crates/aprender-qa-report/Cargo.toml +++ b/crates/aprender-qa-report/Cargo.toml @@ -18,8 +18,8 @@ anyhow = { workspace = true } chrono = { workspace = true } colored = "2.2" csv = "1.3" -aprender-qa-gen = { version = "0.71.0", path = "../aprender-qa-gen" } -aprender-qa-runner = { version = "0.71.0", path = "../aprender-qa-runner" } +aprender-qa-gen = { version = "0.71.1", path = "../aprender-qa-gen" } +aprender-qa-runner = { version = "0.71.1", path = "../aprender-qa-runner" } [dev-dependencies] proptest = { workspace = true } diff --git a/crates/aprender-qa-runner/Cargo.toml b/crates/aprender-qa-runner/Cargo.toml index ebe63bf531..3c3bf80187 100644 --- a/crates/aprender-qa-runner/Cargo.toml +++ b/crates/aprender-qa-runner/Cargo.toml @@ -22,14 +22,14 @@ anyhow = { workspace = true } tokio = { workspace = true } chrono = { workspace = true, features = ["serde"] } tracing = { workspace = true } -aprender-qa-gen = { version = "0.71.0", path = "../aprender-qa-gen" } +aprender-qa-gen = { version = "0.71.1", path = "../aprender-qa-gen" } hostname = "0.4" rayon = "1.10" sha2 = "0.10" regex = { workspace = true } num_cpus = "1.16" safetensors = { workspace = true } -jugar-probar = { version = "0.71.0", path = "../aprender-test-lib", package = "aprender-test-lib", features = ["llm-types"] } +jugar-probar = { version = "0.71.1", path = "../aprender-test-lib", package = "aprender-test-lib", features = ["llm-types"] } # Test fixtures dependencies (optional) tempfile = { version = "3", optional = true } half = { version = "2.4", optional = true } diff --git a/crates/aprender-rag-cli/Cargo.toml b/crates/aprender-rag-cli/Cargo.toml index fc81f0e7a5..2c87459a5b 100644 --- a/crates/aprender-rag-cli/Cargo.toml +++ b/crates/aprender-rag-cli/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "aprender-rag-cli" -version = "0.71.0" +version = "0.71.1" edition = "2021" authors = ["Pragmatic AI Labs"] description = "CLI for Trueno-RAG pipeline" diff --git a/crates/aprender-rag/Cargo.toml b/crates/aprender-rag/Cargo.toml index f7d2e537c9..496c8fa41d 100644 --- a/crates/aprender-rag/Cargo.toml +++ b/crates/aprender-rag/Cargo.toml @@ -63,7 +63,7 @@ fastembed = { version = "5", optional = true } # NVIDIA Embed Nemotron 8B via realizar GGUF infrastructure (GH-3) # Large model (~4-32GB depending on quantization), requires GGUF file -realizar = { version = "0.71.0", path = "../aprender-serve", package = "aprender-serve", default-features = false, optional = true } +realizar = { version = "0.71.1", path = "../aprender-serve", package = "aprender-serve", default-features = false, optional = true } # Eval: LLM-as-judge via Anthropic API (PMAT-015) diff --git a/crates/aprender-registry/Cargo.toml b/crates/aprender-registry/Cargo.toml index 7e11f8d431..edcd2eef20 100644 --- a/crates/aprender-registry/Cargo.toml +++ b/crates/aprender-registry/Cargo.toml @@ -42,7 +42,7 @@ zstd = { version = "0.13", optional = true } clap = { version = "4", features = ["derive"], optional = true } # Ecosystem integration -aprender = { path = "../aprender-core", version = "0.71.0", package = "aprender-core", optional = true } +aprender = { path = "../aprender-core", version = "0.71.1", package = "aprender-core", optional = true } alimentar = { workspace = true, optional = true } # HTTP client (optional) diff --git a/crates/aprender-serve/Cargo.toml b/crates/aprender-serve/Cargo.toml index 6d80fe7e9e..03c9ea457f 100644 --- a/crates/aprender-serve/Cargo.toml +++ b/crates/aprender-serve/Cargo.toml @@ -51,7 +51,7 @@ trueno = { workspace = true, features = ["gpu"] } unicode-properties = { workspace = true } # CUDA PTX generation and runtime for NVIDIA GPUs (optional, pure Rust, no LLVM/nvcc) # v0.4.12: BatchedQ6K, MultiWarp, RopeNeox, FlashDecoding kernels -trueno-gpu = { version = "0.71.0", path = "../aprender-gpu", package = "aprender-gpu", optional = true, features = ["cuda"] } # APR-MONO single-source-of-truth +trueno-gpu = { version = "0.71.1", path = "../aprender-gpu", package = "aprender-gpu", optional = true, features = ["cuda"] } # APR-MONO single-source-of-truth # KV cache storage (for attention caching) trueno-db = { workspace = true, optional = true } # K-quantization formats (Q4_K, Q5_K, Q6_K) - Toyota Way: ONE source of truth @@ -65,7 +65,7 @@ renacer-core = { workspace = true } # ML algorithms and .apr model format (optional for aprender model serving) # v0.24.1: chat templates, GGUF tokenizer preservation (local may be newer) -aprender = { path = "../aprender-core", version = "0.71.0", package = "aprender-core", optional = true } +aprender = { path = "../aprender-core", version = "0.71.1", package = "aprender-core", optional = true } # Data loading library (optional for data pipeline examples) alimentar = { workspace = true, optional = true } diff --git a/crates/aprender-shell/Cargo.toml b/crates/aprender-shell/Cargo.toml index e89a05c27e..0a07a7d88d 100644 --- a/crates/aprender-shell/Cargo.toml +++ b/crates/aprender-shell/Cargo.toml @@ -24,7 +24,7 @@ harness = false [dependencies] # Use path for local dev, crates.io for release -aprender = { path = "../aprender-core", version = "0.71.0", package = "aprender-core", features = ["format-encryption", "format-compression", "format-homomorphic"] } +aprender = { path = "../aprender-core", version = "0.71.1", package = "aprender-core", features = ["format-encryption", "format-compression", "format-homomorphic"] } clap = { version = "4", features = ["derive"] } dirs = "5" serde = { version = "1", features = ["derive"] } diff --git a/crates/aprender-simulate/Cargo.toml b/crates/aprender-simulate/Cargo.toml index f7dd0f6c5b..e1f7e574c8 100644 --- a/crates/aprender-simulate/Cargo.toml +++ b/crates/aprender-simulate/Cargo.toml @@ -59,7 +59,7 @@ crossterm = { version = "0.28", optional = true } # ComputeBlocks from presentar-terminal (SIMD-optimized TUI widgets) # Path deps to avoid jugar-probar prod dep chain (CB-081) presentar-terminal = { workspace = true, optional = true } -presentar-core = { version = "0.71.0", path = "../aprender-present-core", package = "aprender-present-core", default-features = false, optional = true } +presentar-core = { version = "0.71.1", path = "../aprender-present-core", package = "aprender-present-core", default-features = false, optional = true } # WASM (optional) wasm-bindgen = { version = "0.2", optional = true } diff --git a/crates/aprender-test-cli/Cargo.toml b/crates/aprender-test-cli/Cargo.toml index 5c1bf94b47..3469095232 100644 --- a/crates/aprender-test-cli/Cargo.toml +++ b/crates/aprender-test-cli/Cargo.toml @@ -35,7 +35,7 @@ path = "src/main.rs" [dependencies] # Core library (was: jugar-probar) — dep key preserves Rust identifier -jugar-probar = { path = "../aprender-test-lib", version = "0.71.0", package = "aprender-test-lib" } +jugar-probar = { path = "../aprender-test-lib", version = "0.71.1", package = "aprender-test-lib" } # CLI framework clap = { workspace = true } diff --git a/crates/aprender-train-bench/Cargo.toml b/crates/aprender-train-bench/Cargo.toml index 1513b6f14b..2c86e6cb0a 100644 --- a/crates/aprender-train-bench/Cargo.toml +++ b/crates/aprender-train-bench/Cargo.toml @@ -20,7 +20,7 @@ workspace = true [dependencies] entrenar = { workspace = true } -entrenar-common = { version = "0.71.0", path = "../aprender-train-common", package = "aprender-train-common" } +entrenar-common = { version = "0.71.1", path = "../aprender-train-common", package = "aprender-train-common" } clap = { version = "4.5", features = ["derive"] } serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" diff --git a/crates/aprender-train-distill/Cargo.toml b/crates/aprender-train-distill/Cargo.toml index a166a97bf5..7be23d3726 100644 --- a/crates/aprender-train-distill/Cargo.toml +++ b/crates/aprender-train-distill/Cargo.toml @@ -33,7 +33,7 @@ shard-batch-source = [] [dependencies] entrenar = { workspace = true } -entrenar-common = { version = "0.71.0", path = "../aprender-train-common", package = "aprender-train-common" } +entrenar-common = { version = "0.71.1", path = "../aprender-train-common", package = "aprender-train-common" } clap = { version = "4.5", features = ["derive"] } serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" diff --git a/crates/aprender-train-inspect/Cargo.toml b/crates/aprender-train-inspect/Cargo.toml index 66144a3a45..5541abbd81 100644 --- a/crates/aprender-train-inspect/Cargo.toml +++ b/crates/aprender-train-inspect/Cargo.toml @@ -20,7 +20,7 @@ workspace = true [dependencies] entrenar = { workspace = true } -entrenar-common = { version = "0.71.0", path = "../aprender-train-common", package = "aprender-train-common" } +entrenar-common = { version = "0.71.1", path = "../aprender-train-common", package = "aprender-train-common" } clap = { version = "4.5", features = ["derive"] } serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" diff --git a/crates/aprender-train-lora/Cargo.toml b/crates/aprender-train-lora/Cargo.toml index ab58fd6dcb..947c969874 100644 --- a/crates/aprender-train-lora/Cargo.toml +++ b/crates/aprender-train-lora/Cargo.toml @@ -20,7 +20,7 @@ workspace = true [dependencies] entrenar = { workspace = true } -entrenar-common = { version = "0.71.0", path = "../aprender-train-common", package = "aprender-train-common" } +entrenar-common = { version = "0.71.1", path = "../aprender-train-common", package = "aprender-train-common" } clap = { version = "4.5", features = ["derive"] } serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" diff --git a/crates/aprender-train-shell/Cargo.toml b/crates/aprender-train-shell/Cargo.toml index 4f617a064d..eab5d26876 100644 --- a/crates/aprender-train-shell/Cargo.toml +++ b/crates/aprender-train-shell/Cargo.toml @@ -20,7 +20,7 @@ workspace = true [dependencies] entrenar = { workspace = true } -entrenar-common = { version = "0.71.0", path = "../aprender-train-common", package = "aprender-train-common" } +entrenar-common = { version = "0.71.1", path = "../aprender-train-common", package = "aprender-train-common" } clap = { version = "4.5", features = ["derive"] } rustyline = "14" serde = { version = "1.0", features = ["derive"] } diff --git a/crates/aprender-train/Cargo.toml b/crates/aprender-train/Cargo.toml index 18a6ae4d78..5b41707b0d 100644 --- a/crates/aprender-train/Cargo.toml +++ b/crates/aprender-train/Cargo.toml @@ -80,9 +80,9 @@ provable-contracts-macros = { workspace = true } # Core PAIML stack ndarray = "0.16" trueno = { workspace = true, features = ["parallel"] } -trueno-gpu = { version = "0.71.0", path = "../aprender-gpu", package = "aprender-gpu", optional = true } # CUDA compute (APR-MONO single-source-of-truth) +trueno-gpu = { version = "0.71.1", path = "../aprender-gpu", package = "aprender-gpu", optional = true } # CUDA compute (APR-MONO single-source-of-truth) realizar = { workspace = true, optional = true } -aprender = { path = "../aprender-core", version = "0.71.0", package = "aprender-core" } # ML interpret module + .apr format + tokenizers +aprender = { path = "../aprender-core", version = "0.71.1", package = "aprender-core" } # ML interpret module + .apr format + tokenizers trueno-viz = { workspace = true, features = ["terminal"] } batuta-common = { workspace = true, optional = true } # WithDimensions trait for `viz` feature (APR-MONO §S #1978) presentar-terminal = { workspace = true, optional = true } diff --git a/crates/aprender-tsp/Cargo.toml b/crates/aprender-tsp/Cargo.toml index daad82e1a0..1bc905acdf 100644 --- a/crates/aprender-tsp/Cargo.toml +++ b/crates/aprender-tsp/Cargo.toml @@ -18,7 +18,7 @@ path = "src/lib.rs" name = "aprender-tsp" [dependencies] -aprender = { path = "../../", version = "0.71.0" } +aprender = { path = "../../", version = "0.71.1" } clap = { version = "4", features = ["derive"] } rand = "0.9" crc32fast = "1.4" diff --git a/crates/aprender-verify-ml/Cargo.toml b/crates/aprender-verify-ml/Cargo.toml index d5d62258ef..964e1d7d6e 100644 --- a/crates/aprender-verify-ml/Cargo.toml +++ b/crates/aprender-verify-ml/Cargo.toml @@ -56,7 +56,7 @@ uuid = { version = "1.11", features = ["v4", "serde"] } trueno = { workspace = true } # Machine learning for bug prediction (optional) -aprender = { path = "../aprender-core", version = "0.71.0", package = "aprender-core", optional = true } +aprender = { path = "../aprender-core", version = "0.71.1", package = "aprender-core", optional = true } # LLM fine-tuning integration (optional) entrenar = { workspace = true, optional = true } diff --git a/crates/aprender-viz/Cargo.toml b/crates/aprender-viz/Cargo.toml index 8aba030023..4e751aaeff 100644 --- a/crates/aprender-viz/Cargo.toml +++ b/crates/aprender-viz/Cargo.toml @@ -14,7 +14,7 @@ categories = ["visualization", "graphics", "science", "wasm"] [dependencies] # Core SIMD/GPU acceleration -trueno = { version = "0.71.0", path = "../aprender-compute", package = "aprender-compute", default-features = false } +trueno = { version = "0.71.1", path = "../aprender-compute", package = "aprender-compute", default-features = false } # Shared Batuta stack utilities (display traits, formatting) batuta-common = { workspace = true } @@ -52,10 +52,10 @@ kurbo = { version = "0.13", optional = true } resvg = { version = "0.45", optional = true, default-features = false } # Optional: ML integration -aprender = { path = "../aprender-core", version = "0.71.0", package = "aprender-core", optional = true } +aprender = { path = "../aprender-core", version = "0.71.1", package = "aprender-core", optional = true } # Optional: Graph integration -trueno-graph = { version = "0.71.0", path = "../aprender-graph", package = "aprender-graph", default-features = false, optional = true } +trueno-graph = { version = "0.71.1", path = "../aprender-graph", package = "aprender-graph", default-features = false, optional = true } # Optional: Database integration trueno-db = { workspace = true, optional = true } diff --git a/crates/facades/Cargo.lock b/crates/facades/Cargo.lock index 18a4b441c2..62a5907ecd 100644 --- a/crates/facades/Cargo.lock +++ b/crates/facades/Cargo.lock @@ -13,7 +13,7 @@ dependencies = [ [[package]] name = "aprender-contracts" -version = "0.71.0" +version = "0.71.1" dependencies = [ "aprender-contracts-macros", "blake3", @@ -29,7 +29,7 @@ dependencies = [ [[package]] name = "aprender-contracts-macros" -version = "0.71.0" +version = "0.71.1" dependencies = [ "proc-macro2", "quote", diff --git a/crates/facades/provable-contracts-macros/Cargo.toml b/crates/facades/provable-contracts-macros/Cargo.toml index aa240a83a2..d63ce1713c 100644 --- a/crates/facades/provable-contracts-macros/Cargo.toml +++ b/crates/facades/provable-contracts-macros/Cargo.toml @@ -30,7 +30,7 @@ path = "src/lib.rs" # facade resolves. `scripts/check_facade_compat.sh` fails if this version ever # drifts from the aprender workspace version — these manifests are excluded # from the root workspace, so `cargo set-version` does NOT reach them. -upstream = { path = "../../aprender-contracts-macros", version = "0.71.0", package = "aprender-contracts-macros" } +upstream = { path = "../../aprender-contracts-macros", version = "0.71.1", package = "aprender-contracts-macros" } [features] default = [] diff --git a/crates/facades/provable-contracts/Cargo.toml b/crates/facades/provable-contracts/Cargo.toml index e5f3df59cb..15926a6bec 100644 --- a/crates/facades/provable-contracts/Cargo.toml +++ b/crates/facades/provable-contracts/Cargo.toml @@ -31,7 +31,7 @@ path = "src/lib.rs" # facade resolves. `scripts/check_facade_compat.sh` fails if this version ever # drifts from the aprender workspace version — these manifests are excluded # from the root workspace, so `cargo set-version` does NOT reach them. -upstream = { path = "../../aprender-contracts", version = "0.71.0", package = "aprender-contracts" } +upstream = { path = "../../aprender-contracts", version = "0.71.1", package = "aprender-contracts" } [dev-dependencies] # Used by three of the vendored 0.3.1 example programs. They were regular diff --git a/evidence/crux/0.71.1/prompt-certification-inventory.json b/evidence/crux/0.71.1/prompt-certification-inventory.json new file mode 100644 index 0000000000..93705629cf --- /dev/null +++ b/evidence/crux/0.71.1/prompt-certification-inventory.json @@ -0,0 +1,13 @@ +[ + {"model": "Qwen3.5-2B", + "source": {"repo": "Qwen/Qwen3.5-2B", "revision": "15852e8c16360a2fea060d615a32b45270f8a8fc"}, + "bf16_gguf": "dc11be2ca4519954a3e4b013ef59497b5522209987d67891851113eda89619f1", + "quants": {"Q4_K_M": "aaf42c8b7c3cab2bf3d69c355048d4a0ee9973d48f16c731c0520ee914699223"}, + "thinking": ["on", "off"]}, + {"model": "Qwen3.5-4B", + "source": {"repo": "Qwen/Qwen3.5-4B", "revision": "851bf6e806efd8d0a36b00ddf55e13ccb7b8cd0a"}, + "bf16_gguf": "9e6e2841a75f503ccb330831832fd7861266e187e0dbf149a954219ccb8c197a", + "quants": {"Q4_K_M": "00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4", + "UD-Q4_K_XL": "b252c5610a42ca82d20fe2a12813e9d069eed89292907e26c783eeb0bc961bc7"}, + "thinking": ["on", "off"]} +] diff --git a/evidence/crux/0.71.1/prompt-certification.json b/evidence/crux/0.71.1/prompt-certification.json new file mode 100644 index 0000000000..a948db9a92 --- /dev/null +++ b/evidence/crux/0.71.1/prompt-certification.json @@ -0,0 +1,8906 @@ +{ + "schema": "crux-prompt-certification/v1", + "prompts": "scripts/crux_inference_prompts.v2.json", + "prompts_sha256": "cea80df5e785213667e27697fe3d139b99dcba95cfc6e970d76ac005d50fc806", + "apr_commit": "0d607e065665f3a73bea093e40f9971a84398bc2", + "inventory_sha256": "b8e4c4626b20b7764041f5b5fe6d8a98166a952a64b3a844207b796b388778e0", + "manifests": { + "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl": "f56747953b33bb2e4adac4bac12be902224a202b2a07c35ede3e47121d178728", + "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl": "6a5657345a6a61ee95ca7a8d7237d7f9748ec8e6d652bdd95be042a586a4e45f" + }, + "admitted": { + "Qwen3.5-2B/Q4_K_M": [], + "Qwen3.5-4B/Q4_K_M": [ + "ctl-2plus2", + "ctl-code-add", + "fact-capital-japan", + "fact-capital-france", + "golden-paris", + "nonascii-water-kanji", + "arith-17x23", + "arith-precedence", + "arith-word-apples", + "hard-not-prime", + "hard-reverse-word", + "hard-count-letters", + "recall-arith-chain", + "code-count-vowels", + "long-primes-10", + "long-countdown-15" + ], + "Qwen3.5-4B/UD-Q4_K_XL": [ + "ctl-2plus2", + "ctl-code-add", + "fact-capital-japan", + "fact-capital-france", + "golden-paris", + "nonascii-water-kanji", + "arith-17x23", + "arith-precedence", + "arith-word-apples", + "hard-not-prime", + "hard-count-letters", + "recall-arith-chain", + "long-primes-10", + "long-countdown-15" + ] + }, + "admitted_by_sha": { + "aaf42c8b7c3cab2bf3d69c355048d4a0ee9973d48f16c731c0520ee914699223": [], + "00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4": [ + "ctl-2plus2", + "ctl-code-add", + "fact-capital-japan", + "fact-capital-france", + "golden-paris", + "nonascii-water-kanji", + "arith-17x23", + "arith-precedence", + "arith-word-apples", + "hard-not-prime", + "hard-reverse-word", + "hard-count-letters", + "recall-arith-chain", + "code-count-vowels", + "long-primes-10", + "long-countdown-15" + ], + "b252c5610a42ca82d20fe2a12813e9d069eed89292907e26c783eeb0bc961bc7": [ + "ctl-2plus2", + "ctl-code-add", + "fact-capital-japan", + "fact-capital-france", + "golden-paris", + "nonascii-water-kanji", + "arith-17x23", + "arith-precedence", + "arith-word-apples", + "hard-not-prime", + "hard-count-letters", + "recall-arith-chain", + "long-primes-10", + "long-countdown-15" + ] + }, + "admitted_by_sha_thinking": { + "aaf42c8b7c3cab2bf3d69c355048d4a0ee9973d48f16c731c0520ee914699223": { + "on": [], + "off": [ + "ctl-2plus2", + "ctl-recall-7", + "ctl-code-add", + "fact-capital-japan", + "fact-capital-france", + "golden-paris", + "arith-17x23", + "arith-precedence", + "arith-word-apples", + "hard-add-4digit", + "hard-not-prime", + "hard-count-letters", + "recall-cats", + "recall-name", + "recall-arith-chain", + "code-is-palindrome", + "code-fib", + "code-count-vowels", + "long-primes-10", + "long-countdown-15" + ] + }, + "00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4": { + "on": [ + "ctl-2plus2", + "ctl-code-add", + "fact-capital-japan", + "fact-capital-france", + "golden-paris", + "nonascii-water-kanji", + "arith-17x23", + "arith-precedence", + "arith-word-apples", + "hard-not-prime", + "hard-reverse-word", + "hard-count-letters", + "recall-arith-chain", + "code-count-vowels", + "long-primes-10", + "long-countdown-15" + ], + "off": [ + "ctl-2plus2", + "ctl-recall-7", + "ctl-code-add", + "fact-capital-japan", + "fact-capital-france", + "golden-paris", + "nonascii-water-kanji", + "arith-17x23", + "arith-precedence", + "arith-word-apples", + "hard-add-4digit", + "hard-not-prime", + "hard-reverse-word", + "hard-count-letters", + "recall-cats", + "recall-name", + "recall-arith-chain", + "code-is-palindrome", + "code-fib", + "code-count-vowels", + "long-primes-10", + "long-countdown-15" + ] + }, + "b252c5610a42ca82d20fe2a12813e9d069eed89292907e26c783eeb0bc961bc7": { + "on": [ + "ctl-2plus2", + "ctl-code-add", + "fact-capital-japan", + "fact-capital-france", + "golden-paris", + "nonascii-water-kanji", + "arith-17x23", + "arith-precedence", + "arith-word-apples", + "hard-not-prime", + "hard-count-letters", + "recall-arith-chain", + "long-primes-10", + "long-countdown-15" + ], + "off": [ + "ctl-2plus2", + "ctl-recall-7", + "ctl-code-add", + "fact-capital-japan", + "fact-capital-france", + "golden-paris", + "nonascii-water-kanji", + "arith-17x23", + "arith-precedence", + "arith-word-apples", + "hard-add-4digit", + "hard-not-prime", + "hard-reverse-word", + "hard-count-letters", + "recall-cats", + "recall-name", + "recall-arith-chain", + "code-is-palindrome", + "code-fib", + "code-count-vowels", + "long-primes-10", + "long-countdown-15" + ] + } + }, + "rejected": { + "Qwen3.5-2B/Q4_K_M": { + "ctl-2plus2": "ggml@bf16 llama.cpp serve run thinking=on on gx10: unclosed think (budget exhausted)", + "ctl-recall-7": "ggml@bf16 llama.cpp serve run thinking=on on gx10: unclosed think (budget exhausted)", + "ctl-code-add": "hf@bf16 thinking=on: no row", + "fact-capital-japan": "ggml@bf16 llama.cpp serve run thinking=on on gx10: unclosed think (budget exhausted)", + "fact-capital-france": "hf@bf16 thinking=on: no row", + "golden-paris": "ggml@bf16 thinking=on: no row", + "fact-water-formula": "ggml@bf16 llama.cpp serve run thinking=on on gx10: unclosed think (budget exhausted)", + "nonascii-water-kanji": "ggml@bf16 thinking=on: no row", + "arith-17x23": "ggml@bf16 llama.cpp serve run thinking=on on gx10: unclosed think (budget exhausted)", + "arith-precedence": "hf@bf16 thinking=on: no row", + "arith-word-apples": "hf@bf16 thinking=on: no row", + "hard-add-4digit": "hf@bf16 thinking=on: no row", + "hard-not-prime": "ggml@bf16 thinking=on: no row", + "hard-reverse-word": "ggml@bf16 thinking=on: no row", + "hard-count-letters": "ggml@bf16 thinking=on: no row", + "recall-cats": "ggml@bf16 thinking=on: no row", + "recall-name": "ggml@bf16 thinking=on: no row", + "recall-arith-chain": "ggml@bf16 thinking=on: no row", + "code-is-palindrome": "ggml@bf16 thinking=on: no row", + "code-fib": "ggml@bf16 thinking=on: no row", + "code-count-vowels": "ggml@bf16 thinking=on: no row", + "long-primes-10": "ggml@bf16 thinking=on: no row", + "long-countdown-15": "ggml@bf16 thinking=on: no row" + }, + "Qwen3.5-4B/Q4_K_M": { + "ctl-recall-7": "ggml@quant llama.cpp serve run thinking=on on gx10: unclosed think (budget exhausted)", + "fact-water-formula": "ggml@bf16 llama.cpp serve run thinking=on on gx10: unclosed think (budget exhausted)", + "hard-add-4digit": "ggml@quant llama.cpp serve run thinking=on on gx10: unclosed think (budget exhausted)", + "recall-cats": "ggml@bf16 llama.cpp serve run thinking=on on gx10: unclosed think (budget exhausted)", + "recall-name": "ggml@quant llama.cpp serve run thinking=on on gx10: unclosed think (budget exhausted)", + "code-is-palindrome": "vllm@bf16 vllm chat thinking=on on gx10: tests_failed: AssertionError", + "code-fib": "ggml@bf16 llama.cpp serve run thinking=on on gx10: unclosed think (budget exhausted)" + }, + "Qwen3.5-4B/UD-Q4_K_XL": { + "ctl-recall-7": "ggml@quant llama.cpp serve run thinking=on on gx10: unclosed think (budget exhausted)", + "fact-water-formula": "ggml@bf16 llama.cpp serve run thinking=on on gx10: unclosed think (budget exhausted)", + "hard-add-4digit": "ggml@quant llama.cpp serve run thinking=on on gx10: unclosed think (budget exhausted)", + "hard-reverse-word": "ggml@quant llama.cpp serve run thinking=on on gx10: mismatch", + "recall-cats": "ggml@bf16 llama.cpp serve run thinking=on on gx10: unclosed think (budget exhausted)", + "recall-name": "ggml@quant llama.cpp serve run thinking=on on gx10: unclosed think (budget exhausted)", + "code-is-palindrome": "vllm@bf16 vllm chat thinking=on on gx10: tests_failed: AssertionError", + "code-fib": "ggml@bf16 llama.cpp serve run thinking=on on gx10: unclosed think (budget exhausted)", + "code-count-vowels": "ggml@quant llama.cpp serve run thinking=on on gx10: unclosed think (budget exhausted)" + } + }, + "uncontrolled": [ + "Qwen3.5-2B/Q4_K_M" + ], + "uncontrolled_detail": [ + { + "model": "Qwen3.5-2B/Q4_K_M", + "sha256": "aaf42c8b7c3cab2bf3d69c355048d4a0ee9973d48f16c731c0520ee914699223", + "thinking": "on", + "verbs": [ + "chat", + "code", + "run", + "serve run", + "serve stream" + ] + } + ], + "think_closure": { + "Qwen3.5-2B/Q4_K_M|ctl-2plus2": { + "ggml@bf16:llama.cpp": "unclosed" + }, + "Qwen3.5-2B/Q4_K_M|ctl-recall-7": { + "ggml@bf16:llama.cpp": "unclosed" + }, + "Qwen3.5-2B/Q4_K_M|ctl-code-add": { + "ggml@bf16:llama.cpp": "closed" + }, + "Qwen3.5-2B/Q4_K_M|fact-capital-japan": { + "ggml@bf16:llama.cpp": "unclosed" + }, + "Qwen3.5-2B/Q4_K_M|fact-capital-france": { + "ggml@bf16:llama.cpp": "closed" + }, + "Qwen3.5-2B/Q4_K_M|fact-water-formula": { + "ggml@bf16:llama.cpp": "unclosed" + }, + "Qwen3.5-2B/Q4_K_M|arith-17x23": { + "ggml@bf16:llama.cpp": "unclosed" + }, + "Qwen3.5-2B/Q4_K_M|arith-precedence": { + "ggml@bf16:llama.cpp": "closed" + }, + "Qwen3.5-2B/Q4_K_M|arith-word-apples": { + "ggml@bf16:llama.cpp": "closed" + }, + "Qwen3.5-2B/Q4_K_M|hard-add-4digit": { + "ggml@bf16:llama.cpp": "closed" + }, + "Qwen3.5-4B/Q4_K_M|ctl-2plus2": { + "ggml@bf16:llama.cpp": "closed", + "hf@bf16:hf": "closed", + "vllm@bf16:vllm": "closed", + "ggml@quant:llama.cpp": "closed" + }, + "Qwen3.5-4B/Q4_K_M|ctl-recall-7": { + "ggml@bf16:llama.cpp": "closed", + "hf@bf16:hf": "closed", + "vllm@bf16:vllm": "closed", + "ggml@quant:llama.cpp": "unclosed" + }, + "Qwen3.5-4B/Q4_K_M|ctl-code-add": { + "ggml@bf16:llama.cpp": "closed", + "hf@bf16:hf": "closed", + "vllm@bf16:vllm": "closed", + "ggml@quant:llama.cpp": "closed" + }, + "Qwen3.5-4B/Q4_K_M|fact-capital-japan": { + "ggml@bf16:llama.cpp": "closed", + "hf@bf16:hf": "closed", + "vllm@bf16:vllm": "closed", + "ggml@quant:llama.cpp": "closed" + }, + "Qwen3.5-4B/Q4_K_M|fact-capital-france": { + "ggml@bf16:llama.cpp": "closed", + "hf@bf16:hf": "closed", + "vllm@bf16:vllm": "closed", + "ggml@quant:llama.cpp": "closed" + }, + "Qwen3.5-4B/Q4_K_M|golden-paris": { + "ggml@bf16:llama.cpp": "closed", + "hf@bf16:hf": "closed", + "vllm@bf16:vllm": "closed", + "ggml@quant:llama.cpp": "closed" + }, + "Qwen3.5-4B/Q4_K_M|fact-water-formula": { + "ggml@bf16:llama.cpp": "unclosed", + "hf@bf16:hf": "unclosed", + "vllm@bf16:vllm": "unclosed", + "ggml@quant:llama.cpp": "closed" + }, + "Qwen3.5-4B/Q4_K_M|nonascii-water-kanji": { + "ggml@bf16:llama.cpp": "closed", + "hf@bf16:hf": "closed", + "vllm@bf16:vllm": "closed", + "ggml@quant:llama.cpp": "closed" + }, + "Qwen3.5-4B/Q4_K_M|arith-17x23": { + "ggml@bf16:llama.cpp": "closed", + "hf@bf16:hf": "closed", + "vllm@bf16:vllm": "closed", + "ggml@quant:llama.cpp": "closed" + }, + "Qwen3.5-4B/Q4_K_M|arith-precedence": { + "ggml@bf16:llama.cpp": "closed", + "hf@bf16:hf": "closed", + "vllm@bf16:vllm": "closed", + "ggml@quant:llama.cpp": "closed" + }, + "Qwen3.5-4B/Q4_K_M|arith-word-apples": { + "ggml@bf16:llama.cpp": "closed", + "hf@bf16:hf": "closed", + "vllm@bf16:vllm": "closed", + "ggml@quant:llama.cpp": "closed" + }, + "Qwen3.5-4B/Q4_K_M|hard-add-4digit": { + "ggml@bf16:llama.cpp": "closed", + "hf@bf16:hf": "closed", + "vllm@bf16:vllm": "closed", + "ggml@quant:llama.cpp": "unclosed" + }, + "Qwen3.5-4B/Q4_K_M|hard-not-prime": { + "ggml@bf16:llama.cpp": "closed", + "hf@bf16:hf": "closed", + "vllm@bf16:vllm": "closed", + "ggml@quant:llama.cpp": "closed" + }, + "Qwen3.5-4B/Q4_K_M|hard-reverse-word": { + "ggml@bf16:llama.cpp": "closed", + "hf@bf16:hf": "closed", + "vllm@bf16:vllm": "closed", + "ggml@quant:llama.cpp": "closed" + }, + "Qwen3.5-4B/Q4_K_M|hard-count-letters": { + "ggml@bf16:llama.cpp": "closed", + "hf@bf16:hf": "closed", + "vllm@bf16:vllm": "closed", + "ggml@quant:llama.cpp": "closed" + }, + "Qwen3.5-4B/Q4_K_M|recall-cats": { + "ggml@bf16:llama.cpp": "unclosed", + "hf@bf16:hf": "closed", + "vllm@bf16:vllm": "closed", + "ggml@quant:llama.cpp": "unclosed" + }, + "Qwen3.5-4B/Q4_K_M|recall-name": { + "ggml@bf16:llama.cpp": "closed", + "hf@bf16:hf": "closed", + "vllm@bf16:vllm": "closed", + "ggml@quant:llama.cpp": "unclosed" + }, + "Qwen3.5-4B/Q4_K_M|recall-arith-chain": { + "ggml@bf16:llama.cpp": "closed", + "hf@bf16:hf": "closed", + "vllm@bf16:vllm": "closed", + "ggml@quant:llama.cpp": "closed" + }, + "Qwen3.5-4B/Q4_K_M|code-is-palindrome": { + "ggml@bf16:llama.cpp": "closed", + "hf@bf16:hf": "closed", + "vllm@bf16:vllm": "closed", + "ggml@quant:llama.cpp": "unclosed" + }, + "Qwen3.5-4B/Q4_K_M|code-fib": { + "ggml@bf16:llama.cpp": "unclosed", + "hf@bf16:hf": "unclosed", + "vllm@bf16:vllm": "unclosed", + "ggml@quant:llama.cpp": "unclosed" + }, + "Qwen3.5-4B/Q4_K_M|code-count-vowels": { + "ggml@bf16:llama.cpp": "closed", + "hf@bf16:hf": "closed", + "vllm@bf16:vllm": "closed", + "ggml@quant:llama.cpp": "closed" + }, + "Qwen3.5-4B/Q4_K_M|long-primes-10": { + "ggml@bf16:llama.cpp": "closed", + "hf@bf16:hf": "closed", + "vllm@bf16:vllm": "closed", + "ggml@quant:llama.cpp": "closed" + }, + "Qwen3.5-4B/Q4_K_M|long-countdown-15": { + "ggml@bf16:llama.cpp": "closed", + "hf@bf16:hf": "closed", + "vllm@bf16:vllm": "closed", + "ggml@quant:llama.cpp": "closed" + }, + "Qwen3.5-4B/UD-Q4_K_XL|ctl-2plus2": { + "ggml@bf16:llama.cpp": "closed", + "hf@bf16:hf": "closed", + "vllm@bf16:vllm": "closed", + "ggml@quant:llama.cpp": "closed" + }, + "Qwen3.5-4B/UD-Q4_K_XL|ctl-recall-7": { + "ggml@bf16:llama.cpp": "closed", + "hf@bf16:hf": "closed", + "vllm@bf16:vllm": "closed", + "ggml@quant:llama.cpp": "unclosed" + }, + "Qwen3.5-4B/UD-Q4_K_XL|ctl-code-add": { + "ggml@bf16:llama.cpp": "closed", + "hf@bf16:hf": "closed", + "vllm@bf16:vllm": "closed", + "ggml@quant:llama.cpp": "closed" + }, + "Qwen3.5-4B/UD-Q4_K_XL|fact-capital-japan": { + "ggml@bf16:llama.cpp": "closed", + "hf@bf16:hf": "closed", + "vllm@bf16:vllm": "closed", + "ggml@quant:llama.cpp": "closed" + }, + "Qwen3.5-4B/UD-Q4_K_XL|fact-capital-france": { + "ggml@bf16:llama.cpp": "closed", + "hf@bf16:hf": "closed", + "vllm@bf16:vllm": "closed", + "ggml@quant:llama.cpp": "closed" + }, + "Qwen3.5-4B/UD-Q4_K_XL|golden-paris": { + "ggml@bf16:llama.cpp": "closed", + "hf@bf16:hf": "closed", + "vllm@bf16:vllm": "closed", + "ggml@quant:llama.cpp": "closed" + }, + "Qwen3.5-4B/UD-Q4_K_XL|fact-water-formula": { + "ggml@bf16:llama.cpp": "unclosed", + "hf@bf16:hf": "unclosed", + "vllm@bf16:vllm": "unclosed", + "ggml@quant:llama.cpp": "unclosed" + }, + "Qwen3.5-4B/UD-Q4_K_XL|nonascii-water-kanji": { + "ggml@bf16:llama.cpp": "closed", + "hf@bf16:hf": "closed", + "vllm@bf16:vllm": "closed", + "ggml@quant:llama.cpp": "closed" + }, + "Qwen3.5-4B/UD-Q4_K_XL|arith-17x23": { + "ggml@bf16:llama.cpp": "closed", + "hf@bf16:hf": "closed", + "vllm@bf16:vllm": "closed", + "ggml@quant:llama.cpp": "closed" + }, + "Qwen3.5-4B/UD-Q4_K_XL|arith-precedence": { + "ggml@bf16:llama.cpp": "closed", + "hf@bf16:hf": "closed", + "vllm@bf16:vllm": "closed", + "ggml@quant:llama.cpp": "closed" + }, + "Qwen3.5-4B/UD-Q4_K_XL|arith-word-apples": { + "ggml@bf16:llama.cpp": "closed", + "hf@bf16:hf": "closed", + "vllm@bf16:vllm": "closed", + "ggml@quant:llama.cpp": "closed" + }, + "Qwen3.5-4B/UD-Q4_K_XL|hard-add-4digit": { + "ggml@bf16:llama.cpp": "closed", + "hf@bf16:hf": "closed", + "vllm@bf16:vllm": "closed", + "ggml@quant:llama.cpp": "unclosed" + }, + "Qwen3.5-4B/UD-Q4_K_XL|hard-not-prime": { + "ggml@bf16:llama.cpp": "closed", + "hf@bf16:hf": "closed", + "vllm@bf16:vllm": "closed", + "ggml@quant:llama.cpp": "closed" + }, + "Qwen3.5-4B/UD-Q4_K_XL|hard-reverse-word": { + "ggml@bf16:llama.cpp": "closed", + "hf@bf16:hf": "closed", + "vllm@bf16:vllm": "closed", + "ggml@quant:llama.cpp": "closed" + }, + "Qwen3.5-4B/UD-Q4_K_XL|hard-count-letters": { + "ggml@bf16:llama.cpp": "closed", + "hf@bf16:hf": "closed", + "vllm@bf16:vllm": "closed", + "ggml@quant:llama.cpp": "closed" + }, + "Qwen3.5-4B/UD-Q4_K_XL|recall-cats": { + "ggml@bf16:llama.cpp": "unclosed", + "hf@bf16:hf": "closed", + "vllm@bf16:vllm": "closed", + "ggml@quant:llama.cpp": "unclosed" + }, + "Qwen3.5-4B/UD-Q4_K_XL|recall-name": { + "ggml@bf16:llama.cpp": "closed", + "hf@bf16:hf": "closed", + "vllm@bf16:vllm": "closed", + "ggml@quant:llama.cpp": "unclosed" + }, + "Qwen3.5-4B/UD-Q4_K_XL|recall-arith-chain": { + "ggml@bf16:llama.cpp": "closed", + "hf@bf16:hf": "closed", + "vllm@bf16:vllm": "closed", + "ggml@quant:llama.cpp": "closed" + }, + "Qwen3.5-4B/UD-Q4_K_XL|code-is-palindrome": { + "ggml@bf16:llama.cpp": "closed", + "hf@bf16:hf": "closed", + "vllm@bf16:vllm": "closed", + "ggml@quant:llama.cpp": "unclosed" + }, + "Qwen3.5-4B/UD-Q4_K_XL|code-fib": { + "ggml@bf16:llama.cpp": "unclosed", + "hf@bf16:hf": "unclosed", + "vllm@bf16:vllm": "unclosed", + "ggml@quant:llama.cpp": "unclosed" + }, + "Qwen3.5-4B/UD-Q4_K_XL|code-count-vowels": { + "ggml@bf16:llama.cpp": "closed", + "hf@bf16:hf": "closed", + "vllm@bf16:vllm": "closed", + "ggml@quant:llama.cpp": "unclosed" + }, + "Qwen3.5-4B/UD-Q4_K_XL|long-primes-10": { + "ggml@bf16:llama.cpp": "closed", + "hf@bf16:hf": "closed", + "vllm@bf16:vllm": "closed", + "ggml@quant:llama.cpp": "closed" + }, + "Qwen3.5-4B/UD-Q4_K_XL|long-countdown-15": { + "ggml@bf16:llama.cpp": "closed", + "hf@bf16:hf": "closed", + "vllm@bf16:vllm": "closed", + "ggml@quant:llama.cpp": "closed" + } + }, + "cells": [ + { + "leg": "ggml@bf16", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": false, + "why": "unclosed think (budget exhausted)", + "extracted": null, + "think": "unclosed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L1", + "prompt_id": "ctl-2plus2", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": false, + "why": "unclosed think (budget exhausted)", + "extracted": null, + "think": "unclosed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L263", + "prompt_id": "ctl-2plus2", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "ctl-2plus2", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "ctl-2plus2", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "ctl-2plus2", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "4", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L11", + "prompt_id": "ctl-2plus2", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "4", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L1", + "prompt_id": "ctl-2plus2", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "4", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L23", + "prompt_id": "ctl-2plus2", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "4", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L33", + "prompt_id": "ctl-2plus2", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": false, + "why": "unclosed think (budget exhausted)", + "extracted": null, + "think": "unclosed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L2", + "prompt_id": "ctl-recall-7", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": false, + "why": "unclosed think (budget exhausted)", + "extracted": null, + "think": "unclosed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L264", + "prompt_id": "ctl-recall-7", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "ctl-recall-7", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "ctl-recall-7", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "ctl-recall-7", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "7", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L12", + "prompt_id": "ctl-recall-7", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "7", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L2", + "prompt_id": "ctl-recall-7", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "7", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L24", + "prompt_id": "ctl-recall-7", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "7", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L34", + "prompt_id": "ctl-recall-7", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "def add(a, b):\n return a + b\n", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L3", + "prompt_id": "ctl-code-add", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "ctl-code-add", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "ctl-code-add", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "ctl-code-add", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "def add(a, b):\n return a + b\n", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L13", + "prompt_id": "ctl-code-add", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "def add(a, b):\n return a + b\n", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L3", + "prompt_id": "ctl-code-add", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "def add(a, b):\n return a + b\n", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L25", + "prompt_id": "ctl-code-add", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "def add(a, b):\n return a + b\n", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L35", + "prompt_id": "ctl-code-add", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": false, + "why": "unclosed think (budget exhausted)", + "extracted": null, + "think": "unclosed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L4", + "prompt_id": "fact-capital-japan", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "fact-capital-japan", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "fact-capital-japan", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "fact-capital-japan", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "Tokyo", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L14", + "prompt_id": "fact-capital-japan", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "Tokyo", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L4", + "prompt_id": "fact-capital-japan", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "Tokyo", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L26", + "prompt_id": "fact-capital-japan", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "Tokyo", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L36", + "prompt_id": "fact-capital-japan", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "Paris", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L5", + "prompt_id": "fact-capital-france", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "fact-capital-france", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "fact-capital-france", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "fact-capital-france", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "Paris", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L15", + "prompt_id": "fact-capital-france", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "Paris", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L5", + "prompt_id": "fact-capital-france", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "Paris", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L27", + "prompt_id": "fact-capital-france", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "Paris", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L37", + "prompt_id": "fact-capital-france", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "golden-paris", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "golden-paris", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "golden-paris", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "golden-paris", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "Paris", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L16", + "prompt_id": "golden-paris", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "Paris", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L6", + "prompt_id": "golden-paris", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "Paris", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L28", + "prompt_id": "golden-paris", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "Paris", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L38", + "prompt_id": "golden-paris", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": false, + "why": "unclosed think (budget exhausted)", + "extracted": null, + "think": "unclosed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L6", + "prompt_id": "fact-water-formula", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "fact-water-formula", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "fact-water-formula", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "fact-water-formula", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": false, + "why": "mismatch", + "extracted": "H_2O", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L17", + "prompt_id": "fact-water-formula", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": false, + "why": "mismatch", + "extracted": "H_2O", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L7", + "prompt_id": "fact-water-formula", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "lambda", + "correct": false, + "why": "mismatch", + "extracted": "H_2O", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L29", + "prompt_id": "fact-water-formula", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": false, + "why": "mismatch", + "extracted": "H₂O", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L39", + "prompt_id": "fact-water-formula", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "nonascii-water-kanji", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "nonascii-water-kanji", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "nonascii-water-kanji", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "nonascii-water-kanji", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "nonascii-water-kanji", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "nonascii-water-kanji", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "nonascii-water-kanji", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "nonascii-water-kanji", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": false, + "why": "unclosed think (budget exhausted)", + "extracted": null, + "think": "unclosed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L7", + "prompt_id": "arith-17x23", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "arith-17x23", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "arith-17x23", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "arith-17x23", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "391", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L18", + "prompt_id": "arith-17x23", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "391", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L8", + "prompt_id": "arith-17x23", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "391", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L30", + "prompt_id": "arith-17x23", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "391", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L40", + "prompt_id": "arith-17x23", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "23", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L8", + "prompt_id": "arith-precedence", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "arith-precedence", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "arith-precedence", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "arith-precedence", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "23", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L19", + "prompt_id": "arith-precedence", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "23", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L9", + "prompt_id": "arith-precedence", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "23", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L31", + "prompt_id": "arith-precedence", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "23", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L41", + "prompt_id": "arith-precedence", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "56", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L9", + "prompt_id": "arith-word-apples", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "arith-word-apples", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "arith-word-apples", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "arith-word-apples", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "56", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L20", + "prompt_id": "arith-word-apples", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "56", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L10", + "prompt_id": "arith-word-apples", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "56", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L32", + "prompt_id": "arith-word-apples", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "56", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L42", + "prompt_id": "arith-word-apples", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "8613", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L10", + "prompt_id": "hard-add-4digit", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "hard-add-4digit", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "hard-add-4digit", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "hard-add-4digit", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "8613", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L21", + "prompt_id": "hard-add-4digit", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "8613", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L11", + "prompt_id": "hard-add-4digit", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "8613", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L33", + "prompt_id": "hard-add-4digit", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "8613", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L43", + "prompt_id": "hard-add-4digit", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "hard-not-prime", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "hard-not-prime", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "hard-not-prime", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "hard-not-prime", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "15", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L22", + "prompt_id": "hard-not-prime", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "15", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L12", + "prompt_id": "hard-not-prime", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "15", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L34", + "prompt_id": "hard-not-prime", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "15", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L44", + "prompt_id": "hard-not-prime", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "hard-reverse-word", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "hard-reverse-word", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "hard-reverse-word", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "hard-reverse-word", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": false, + "why": "mismatch", + "extracted": "tnipal", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L23", + "prompt_id": "hard-reverse-word", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": false, + "why": "mismatch", + "extracted": "tnipal", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L13", + "prompt_id": "hard-reverse-word", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "lambda", + "correct": false, + "why": "mismatch", + "extracted": "tnipal", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L35", + "prompt_id": "hard-reverse-word", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": false, + "why": "mismatch", + "extracted": "tenap", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L45", + "prompt_id": "hard-reverse-word", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "hard-count-letters", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "hard-count-letters", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "hard-count-letters", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "hard-count-letters", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "3", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L24", + "prompt_id": "hard-count-letters", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "3", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L14", + "prompt_id": "hard-count-letters", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "3", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L36", + "prompt_id": "hard-count-letters", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "3", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L46", + "prompt_id": "hard-count-letters", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "recall-cats", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "recall-cats", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "recall-cats", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "recall-cats", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "5", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L25", + "prompt_id": "recall-cats", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "5", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L15", + "prompt_id": "recall-cats", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "5", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L37", + "prompt_id": "recall-cats", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "5", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L47", + "prompt_id": "recall-cats", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "recall-name", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "recall-name", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "recall-name", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "recall-name", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "Ada", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L26", + "prompt_id": "recall-name", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "Ada", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L16", + "prompt_id": "recall-name", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "Ada", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L38", + "prompt_id": "recall-name", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "Ada", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L48", + "prompt_id": "recall-name", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "recall-arith-chain", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "recall-arith-chain", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "recall-arith-chain", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "recall-arith-chain", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "7", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L27", + "prompt_id": "recall-arith-chain", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "7", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L17", + "prompt_id": "recall-arith-chain", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "7", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L39", + "prompt_id": "recall-arith-chain", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "7", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L49", + "prompt_id": "recall-arith-chain", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "code-is-palindrome", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "code-is-palindrome", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "code-is-palindrome", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "code-is-palindrome", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "def is_palindrome(s):\n s = s.lower()\n return s == s[::-1]\n", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L28", + "prompt_id": "code-is-palindrome", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "def is_palindrome(s):\n s = s.lower()\n return s == s[::-1]\n", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L18", + "prompt_id": "code-is-palindrome", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "def is_palindrome(s):\n s = s.lower()\n return s == s[::-1]\n", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L40", + "prompt_id": "code-is-palindrome", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "def is_palindrome(s):\n s = s.lower()\n return s == s[::-1]\n", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L50", + "prompt_id": "code-is-palindrome", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "code-fib", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "code-fib", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "code-fib", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "code-fib", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "def fib(n):\n if n < 0:\n raise ValueError(\"n must be a non-negative integer\")\n if n == 0:\n return 0\n if n == 1:\n return 1\n a, b = 0, 1\n for _ in range(2, n + 1):\n a, b = b, a + b\n return b\n", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L29", + "prompt_id": "code-fib", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "def fib(n):\n if n < 0:\n raise ValueError(\"n must be a non-negative integer\")\n if n == 0:\n return 0\n if n == 1:\n return 1\n a, b = 0, 1\n for _ in range(2, n + 1):\n a, b = b, a + b\n return b\n", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L19", + "prompt_id": "code-fib", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "def fib(n):\n if n < 0:\n raise ValueError(\"n must be a non-negative integer\")\n if n == 0:\n return 0\n if n == 1:\n return 1\n a, b = 0, 1\n for _ in range(2, n + 1):\n a, b = b, a + b\n return b\n", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L41", + "prompt_id": "code-fib", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "def fib(n):\n if n < 0:\n raise ValueError(\"n must be a non-negative integer\")\n if n == 0:\n return 0\n if n == 1:\n return 1\n a, b = 0, 1\n for _ in range(2, n + 1):\n a, b = b, a + b\n return b\n", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L51", + "prompt_id": "code-fib", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "code-count-vowels", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "code-count-vowels", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "code-count-vowels", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "code-count-vowels", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "def count_vowels(s):\n vowels = set('aeiouAEIOU')\n return sum(1 for char in s if char.lower() in vowels)\n", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L30", + "prompt_id": "code-count-vowels", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "def count_vowels(s):\n vowels = set('aeiouAEIOU')\n return sum(1 for char in s if char.lower() in vowels)\n", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L20", + "prompt_id": "code-count-vowels", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "def count_vowels(s):\n vowels = set('aeiouAEIOU')\n return sum(1 for char in s if char.lower() in vowels)\n", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L42", + "prompt_id": "code-count-vowels", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "def count_vowels(s):\n vowels = set('aeiouAEIOU')\n return sum(1 for char in s if char in vowels)\n", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L52", + "prompt_id": "code-count-vowels", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "long-primes-10", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "long-primes-10", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "long-primes-10", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "long-primes-10", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": [ + "2", + "3", + "5", + "7", + "11", + "13", + "17", + "19", + "23", + "29" + ], + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L31", + "prompt_id": "long-primes-10", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": [ + "2", + "3", + "5", + "7", + "11", + "13", + "17", + "19", + "23", + "29" + ], + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L21", + "prompt_id": "long-primes-10", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": [ + "2", + "3", + "5", + "7", + "11", + "13", + "17", + "19", + "23", + "29" + ], + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L43", + "prompt_id": "long-primes-10", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": [ + "2", + "3", + "5", + "7", + "11", + "13", + "17", + "19", + "23", + "29" + ], + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L53", + "prompt_id": "long-primes-10", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "long-countdown-15", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "long-countdown-15", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "long-countdown-15", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "long-countdown-15", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": [ + "15", + "14", + "13", + "12", + "11", + "10", + "9", + "8", + "7", + "6", + "5", + "4", + "3", + "2", + "1" + ], + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L32", + "prompt_id": "long-countdown-15", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": [ + "15", + "14", + "13", + "12", + "11", + "10", + "9", + "8", + "7", + "6", + "5", + "4", + "3", + "2", + "1" + ], + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L22", + "prompt_id": "long-countdown-15", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": [ + "15", + "14", + "13", + "12", + "11", + "10", + "9", + "8", + "7", + "6", + "5", + "4", + "3", + "2", + "1" + ], + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L44", + "prompt_id": "long-countdown-15", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": [ + "15", + "14", + "13", + "12", + "11", + "10", + "9", + "8", + "7", + "6", + "5", + "4", + "3", + "2", + "1" + ], + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L54", + "prompt_id": "long-countdown-15", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "4", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L121", + "prompt_id": "ctl-2plus2", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "engine": "hf", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "4", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L187", + "prompt_id": "ctl-2plus2", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "4", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L235", + "prompt_id": "ctl-2plus2", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "4", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L143", + "prompt_id": "ctl-2plus2", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "4", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L55", + "prompt_id": "ctl-2plus2", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "4", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L45", + "prompt_id": "ctl-2plus2", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "4", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L210", + "prompt_id": "ctl-2plus2", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "4", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L77", + "prompt_id": "ctl-2plus2", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "7", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L122", + "prompt_id": "ctl-recall-7", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "engine": "hf", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "7", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L188", + "prompt_id": "ctl-recall-7", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "7", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L236", + "prompt_id": "ctl-recall-7", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": false, + "why": "unclosed think (budget exhausted)", + "extracted": null, + "think": "unclosed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L144", + "prompt_id": "ctl-recall-7", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "7", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L56", + "prompt_id": "ctl-recall-7", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "7", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L46", + "prompt_id": "ctl-recall-7", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "7", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L211", + "prompt_id": "ctl-recall-7", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "7", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L78", + "prompt_id": "ctl-recall-7", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "def add(a, b):\n return a + b\n", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L127", + "prompt_id": "ctl-code-add", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "engine": "hf", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "def add(a, b):\n return a + b\n", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L189", + "prompt_id": "ctl-code-add", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "def add(a, b):\n return a + b\n", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L237", + "prompt_id": "ctl-code-add", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "def add(a, b):\n return a + b\n", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L145", + "prompt_id": "ctl-code-add", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "def add(a, b):\n return a + b\n", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L57", + "prompt_id": "ctl-code-add", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "def add(a, b):\n return a + b\n", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L47", + "prompt_id": "ctl-code-add", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "def add(a, b):\n return a + b\n", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L212", + "prompt_id": "ctl-code-add", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "def add(a, b):\n return a + b\n", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L79", + "prompt_id": "ctl-code-add", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "Tokyo", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L123", + "prompt_id": "fact-capital-japan", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "engine": "hf", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "Tokyo", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L190", + "prompt_id": "fact-capital-japan", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "Tokyo", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L238", + "prompt_id": "fact-capital-japan", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "Tokyo", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L146", + "prompt_id": "fact-capital-japan", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "Tokyo", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L58", + "prompt_id": "fact-capital-japan", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "Tokyo", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L48", + "prompt_id": "fact-capital-japan", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "Tokyo", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L213", + "prompt_id": "fact-capital-japan", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "Tokyo", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L80", + "prompt_id": "fact-capital-japan", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "Paris", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L124", + "prompt_id": "fact-capital-france", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "engine": "hf", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "Paris", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L191", + "prompt_id": "fact-capital-france", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "Paris", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L239", + "prompt_id": "fact-capital-france", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "Paris", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L147", + "prompt_id": "fact-capital-france", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "Paris", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L59", + "prompt_id": "fact-capital-france", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "Paris", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L49", + "prompt_id": "fact-capital-france", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "Paris", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L214", + "prompt_id": "fact-capital-france", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "Paris", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L81", + "prompt_id": "fact-capital-france", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "Paris", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L128", + "prompt_id": "golden-paris", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "engine": "hf", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "Paris", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L192", + "prompt_id": "golden-paris", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "Paris", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L240", + "prompt_id": "golden-paris", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "Paris", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L148", + "prompt_id": "golden-paris", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "Paris", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L60", + "prompt_id": "golden-paris", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "Paris", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L50", + "prompt_id": "golden-paris", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "Paris", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L215", + "prompt_id": "golden-paris", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "Paris", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L82", + "prompt_id": "golden-paris", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": false, + "why": "unclosed think (budget exhausted)", + "extracted": null, + "think": "unclosed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L125", + "prompt_id": "fact-water-formula", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "engine": "hf", + "verb": "chat", + "host": "gx10", + "correct": false, + "why": "unclosed think (budget exhausted)", + "extracted": null, + "think": "unclosed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L193", + "prompt_id": "fact-water-formula", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": false, + "why": "unclosed think (budget exhausted)", + "extracted": null, + "think": "unclosed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L241", + "prompt_id": "fact-water-formula", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": false, + "why": "mismatch", + "extracted": "H_2O", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L149", + "prompt_id": "fact-water-formula", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": false, + "why": "mismatch", + "extracted": "H₂O", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L61", + "prompt_id": "fact-water-formula", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": false, + "why": "mismatch", + "extracted": "H₂O", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L51", + "prompt_id": "fact-water-formula", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": false, + "why": "mismatch", + "extracted": "H₂O", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L216", + "prompt_id": "fact-water-formula", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": false, + "why": "mismatch", + "extracted": "H2O", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L83", + "prompt_id": "fact-water-formula", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "水", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L233", + "prompt_id": "nonascii-water-kanji", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "engine": "hf", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "水", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L194", + "prompt_id": "nonascii-water-kanji", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "水", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L242", + "prompt_id": "nonascii-water-kanji", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "水", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L258", + "prompt_id": "nonascii-water-kanji", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "水", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L234", + "prompt_id": "nonascii-water-kanji", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "水", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L262", + "prompt_id": "nonascii-water-kanji", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "水", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L217", + "prompt_id": "nonascii-water-kanji", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "水", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L259", + "prompt_id": "nonascii-water-kanji", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "391", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L126", + "prompt_id": "arith-17x23", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "engine": "hf", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "391", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L195", + "prompt_id": "arith-17x23", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "391", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L243", + "prompt_id": "arith-17x23", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "391", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L150", + "prompt_id": "arith-17x23", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "391", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L62", + "prompt_id": "arith-17x23", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "391", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L52", + "prompt_id": "arith-17x23", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "391", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L218", + "prompt_id": "arith-17x23", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "391", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L84", + "prompt_id": "arith-17x23", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "23", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L129", + "prompt_id": "arith-precedence", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "engine": "hf", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "23", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L196", + "prompt_id": "arith-precedence", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "23", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L244", + "prompt_id": "arith-precedence", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "23", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L151", + "prompt_id": "arith-precedence", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "23", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L63", + "prompt_id": "arith-precedence", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "23", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L53", + "prompt_id": "arith-precedence", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "23", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L219", + "prompt_id": "arith-precedence", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "23", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L85", + "prompt_id": "arith-precedence", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "56", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L130", + "prompt_id": "arith-word-apples", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "engine": "hf", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "56", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L197", + "prompt_id": "arith-word-apples", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "56", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L245", + "prompt_id": "arith-word-apples", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "56", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L152", + "prompt_id": "arith-word-apples", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "56", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L64", + "prompt_id": "arith-word-apples", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "56", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L54", + "prompt_id": "arith-word-apples", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "56", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L220", + "prompt_id": "arith-word-apples", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "56", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L86", + "prompt_id": "arith-word-apples", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "8613", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L131", + "prompt_id": "hard-add-4digit", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "engine": "hf", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "8613", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L198", + "prompt_id": "hard-add-4digit", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "8613", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L246", + "prompt_id": "hard-add-4digit", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": false, + "why": "unclosed think (budget exhausted)", + "extracted": null, + "think": "unclosed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L153", + "prompt_id": "hard-add-4digit", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "8613", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L65", + "prompt_id": "hard-add-4digit", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "8613", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L55", + "prompt_id": "hard-add-4digit", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "8613", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L221", + "prompt_id": "hard-add-4digit", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "8613", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L87", + "prompt_id": "hard-add-4digit", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "15", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L132", + "prompt_id": "hard-not-prime", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "engine": "hf", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "15", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L199", + "prompt_id": "hard-not-prime", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "15", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L247", + "prompt_id": "hard-not-prime", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "15", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L154", + "prompt_id": "hard-not-prime", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "15", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L66", + "prompt_id": "hard-not-prime", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "15", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L56", + "prompt_id": "hard-not-prime", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "15", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L222", + "prompt_id": "hard-not-prime", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "15", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L88", + "prompt_id": "hard-not-prime", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "tenalp", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L133", + "prompt_id": "hard-reverse-word", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "engine": "hf", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "tenalp", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L200", + "prompt_id": "hard-reverse-word", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "tenalp", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L248", + "prompt_id": "hard-reverse-word", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "tenalp", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L155", + "prompt_id": "hard-reverse-word", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "tenalp", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L67", + "prompt_id": "hard-reverse-word", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "tenalp", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L57", + "prompt_id": "hard-reverse-word", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "tenalp", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L223", + "prompt_id": "hard-reverse-word", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "tenalp", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L89", + "prompt_id": "hard-reverse-word", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "3", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L134", + "prompt_id": "hard-count-letters", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "engine": "hf", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "3", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L201", + "prompt_id": "hard-count-letters", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "3", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L249", + "prompt_id": "hard-count-letters", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "3", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L156", + "prompt_id": "hard-count-letters", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "3", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L68", + "prompt_id": "hard-count-letters", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "3", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L58", + "prompt_id": "hard-count-letters", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "3", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L224", + "prompt_id": "hard-count-letters", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "3", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L90", + "prompt_id": "hard-count-letters", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": false, + "why": "unclosed think (budget exhausted)", + "extracted": null, + "think": "unclosed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L135", + "prompt_id": "recall-cats", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "engine": "hf", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "5", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L202", + "prompt_id": "recall-cats", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "5", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L250", + "prompt_id": "recall-cats", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": false, + "why": "unclosed think (budget exhausted)", + "extracted": null, + "think": "unclosed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L157", + "prompt_id": "recall-cats", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "5", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L69", + "prompt_id": "recall-cats", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "5", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L59", + "prompt_id": "recall-cats", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "5", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L225", + "prompt_id": "recall-cats", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "5", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L91", + "prompt_id": "recall-cats", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "Ada", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L136", + "prompt_id": "recall-name", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "engine": "hf", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "Ada", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L203", + "prompt_id": "recall-name", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "Ada", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L251", + "prompt_id": "recall-name", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": false, + "why": "unclosed think (budget exhausted)", + "extracted": null, + "think": "unclosed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L158", + "prompt_id": "recall-name", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "Ada", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L70", + "prompt_id": "recall-name", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "Ada", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L60", + "prompt_id": "recall-name", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "Ada", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L226", + "prompt_id": "recall-name", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "Ada", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L92", + "prompt_id": "recall-name", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "7", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L137", + "prompt_id": "recall-arith-chain", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "engine": "hf", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "7", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L204", + "prompt_id": "recall-arith-chain", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "7", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L252", + "prompt_id": "recall-arith-chain", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "7", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L159", + "prompt_id": "recall-arith-chain", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "7", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L71", + "prompt_id": "recall-arith-chain", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "7", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L61", + "prompt_id": "recall-arith-chain", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "7", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L227", + "prompt_id": "recall-arith-chain", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "7", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L93", + "prompt_id": "recall-arith-chain", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "def is_palindrome(s):\n s = s.lower()\n return s == s[::-1]\n", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L138", + "prompt_id": "code-is-palindrome", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "engine": "hf", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "def is_palindrome(s):\n return s.lower() == s[::-1].lower()\n", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L205", + "prompt_id": "code-is-palindrome", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": false, + "why": "tests_failed: AssertionError", + "extracted": "def is_palindrome(s):\n return s.lower() == s[::-1]\n", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L253", + "prompt_id": "code-is-palindrome", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": false, + "why": "unclosed think (budget exhausted)", + "extracted": null, + "think": "unclosed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L160", + "prompt_id": "code-is-palindrome", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "def is_palindrome(s):\n cleaned = ''.join(char.lower() for char in s if char.isalnum())\n return cleaned == cleaned[::-1]\n", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L72", + "prompt_id": "code-is-palindrome", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "def is_palindrome(s):\n cleaned = ''.join(char.lower() for char in s if char.isalnum())\n return cleaned == cleaned[::-1]\n", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L62", + "prompt_id": "code-is-palindrome", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "def is_palindrome(s):\n cleaned = ''.join(char.lower() for char in s if char.isalnum())\n return cleaned == cleaned[::-1]\n", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L228", + "prompt_id": "code-is-palindrome", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "def is_palindrome(s):\n cleaned = ''.join(char.lower() for char in s if char.isalnum())\n return cleaned == cleaned[::-1]\n", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L94", + "prompt_id": "code-is-palindrome", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": false, + "why": "unclosed think (budget exhausted)", + "extracted": null, + "think": "unclosed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L139", + "prompt_id": "code-fib", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "engine": "hf", + "verb": "chat", + "host": "gx10", + "correct": false, + "why": "unclosed think (budget exhausted)", + "extracted": null, + "think": "unclosed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L206", + "prompt_id": "code-fib", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": false, + "why": "unclosed think (budget exhausted)", + "extracted": null, + "think": "unclosed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L254", + "prompt_id": "code-fib", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": false, + "why": "unclosed think (budget exhausted)", + "extracted": null, + "think": "unclosed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L161", + "prompt_id": "code-fib", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "def fib(n):\n if n == 0:\n return 0\n elif n == 1:\n return 1\n else:\n a, b = 0, 1\n for _ in range(2, n + 1):\n a, b = b, a + b\n return b\n", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L73", + "prompt_id": "code-fib", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "def fib(n):\n if n == 0:\n return 0\n elif n == 1:\n return 1\n a, b = 0, 1\n for _ in range(2, n + 1):\n a, b = b, a + b\n return b\n", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L63", + "prompt_id": "code-fib", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "def fib(n):\n if n == 0:\n return 0\n elif n == 1:\n return 1\n a, b = 0, 1\n for _ in range(2, n + 1):\n a, b = b, a + b\n return b\n", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L229", + "prompt_id": "code-fib", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "def fib(n):\n if n == 0:\n return 0\n elif n == 1:\n return 1\n else:\n a, b = 0, 1\n for _ in range(2, n + 1):\n a, b = b, a + b\n return b\n", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L95", + "prompt_id": "code-fib", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "def count_vowels(s):\n count = 0\n for char in s:\n if char.lower() in 'aeiou':\n count += 1\n return count\n", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L140", + "prompt_id": "code-count-vowels", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "engine": "hf", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "def count_vowels(s):\n count = 0\n vowels = \"aeiou\"\n for char in s:\n if char.lower() in vowels:\n count += 1\n return count\n", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L207", + "prompt_id": "code-count-vowels", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "def count_vowels(s):\n return sum(1 for char in s if char.lower() in 'aeiou')\n", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L255", + "prompt_id": "code-count-vowels", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "def count_vowels(s):\n vowels = \"aeiou\"\n count = 0\n for char in s:\n if char.lower() in vowels:\n count += 1\n return count\n", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L162", + "prompt_id": "code-count-vowels", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "def count_vowels(s):\n vowels = set('aeiouAEIOU')\n return sum(1 for char in s if char in vowels)\n", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L74", + "prompt_id": "code-count-vowels", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "def count_vowels(s):\n vowels = set('aeiouAEIOU')\n return sum(1 for char in s if char in vowels)\n", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L64", + "prompt_id": "code-count-vowels", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "def count_vowels(s):\n vowels = set('aeiouAEIOU')\n return sum(1 for char in s if char in vowels)\n", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L230", + "prompt_id": "code-count-vowels", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "def count_vowels(s):\n vowels = \"aeiouAEIOU\"\n count = 0\n for char in s:\n if char in vowels:\n count += 1\n return count\n", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L96", + "prompt_id": "code-count-vowels", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": [ + "2", + "3", + "5", + "7", + "11", + "13", + "17", + "19", + "23", + "29" + ], + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L141", + "prompt_id": "long-primes-10", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "engine": "hf", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": [ + "2", + "3", + "5", + "7", + "11", + "13", + "17", + "19", + "23", + "29" + ], + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L208", + "prompt_id": "long-primes-10", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": [ + "2", + "3", + "5", + "7", + "11", + "13", + "17", + "19", + "23", + "29" + ], + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L256", + "prompt_id": "long-primes-10", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": [ + "2", + "3", + "5", + "7", + "11", + "13", + "17", + "19", + "23", + "29" + ], + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L163", + "prompt_id": "long-primes-10", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": [ + "2", + "3", + "5", + "7", + "11", + "13", + "17", + "19", + "23", + "29" + ], + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L75", + "prompt_id": "long-primes-10", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": [ + "2", + "3", + "5", + "7", + "11", + "13", + "17", + "19", + "23", + "29" + ], + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L65", + "prompt_id": "long-primes-10", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": [ + "2", + "3", + "5", + "7", + "11", + "13", + "17", + "19", + "23", + "29" + ], + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L231", + "prompt_id": "long-primes-10", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": [ + "2", + "3", + "5", + "7", + "11", + "13", + "17", + "19", + "23", + "29" + ], + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L97", + "prompt_id": "long-primes-10", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": [ + "15", + "14", + "13", + "12", + "11", + "10", + "9", + "8", + "7", + "6", + "5", + "4", + "3", + "2", + "1" + ], + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L142", + "prompt_id": "long-countdown-15", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "engine": "hf", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": [ + "15", + "14", + "13", + "12", + "11", + "10", + "9", + "8", + "7", + "6", + "5", + "4", + "3", + "2", + "1" + ], + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L209", + "prompt_id": "long-countdown-15", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": [ + "15", + "14", + "13", + "12", + "11", + "10", + "9", + "8", + "7", + "6", + "5", + "4", + "3", + "2", + "1" + ], + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L257", + "prompt_id": "long-countdown-15", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": [ + "15", + "14", + "13", + "12", + "11", + "10", + "9", + "8", + "7", + "6", + "5", + "4", + "3", + "2", + "1" + ], + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L164", + "prompt_id": "long-countdown-15", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": [ + "15", + "14", + "13", + "12", + "11", + "10", + "9", + "8", + "7", + "6", + "5", + "4", + "3", + "2", + "1" + ], + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L76", + "prompt_id": "long-countdown-15", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": [ + "15", + "14", + "13", + "12", + "11", + "10", + "9", + "8", + "7", + "6", + "5", + "4", + "3", + "2", + "1" + ], + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L66", + "prompt_id": "long-countdown-15", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": [ + "15", + "14", + "13", + "12", + "11", + "10", + "9", + "8", + "7", + "6", + "5", + "4", + "3", + "2", + "1" + ], + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L232", + "prompt_id": "long-countdown-15", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": [ + "15", + "14", + "13", + "12", + "11", + "10", + "9", + "8", + "7", + "6", + "5", + "4", + "3", + "2", + "1" + ], + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L98", + "prompt_id": "long-countdown-15", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "4", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L121", + "prompt_id": "ctl-2plus2", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "engine": "hf", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "4", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L187", + "prompt_id": "ctl-2plus2", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "4", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L235", + "prompt_id": "ctl-2plus2", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "4", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L165", + "prompt_id": "ctl-2plus2", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "4", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L55", + "prompt_id": "ctl-2plus2", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "4", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L45", + "prompt_id": "ctl-2plus2", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "4", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L210", + "prompt_id": "ctl-2plus2", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "4", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L99", + "prompt_id": "ctl-2plus2", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "7", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L122", + "prompt_id": "ctl-recall-7", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "engine": "hf", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "7", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L188", + "prompt_id": "ctl-recall-7", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "7", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L236", + "prompt_id": "ctl-recall-7", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": false, + "why": "unclosed think (budget exhausted)", + "extracted": null, + "think": "unclosed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L166", + "prompt_id": "ctl-recall-7", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "7", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L56", + "prompt_id": "ctl-recall-7", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "7", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L46", + "prompt_id": "ctl-recall-7", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "7", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L211", + "prompt_id": "ctl-recall-7", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "7", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L100", + "prompt_id": "ctl-recall-7", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "def add(a, b):\n return a + b\n", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L127", + "prompt_id": "ctl-code-add", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "engine": "hf", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "def add(a, b):\n return a + b\n", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L189", + "prompt_id": "ctl-code-add", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "def add(a, b):\n return a + b\n", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L237", + "prompt_id": "ctl-code-add", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "def add(a, b):\n return a + b\n", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L167", + "prompt_id": "ctl-code-add", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "def add(a, b):\n return a + b\n", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L57", + "prompt_id": "ctl-code-add", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "def add(a, b):\n return a + b\n", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L47", + "prompt_id": "ctl-code-add", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "def add(a, b):\n return a + b\n", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L212", + "prompt_id": "ctl-code-add", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "def add(a, b):\n return a + b\n", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L101", + "prompt_id": "ctl-code-add", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "Tokyo", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L123", + "prompt_id": "fact-capital-japan", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "engine": "hf", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "Tokyo", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L190", + "prompt_id": "fact-capital-japan", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "Tokyo", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L238", + "prompt_id": "fact-capital-japan", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "Tokyo", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L168", + "prompt_id": "fact-capital-japan", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "Tokyo", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L58", + "prompt_id": "fact-capital-japan", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "Tokyo", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L48", + "prompt_id": "fact-capital-japan", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "Tokyo", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L213", + "prompt_id": "fact-capital-japan", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "Tokyo", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L102", + "prompt_id": "fact-capital-japan", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "Paris", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L124", + "prompt_id": "fact-capital-france", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "engine": "hf", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "Paris", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L191", + "prompt_id": "fact-capital-france", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "Paris", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L239", + "prompt_id": "fact-capital-france", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "Paris", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L169", + "prompt_id": "fact-capital-france", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "Paris", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L59", + "prompt_id": "fact-capital-france", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "Paris", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L49", + "prompt_id": "fact-capital-france", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "Paris", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L214", + "prompt_id": "fact-capital-france", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "Paris", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L103", + "prompt_id": "fact-capital-france", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "Paris", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L128", + "prompt_id": "golden-paris", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "engine": "hf", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "Paris", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L192", + "prompt_id": "golden-paris", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "Paris", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L240", + "prompt_id": "golden-paris", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "Paris", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L170", + "prompt_id": "golden-paris", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "Paris", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L60", + "prompt_id": "golden-paris", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "Paris", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L50", + "prompt_id": "golden-paris", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "Paris", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L215", + "prompt_id": "golden-paris", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "Paris", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L104", + "prompt_id": "golden-paris", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": false, + "why": "unclosed think (budget exhausted)", + "extracted": null, + "think": "unclosed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L125", + "prompt_id": "fact-water-formula", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "engine": "hf", + "verb": "chat", + "host": "gx10", + "correct": false, + "why": "unclosed think (budget exhausted)", + "extracted": null, + "think": "unclosed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L193", + "prompt_id": "fact-water-formula", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": false, + "why": "unclosed think (budget exhausted)", + "extracted": null, + "think": "unclosed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L241", + "prompt_id": "fact-water-formula", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": false, + "why": "unclosed think (budget exhausted)", + "extracted": null, + "think": "unclosed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L171", + "prompt_id": "fact-water-formula", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": false, + "why": "mismatch", + "extracted": "H₂O", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L61", + "prompt_id": "fact-water-formula", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": false, + "why": "mismatch", + "extracted": "H₂O", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L51", + "prompt_id": "fact-water-formula", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": false, + "why": "mismatch", + "extracted": "H₂O", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L216", + "prompt_id": "fact-water-formula", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": false, + "why": "mismatch", + "extracted": "H₂O", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L105", + "prompt_id": "fact-water-formula", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "水", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L233", + "prompt_id": "nonascii-water-kanji", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "engine": "hf", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "水", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L194", + "prompt_id": "nonascii-water-kanji", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "水", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L242", + "prompt_id": "nonascii-water-kanji", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "水", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L260", + "prompt_id": "nonascii-water-kanji", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "水", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L234", + "prompt_id": "nonascii-water-kanji", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "水", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L262", + "prompt_id": "nonascii-water-kanji", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "水", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L217", + "prompt_id": "nonascii-water-kanji", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "水", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L261", + "prompt_id": "nonascii-water-kanji", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "391", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L126", + "prompt_id": "arith-17x23", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "engine": "hf", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "391", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L195", + "prompt_id": "arith-17x23", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "391", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L243", + "prompt_id": "arith-17x23", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "391", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L172", + "prompt_id": "arith-17x23", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "391", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L62", + "prompt_id": "arith-17x23", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "391", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L52", + "prompt_id": "arith-17x23", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "391", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L218", + "prompt_id": "arith-17x23", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "391", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L106", + "prompt_id": "arith-17x23", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "23", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L129", + "prompt_id": "arith-precedence", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "engine": "hf", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "23", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L196", + "prompt_id": "arith-precedence", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "23", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L244", + "prompt_id": "arith-precedence", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "23", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L173", + "prompt_id": "arith-precedence", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "23", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L63", + "prompt_id": "arith-precedence", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "23", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L53", + "prompt_id": "arith-precedence", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "23", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L219", + "prompt_id": "arith-precedence", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "23", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L107", + "prompt_id": "arith-precedence", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "56", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L130", + "prompt_id": "arith-word-apples", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "engine": "hf", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "56", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L197", + "prompt_id": "arith-word-apples", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "56", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L245", + "prompt_id": "arith-word-apples", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "56", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L174", + "prompt_id": "arith-word-apples", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "56", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L64", + "prompt_id": "arith-word-apples", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "56", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L54", + "prompt_id": "arith-word-apples", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "56", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L220", + "prompt_id": "arith-word-apples", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "56", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L108", + "prompt_id": "arith-word-apples", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "8613", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L131", + "prompt_id": "hard-add-4digit", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "engine": "hf", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "8613", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L198", + "prompt_id": "hard-add-4digit", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "8613", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L246", + "prompt_id": "hard-add-4digit", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": false, + "why": "unclosed think (budget exhausted)", + "extracted": null, + "think": "unclosed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L175", + "prompt_id": "hard-add-4digit", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "8613", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L65", + "prompt_id": "hard-add-4digit", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "8613", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L55", + "prompt_id": "hard-add-4digit", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "8613", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L221", + "prompt_id": "hard-add-4digit", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "8613", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L109", + "prompt_id": "hard-add-4digit", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "15", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L132", + "prompt_id": "hard-not-prime", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "engine": "hf", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "15", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L199", + "prompt_id": "hard-not-prime", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "15", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L247", + "prompt_id": "hard-not-prime", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "15", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L176", + "prompt_id": "hard-not-prime", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "15", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L66", + "prompt_id": "hard-not-prime", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "15", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L56", + "prompt_id": "hard-not-prime", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "15", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L222", + "prompt_id": "hard-not-prime", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "15", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L110", + "prompt_id": "hard-not-prime", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "tenalp", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L133", + "prompt_id": "hard-reverse-word", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "engine": "hf", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "tenalp", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L200", + "prompt_id": "hard-reverse-word", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "tenalp", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L248", + "prompt_id": "hard-reverse-word", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": false, + "why": "mismatch", + "extracted": "teanlap", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L177", + "prompt_id": "hard-reverse-word", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "tenalp", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L67", + "prompt_id": "hard-reverse-word", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "tenalp", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L57", + "prompt_id": "hard-reverse-word", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "tenalp", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L223", + "prompt_id": "hard-reverse-word", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "tenalp", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L111", + "prompt_id": "hard-reverse-word", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "3", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L134", + "prompt_id": "hard-count-letters", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "engine": "hf", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "3", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L201", + "prompt_id": "hard-count-letters", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "3", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L249", + "prompt_id": "hard-count-letters", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "3", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L178", + "prompt_id": "hard-count-letters", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "3", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L68", + "prompt_id": "hard-count-letters", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "3", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L58", + "prompt_id": "hard-count-letters", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "3", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L224", + "prompt_id": "hard-count-letters", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "3", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L112", + "prompt_id": "hard-count-letters", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": false, + "why": "unclosed think (budget exhausted)", + "extracted": null, + "think": "unclosed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L135", + "prompt_id": "recall-cats", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "engine": "hf", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "5", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L202", + "prompt_id": "recall-cats", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "5", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L250", + "prompt_id": "recall-cats", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": false, + "why": "unclosed think (budget exhausted)", + "extracted": null, + "think": "unclosed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L179", + "prompt_id": "recall-cats", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "5", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L69", + "prompt_id": "recall-cats", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "5", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L59", + "prompt_id": "recall-cats", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "5", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L225", + "prompt_id": "recall-cats", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "5", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L113", + "prompt_id": "recall-cats", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "Ada", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L136", + "prompt_id": "recall-name", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "engine": "hf", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "Ada", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L203", + "prompt_id": "recall-name", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "Ada", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L251", + "prompt_id": "recall-name", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": false, + "why": "unclosed think (budget exhausted)", + "extracted": null, + "think": "unclosed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L180", + "prompt_id": "recall-name", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "Ada", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L70", + "prompt_id": "recall-name", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "Ada", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L60", + "prompt_id": "recall-name", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "Ada", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L226", + "prompt_id": "recall-name", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "Ada", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L114", + "prompt_id": "recall-name", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "7", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L137", + "prompt_id": "recall-arith-chain", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "engine": "hf", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "7", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L204", + "prompt_id": "recall-arith-chain", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "7", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L252", + "prompt_id": "recall-arith-chain", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "7", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L181", + "prompt_id": "recall-arith-chain", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "7", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L71", + "prompt_id": "recall-arith-chain", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "7", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L61", + "prompt_id": "recall-arith-chain", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "7", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L227", + "prompt_id": "recall-arith-chain", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "7", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L115", + "prompt_id": "recall-arith-chain", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "def is_palindrome(s):\n s = s.lower()\n return s == s[::-1]\n", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L138", + "prompt_id": "code-is-palindrome", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "engine": "hf", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "def is_palindrome(s):\n return s.lower() == s[::-1].lower()\n", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L205", + "prompt_id": "code-is-palindrome", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": false, + "why": "tests_failed: AssertionError", + "extracted": "def is_palindrome(s):\n return s.lower() == s[::-1]\n", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L253", + "prompt_id": "code-is-palindrome", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": false, + "why": "unclosed think (budget exhausted)", + "extracted": null, + "think": "unclosed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L182", + "prompt_id": "code-is-palindrome", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "def is_palindrome(s):\n cleaned = ''.join(char.lower() for char in s if char.isalnum())\n return cleaned == cleaned[::-1]\n", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L72", + "prompt_id": "code-is-palindrome", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "def is_palindrome(s):\n cleaned = ''.join(char.lower() for char in s if char.isalnum())\n return cleaned == cleaned[::-1]\n", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L62", + "prompt_id": "code-is-palindrome", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "def is_palindrome(s):\n cleaned = ''.join(char.lower() for char in s if char.isalnum())\n return cleaned == cleaned[::-1]\n", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L228", + "prompt_id": "code-is-palindrome", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "def is_palindrome(s):\n cleaned = ''.join(char.lower() for char in s if char.isalnum())\n return cleaned == cleaned[::-1]\n", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L116", + "prompt_id": "code-is-palindrome", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": false, + "why": "unclosed think (budget exhausted)", + "extracted": null, + "think": "unclosed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L139", + "prompt_id": "code-fib", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "engine": "hf", + "verb": "chat", + "host": "gx10", + "correct": false, + "why": "unclosed think (budget exhausted)", + "extracted": null, + "think": "unclosed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L206", + "prompt_id": "code-fib", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": false, + "why": "unclosed think (budget exhausted)", + "extracted": null, + "think": "unclosed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L254", + "prompt_id": "code-fib", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": false, + "why": "unclosed think (budget exhausted)", + "extracted": null, + "think": "unclosed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L183", + "prompt_id": "code-fib", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "def fib(n):\n if n == 0:\n return 0\n elif n == 1:\n return 1\n else:\n a, b = 0, 1\n for _ in range(2, n + 1):\n a, b = b, a + b\n return b\n", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L73", + "prompt_id": "code-fib", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "def fib(n):\n if n == 0:\n return 0\n elif n == 1:\n return 1\n a, b = 0, 1\n for _ in range(2, n + 1):\n a, b = b, a + b\n return b\n", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L63", + "prompt_id": "code-fib", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "def fib(n):\n if n == 0:\n return 0\n elif n == 1:\n return 1\n a, b = 0, 1\n for _ in range(2, n + 1):\n a, b = b, a + b\n return b\n", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L229", + "prompt_id": "code-fib", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "def fib(n):\n if n == 0:\n return 0\n elif n == 1:\n return 1\n else:\n a, b = 0, 1\n for _ in range(2, n + 1):\n a, b = b, a + b\n return b\n", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L117", + "prompt_id": "code-fib", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "def count_vowels(s):\n count = 0\n for char in s:\n if char.lower() in 'aeiou':\n count += 1\n return count\n", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L140", + "prompt_id": "code-count-vowels", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "engine": "hf", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "def count_vowels(s):\n count = 0\n vowels = \"aeiou\"\n for char in s:\n if char.lower() in vowels:\n count += 1\n return count\n", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L207", + "prompt_id": "code-count-vowels", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "def count_vowels(s):\n return sum(1 for char in s if char.lower() in 'aeiou')\n", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L255", + "prompt_id": "code-count-vowels", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": false, + "why": "unclosed think (budget exhausted)", + "extracted": null, + "think": "unclosed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L184", + "prompt_id": "code-count-vowels", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "def count_vowels(s):\n vowels = set('aeiouAEIOU')\n return sum(1 for char in s if char in vowels)\n", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L74", + "prompt_id": "code-count-vowels", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "def count_vowels(s):\n vowels = set('aeiouAEIOU')\n return sum(1 for char in s if char in vowels)\n", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L64", + "prompt_id": "code-count-vowels", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "def count_vowels(s):\n vowels = set('aeiouAEIOU')\n return sum(1 for char in s if char in vowels)\n", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L230", + "prompt_id": "code-count-vowels", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "def count_vowels(s):\n vowels = \"aeiouAEIOU\"\n count = 0\n for char in s:\n if char in vowels:\n count += 1\n return count\n", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L118", + "prompt_id": "code-count-vowels", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": [ + "2", + "3", + "5", + "7", + "11", + "13", + "17", + "19", + "23", + "29" + ], + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L141", + "prompt_id": "long-primes-10", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "engine": "hf", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": [ + "2", + "3", + "5", + "7", + "11", + "13", + "17", + "19", + "23", + "29" + ], + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L208", + "prompt_id": "long-primes-10", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": [ + "2", + "3", + "5", + "7", + "11", + "13", + "17", + "19", + "23", + "29" + ], + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L256", + "prompt_id": "long-primes-10", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": [ + "2", + "3", + "5", + "7", + "11", + "13", + "17", + "19", + "23", + "29" + ], + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L185", + "prompt_id": "long-primes-10", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": [ + "2", + "3", + "5", + "7", + "11", + "13", + "17", + "19", + "23", + "29" + ], + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L75", + "prompt_id": "long-primes-10", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": [ + "2", + "3", + "5", + "7", + "11", + "13", + "17", + "19", + "23", + "29" + ], + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L65", + "prompt_id": "long-primes-10", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": [ + "2", + "3", + "5", + "7", + "11", + "13", + "17", + "19", + "23", + "29" + ], + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L231", + "prompt_id": "long-primes-10", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": [ + "2", + "3", + "5", + "7", + "11", + "13", + "17", + "19", + "23", + "29" + ], + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L119", + "prompt_id": "long-primes-10", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": [ + "15", + "14", + "13", + "12", + "11", + "10", + "9", + "8", + "7", + "6", + "5", + "4", + "3", + "2", + "1" + ], + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L142", + "prompt_id": "long-countdown-15", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "engine": "hf", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": [ + "15", + "14", + "13", + "12", + "11", + "10", + "9", + "8", + "7", + "6", + "5", + "4", + "3", + "2", + "1" + ], + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L209", + "prompt_id": "long-countdown-15", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": [ + "15", + "14", + "13", + "12", + "11", + "10", + "9", + "8", + "7", + "6", + "5", + "4", + "3", + "2", + "1" + ], + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L257", + "prompt_id": "long-countdown-15", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": [ + "15", + "14", + "13", + "12", + "11", + "10", + "9", + "8", + "7", + "6", + "5", + "4", + "3", + "2", + "1" + ], + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L186", + "prompt_id": "long-countdown-15", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": [ + "15", + "14", + "13", + "12", + "11", + "10", + "9", + "8", + "7", + "6", + "5", + "4", + "3", + "2", + "1" + ], + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L76", + "prompt_id": "long-countdown-15", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": [ + "15", + "14", + "13", + "12", + "11", + "10", + "9", + "8", + "7", + "6", + "5", + "4", + "3", + "2", + "1" + ], + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L66", + "prompt_id": "long-countdown-15", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": [ + "15", + "14", + "13", + "12", + "11", + "10", + "9", + "8", + "7", + "6", + "5", + "4", + "3", + "2", + "1" + ], + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L232", + "prompt_id": "long-countdown-15", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": [ + "15", + "14", + "13", + "12", + "11", + "10", + "9", + "8", + "7", + "6", + "5", + "4", + "3", + "2", + "1" + ], + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L120", + "prompt_id": "long-countdown-15", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + } + ] +}