From 281b3fa937d98454065ed2f37696111037020b19 Mon Sep 17 00:00:00 2001 From: t Date: Mon, 26 Oct 2026 11:34:51 +0100 Subject: [PATCH 1/2] release: 0.72.0 bump-version.sh 0.72.0 across every workspace (both Cargo.lock files synced so package_tarball_build leaves the tree clean), the CHANGELOG [0.72.0] section for v0.71.4..a781138719, and the CRUX prompt certification carried from 0.71.3 (apr_commit a781138719bc, same prompt set, sha256 checked). Refs #6390 PMAT-Ticket: PMAT-6390 Co-Authored-By: Claude Sonnet 5.5 --- CHANGELOG.md | 160 + Cargo.lock | 170 +- Cargo.toml | 150 +- crates/apr-cli/Cargo.toml | 8 +- crates/aprender-bench-compute/Cargo.toml | 2 +- crates/aprender-bench-tokenizer/Cargo.toml | 2 +- crates/aprender-cbtop/Cargo.toml | 2 +- crates/aprender-cgp/Cargo.toml | 2 +- crates/aprender-compute/Cargo.toml | 4 +- crates/aprender-contracts-cli/Cargo.toml | 2 +- crates/aprender-contracts/Cargo.toml | 2 +- crates/aprender-core/Cargo.toml | 2 +- crates/aprender-cuda-edge/Cargo.toml | 2 +- crates/aprender-explain/Cargo.toml | 2 +- crates/aprender-graph/Cargo.toml | 2 +- crates/aprender-monte-carlo/Cargo.toml | 2 +- crates/aprender-orchestrate/Cargo.toml | 2 +- crates/aprender-profile/Cargo.toml | 4 +- crates/aprender-qa-cli/Cargo.toml | 8 +- crates/aprender-qa-report/Cargo.toml | 4 +- crates/aprender-qa-runner/Cargo.toml | 4 +- crates/aprender-rag-cli/Cargo.toml | 2 +- crates/aprender-rag/Cargo.toml | 2 +- crates/aprender-registry/Cargo.toml | 2 +- crates/aprender-serve/Cargo.toml | 4 +- crates/aprender-shell/Cargo.toml | 2 +- crates/aprender-simulate/Cargo.toml | 2 +- crates/aprender-test-cli/Cargo.toml | 2 +- crates/aprender-train-bench/Cargo.toml | 2 +- crates/aprender-train-distill/Cargo.toml | 2 +- crates/aprender-train-inspect/Cargo.toml | 2 +- crates/aprender-train-lora/Cargo.toml | 2 +- crates/aprender-train-shell/Cargo.toml | 2 +- crates/aprender-train/Cargo.toml | 4 +- crates/aprender-tsp/Cargo.toml | 2 +- crates/aprender-verify-ml/Cargo.toml | 2 +- crates/aprender-viz/Cargo.toml | 6 +- crates/facades/Cargo.lock | 4 +- .../provable-contracts-macros/Cargo.toml | 2 +- crates/facades/provable-contracts/Cargo.toml | 2 +- .../prompt-certification-inventory.json | 13 + .../crux/0.72.0/prompt-certification.json | 8906 +++++++++++++++++ 42 files changed, 9291 insertions(+), 212 deletions(-) create mode 100644 evidence/crux/0.72.0/prompt-certification-inventory.json create mode 100644 evidence/crux/0.72.0/prompt-certification.json diff --git a/CHANGELOG.md b/CHANGELOG.md index 91494854d2..8fccf329b5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,166 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [0.72.0] - 2026-10-26 + +Release commit-range: v0.71.4..a781138719 (250 non-merge commits). Entries are commit subjects. + +### Added + +- feat(roadmap): fragment shape check, newline-safe aggregation, bump regenerates (Refs #6288) +- feat(ci): one-PR-per-ticket R1 allowance for EMR-001 squash PRs (Refs #6267) +- feat(run): apr run --json response contract: status, error, model_digest (Refs #3720) +- feat(ladder): per-cell memory admission, deferred cells are not_measured (Refs #6211) +- feat(roadmap): generator and guard learn the frozen roadmap-base.yaml (S1a, Refs #6185) +- feat(affected-set): docs/contracts/*.md give READERS, not ALL; dormant, no consumer (Refs #6042) +- feat(ci-tools): lockfile-registry-packages, the Rust port of lockfile_registry_packages.py (Refs #4827) +- feat(pre-push): keyed warm cache + parallel guard-tree for pre_push_local.sh (#5983) +- feat(train): train-step profiler JSON report contract (Refs #5577) +- feat(flow-010): affected_set.sh helper, diff to crates + reverse-dep closure + nextest filter (E5) +- feat(rollup): rollup bot — select, build, verify, close-with-SHA (Refs #5737) (#5742) +- feat(simulate): PR-flow, repo-flow, course-flow scenarios and flow-model rows B1-B7 (Refs #5511) (#5559) +- feat(viz): GGPlot::to_svg draws title, xlab and ylab (Refs #5334, Refs #3590) (#5453) +- feat(apr-cli): derived-cell coverage for release surfaces, LAB checks and zero-cell refusal (#3745) (#5417) +- feat(bench): parity receipts carry per-replicate server resources (Refs #5322) (#5419) +- feat(ci-tools): port model_ladder_cells_produce.py to ladder-cells-produce (#5225) (#5410) +- feat(rex): land the review-experiment epic work on main (PMAT-4524) (#5393) +- feat(embed): EmbeddingGemma 2 EG-3..EG-6 — serve, CUDA f32, refusals + qa gate, GGUF round trip (Refs #5319) (#5390) +- feat(serve): Qwen3.5 on CUDA batches concurrent chat requests, sampled slots seeded per request (#5312) (#5364) +- feat(contracts): pv census ratchet, LAB nightly (#4709) (#5359) +- feat(edge): ollama parity rows, chat-template fix and verb x device matrix (#5254) (#5292) + +### Performance + +- perf(ci): balance workspace-test shards — GPU crates + their Σ check + two guards to shard 2 (Refs #5486) (#5487) +- perf(ci): CARGO_PROFILE_{DEV,TEST}_DEBUG=0 in workspace-test-shard (P1a, Refs #5486) (#5745) +- perf(guard): check_crux_ollama_in_lock runs its 6 dogfood rows 3 at a time, 71s -> 37s, output identical (#5485) +- perf(guard): check_silicon_coverage forked tr twice per label test (3680 forks per run) (#5484) +- perf(guard): cut forks in the guard-cargo apr-pinning scan (same verdicts) (#5482) +- perf(ci): tree-reader derive in one grep pass; guard-tree m0 1404s -> 606s (Refs #5427) (#5434) + +### Fixed + +- fix(contracts): regenerate contracts.nt for the 917 breakdown (Refs #6385) +- fix(audit): contract breakdowns sum to 917 (per-binary aprender-rollup, test-harness, q5_6) +- fix(audit): ledger the aprender-rollup verify verb and --apply (917 rows) +- fix(audit): restate the dogfood coverage totals for the 915-row ledger +- fix(audit): surface ledger rows for the aprender-rollup bin (T-2 pre-publish 31/31) +- fix(release): infra_gh names the path and status of a refused read (Refs #6386) +- fix(crux): classify POST /v1/chat/prompt-ids as a not-generation route (Refs #6383) +- fix(roadmap): single-quote the PMAT-6360 criterion that held a colon (Refs #6360) +- fix(examples): replay prints a Usage: line on a bare run (Refs #6360) +- fix(clippy): clippy 1.99 findings, part 1 of the workspace sweep (Refs #5782) +- fix(deps): tokio 1.52.3 -> 1.53.3 for RUSTSEC-2026-0386 +- fix(cbtop): pcie collector reads link width from a fixture-testable root and ignores sentinel widths (Refs #6282) +- fix(guards): --list --shard lists GITHUB_PATH/ENV writers in every shard before the m check, as the run path runs them (Refs #6255) +- fix(ci): guard-cargo shards wait 600 s for ci-target-admit; 3 sibling shards meet the live<2 cap (Refs #6255) +- fix(guards): --list --shard rejects a non-m step id like the run path does (Refs #6255) +- fix(ci): the guard-cargo verdict job checks out scripts/ so ci_guards.sh --verdict exists (Refs #6255) +- fix(guards): state-writer match is an append to GITHUB_PATH/ENV; regenerate roadmap.yaml (#6255) +- fix(ci): skip absent dirs in the post-job chown, fix the stale comment, quiet SEC011 (Refs #6256) +- fix(ci): ownership restores chown to the runner uid and say when they fail (Refs #6256) +- fix(run): --benchmark exits 8 on an empty completion like every other surface (Refs #3720) +- fix(ci): pre-checkout ownership restore chowns to the runner uid (Refs #6235) +- fix(release): the LAB model ladder runs with per-cell admission on (Refs #6211) +- fix(ci): pr-light runs on an X64 clean-room runner, off the gx10 build host (Refs #6239) +- fix(affected-set): rex-001/prereg.lock is a build input, build_inputs reads each include whole, shard table narrowing dropped (Refs #6042) +- fix(train): restore the #5686 ClassifyTrainer::train tests train-loop-v1 cites (Refs #5678) +- fix(ci): carry #5724 test fold onto current main, overlap with #5935 kept as nested mod (Refs #5678) +- fix(simulate): repo-flow binding_station is the argmin of the station bounds (FLOW-007 B1, Refs #6068) +- fix(ci): check_ci_wait_cancel gate rows read the step env EVT, so merge_group rows ran as pull_request (#6061) +- fix(ci): ci_resolve_dirty self-test reads its output by here-string, not a pipe into grep -q (Refs #4780) +- fix(pre-push): checked CRATES selection, one trapped scratch dir, clone-isolation row in --prove (#5983) +- fix(ladder): hung health-wait case asserts the stall event, not stall+3 ticks +- fix(guards): check_ladder_cells_producer reaps its fake serves when SIGKILLed +- fix(resolve_base): name merge_group.base_sha and pull_request.base.sha so a REBASE-replayed PR is judged whole +- fix(ci): silicon coverage re-reads an all-empty listing once and prints request ids (Refs #5978) +- fix(ci): silicon coverage reads the workflow's newest run when listing and probe both come back empty (Refs #5974) +- fix(ci): draft_skip shard-unguarded mutant missed the shard once needs: sat above its if: (main red since #5891) (#5905) +- fix(rollup): aprender-rollup answers --version with the build sha (Refs #5888) (#5890) +- fix(ci): pv_bin.sh keeps exit status and raw output of an empty --version (Refs #5894) +- fix(ci): remove check_harness_server_cleanup.sh + its 817 explicit command; keep #5806's launcher fixes (Refs #5806) +- fix: read pull_request.base.sha past the nested repo/user objects of a real payload (#5859) +- fix(laya-rules): judge merge_group.base_sha..head so a REBASE-replayed multi-commit PR is not exit 2 (#5855) +- fix(harness): crux launchers kill their fixture servers on TERM/INT/KILL (#5806) +- fix(ci): bogus-m-crux fixture hermetic CARGO_TARGET_DIR (PMAT-5817) (#5818) +- fix(deps): ureq 2→3 + webpki bumps — RUSTSEC-2026-0381/0382/0383 (main-red P0) (#5813) +- fix(arm): plain gh pr merge on merge-queue repos, RED scanner on method flags (#5796) +- fix(deps): wasmtime 48.0.7 clears RUSTSEC-2026-0378/0379 (lock only) (#5789) +- fix(contracts): stop writing volatile reasoner_git_sha and cpu_ms to the witness (P5) (#5752) +- fix(data): validate_readme reports unparseable YAML front matter instead of returning Ok (Refs #5500, #5328) (#5513) +- fix(serve): accept absent/null assistant content, max_completion_tokens alias, gate the GGUF router (supersedes #5411) (#5544) +- fix(deps): aprender-viz-ttop build.rs reads YAML through serde-saphyr; drop unused serde_yaml_ng from the present/test workspace shells (Refs #5326) (#5499) +- fix(ci-tools): Serve::signal_group used a kill executable the rust-slim clean-room lacks (#5492) (#5493) +- fix(deps): aprender-decide reads YAML through serde-saphyr, not serde_yaml (Refs #5326) (#5413) +- fix(ci): vendored-schemas mutant harness runs its nine guard runs in parallel (#5481) +- fix(build-sha): stamp apr --version with --short=9, the comparators' length (#5262) +- fix(pr-review): current quorum-lane model ids (gemini-4-argon-high, claude-sonnet-5-5, claude-haiku-5-5) (#5428) +- fix(gate): laya-train SCOPE no longer reads the judge crate's laya-named files as model code (Refs #5448) (#5450) +- fix(guard): dogfood.sh C14 reads its output with here-strings, not printf into grep -q (Refs #5457) (#5458) +- fix(apr-cli): apr ptx --help carries the unavailable-in-this-build marker (Refs #4474) (#5456) +- fix(deps): zip 2.4.2 -> 9.0.4 for RUSTSEC-2026-0367, deflate only (Refs #5459) (#5460) +- fix(deps): port aprender-orchestrate to serde-saphyr (Refs #5209) (#5421) +- fix(guard): check_bashrs_gate.sh feeds grep -q without a pipe (Refs #5374) (#5424) +- fix(guard): check_row_pr_write_set.sh feeds grep -q without a pipe (Refs #5374) (#5422) +- fix(#3431): GgmlType matches name every id instead of swallowing them with _ (Refs #3431) (#5420) +- fix(guard): ci_gpu_touched.sh feeds grep -q without a pipe (Refs #5374) (#5425) +- fix(serve): apr run -v prints every encoded prompt id, not the first 30 (Refs #5056) (#5418) +- fix(rex): S-4 cap in decide() records its reason (FALSIFY-RXG-008) (#5415) +- fix(serve): Ollama shim reports done_reason "length" when num_predict is hit (Refs #5357) (#5414) +- fix(serve): GPU/CUDA routes stop on the EOS plus every EOG marker (Refs #4345) (#5386) +- fix(ci): a kicked merge-queue entry's re-enqueue cancels the dead run on its ref (#5381) (#5385) +- fix(serve): constrained EOS and 499, response_format type refusal, Ollama done_reason, code help (#3568 #5357 #5211) (#5407) +- fix(ci): test tier diffs the PR from its merge-base, not the base tip (#5391) (#5405) +- fix(release): rehearsal guard refuses a cargo config token (Refs #4844) (#5402) +- fix(docs): no README or manifest points at an archived pre-monorepo repo; pv ontology readme gates it (Refs #5382) (#5392) +- fix(ci): install the pinned toolchain before the x86-main sections (#5403) (#5406) +- fix(ci): the merge_group tier step retries its two GitHub lookups, then falls to full (Refs #5399) (#5400) +- fix(finetune): from_pretrained refuses a trained adapter it cannot read (#4698) (#5377) +- fix(deps): aprender-present-terminal reads YAML through serde-saphyr (S3b, Refs #5326) (#5378) +- fix(guard): four self-test rows read a match as a miss under pipefail (Refs #5374) (#5379) +- fix(build): serve reads its arch-requirements and tensor-names contracts in-tree; three dead binding stubs removed (Refs #4380) (#5372) +- fix(contracts): 370 contracts bound to real tests, off the no-test debt list (803 -> 433) (#5291) (#5369) +- fix(deps): 14 small crates read YAML through serde-saphyr, not serde_yaml (Refs #5326) (#5365) +- fix(pv): refuse empty or missing falsification tests; walk kaizen; no-test debt 803 only shrinks, capped per release (Refs #5291) (#5362) +- fix(rehearse): D5 matched a refusal line prepare_bump no longer prints (#5060) (#5354) +- fix(nightly): the lanes rows test pinned rows main no longer has, so the train self-test broke before any verdict (#5079) (#5355) +- fix(deps): wasmtime 48.0.4 -> 48.0.6 for the RUSTSEC-2026-0353..0364 advisories (Refs #4633) (#5361) +- fix(flake): test_stack_unwinding_does_not_crash lists a dir it owns, and says why it failed (Refs #5018) (#5352) +- fix(deps): drop the unused bashrs crate from aprender-compute-xtask and aprender-orchestrate (#5205) (#5350) +- fix(deps): rustls 0.23.46 and drop aws-sdk-s3's legacy rustls 0.21 client (RUSTSEC-2026-0346..0349) (#5343) +- fix(apr-wgpu): the stop token ended the reply and was printed as its id (Refs #4984) (#5269) + +### Changed + +- refactor(simulate): split tsp_instance.rs so each file passes the pre-commit complexity limit (Refs #6025) +- refactor(simulate): split edd/loader.rs so each file passes the pre-commit complexity limit (Refs #6025) +- refactor(simulate): split tsp_engine.rs so each file passes the pre-commit complexity limit (Refs #6025) +- refactor(yaml): aprender-simulate build.rs reads contract YAML through serde-saphyr (S2, Refs #6020) +- refactor(yaml): aprender-contracts-cli reads YAML through serde-saphyr (S4, Refs #5209) (#5507) +- refactor(apr-cli): five hook-refused files under complexity 30/25 (unblocks DIRTY #5262/#5295/#5440/#5447) (#5464) +- refactor(yaml): apr-cli reads YAML through serde-saphyr (S4, Refs #5209) (#5416) +- refactor(yaml): five crates read YAML through serde-saphyr (S2, Refs #5208) (#5363) + +### Other + +- evidence(perf041): refresh the PP-26 witness and marker on lambda at 046ad6a84d +- roadmap: regenerate roadmap.yaml from fragments (main + PMAT-6200 only) (Refs #6200) +- lab: guard against per-function coverage tickets; the scan updates epic #5875 (Refs #5879) +- batch: lane-a-1 coverage fold, regenerate (Refs #5784) +- Revert "ci: drop X64 from arch-neutral runs-on, pin the FULL tier's shards to X64 ... (#5891)" (#5907) +- tooling(arm): arm_pr_automerge.sh refuses a PR whose base is not main (#5824) +- batch 6: draw_scrollbar + render_disk_box tests (Refs #5623, #5603) (#5777) +- batch 5: StepProfiler split + /health binary_sha256 (Refs #5577, #5348) (#5776) +- batch 4: SIM-4 capacity-bound check (Refs #5511) (#5774) +- batch(manual-1615): fold 6 ready branches (#5735 #5592 #5586 #5657 #5672 #5762) (#5765) +- batch(manual-b2): fold 3 ready branches (#5621 #5625 #5624) (#5766) +- test+refactor: fold cbtop bottleneck (#5548) and process_tracer next cog (#5568) (#5709) +- evidence(perf041): Qwen3.5-4B batched-serving witnesses, c 1/4/8/16 (Refs #5312) (#5495) +- ci-tools: nextest-fail-fast, the nextest guard's TOML judge in Rust (Refs #5032) (#5370) + +Also in this range: 112 ci, test, chore and docs commits (not listed). + + ## [0.71.3] - 2026-10-14 A patch release on 0.71.2: `apr embed` loads EmbeddingGemma 2 from GGUF, the chat route returns logprobs, CRUX gains category R (SHACL ontologies against Apache Jena), the regression tree's split search sorts once instead of rescanning per candidate, the release model run binds its binary by sha prefix under one run lock, memory headroom scales with the device instead of assuming 16 GiB, and the shrink-only ratchets, the weekly mutants run and a new edge-device fast-verbs run all live on nightly LAB lanes, off the PR path. diff --git a/Cargo.lock b/Cargo.lock index b54c7a2e8d..873f18af94 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -247,7 +247,7 @@ dependencies = [ [[package]] name = "apr-cli" -version = "0.71.3" +version = "0.72.0" dependencies = [ "anyhow", "aprender-build-sha", @@ -325,7 +325,7 @@ dependencies = [ [[package]] name = "apr-format" -version = "0.71.3" +version = "0.72.0" dependencies = [ "bincode", "cargo_metadata", @@ -343,7 +343,7 @@ dependencies = [ [[package]] name = "aprender" -version = "0.71.3" +version = "0.72.0" dependencies = [ "apr-cli", "aprender-core", @@ -351,7 +351,7 @@ dependencies = [ [[package]] name = "aprender-bench-compute" -version = "0.71.3" +version = "0.72.0" dependencies = [ "aprender-core", "criterion 0.7.0", @@ -360,7 +360,7 @@ dependencies = [ [[package]] name = "aprender-bench-tokenizer" -version = "0.71.3" +version = "0.72.0" dependencies = [ "aprender-core", "criterion 0.7.0", @@ -369,14 +369,14 @@ dependencies = [ [[package]] name = "aprender-build-sha" -version = "0.71.3" +version = "0.72.0" dependencies = [ "tempfile", ] [[package]] name = "aprender-cbtop" -version = "0.71.3" +version = "0.72.0" dependencies = [ "anyhow", "aprender-build-sha", @@ -402,7 +402,7 @@ dependencies = [ [[package]] name = "aprender-cgp" -version = "0.71.3" +version = "0.72.0" dependencies = [ "anyhow", "aprender-build-sha", @@ -423,7 +423,7 @@ dependencies = [ [[package]] name = "aprender-ci-tools" -version = "0.71.3" +version = "0.72.0" dependencies = [ "aprender-build-sha", "aprender-rollup", @@ -439,7 +439,7 @@ dependencies = [ [[package]] name = "aprender-common" -version = "0.71.3" +version = "0.72.0" dependencies = [ "clap", "lz4_flex 0.11.6", @@ -448,7 +448,7 @@ dependencies = [ [[package]] name = "aprender-compute" -version = "0.71.3" +version = "0.72.0" dependencies = [ "anyhow", "aprender-contracts", @@ -498,7 +498,7 @@ dependencies = [ [[package]] name = "aprender-compute-xtask" -version = "0.71.3" +version = "0.72.0" dependencies = [ "anyhow", "aprender-build-sha", @@ -512,7 +512,7 @@ dependencies = [ [[package]] name = "aprender-contracts" -version = "0.71.3" +version = "0.72.0" dependencies = [ "aprender-contracts-macros", "blake3", @@ -532,7 +532,7 @@ dependencies = [ [[package]] name = "aprender-contracts-cli" -version = "0.71.3" +version = "0.72.0" dependencies = [ "aprender-build-sha", "aprender-common", @@ -548,7 +548,7 @@ dependencies = [ [[package]] name = "aprender-contracts-macros" -version = "0.71.3" +version = "0.72.0" dependencies = [ "proc-macro2", "quote", @@ -558,7 +558,7 @@ dependencies = [ [[package]] name = "aprender-contrastive-data" -version = "0.71.3" +version = "0.72.0" dependencies = [ "aprender-contracts-macros", "aprender-rand", @@ -573,7 +573,7 @@ dependencies = [ [[package]] name = "aprender-core" -version = "0.71.3" +version = "0.72.0" dependencies = [ "aes-gcm", "alsa", @@ -627,7 +627,7 @@ dependencies = [ [[package]] name = "aprender-cuda-edge" -version = "0.71.3" +version = "0.72.0" dependencies = [ "aprender-gpu", "proptest", @@ -638,7 +638,7 @@ dependencies = [ [[package]] name = "aprender-cupti" -version = "0.71.3" +version = "0.72.0" dependencies = [ "bindgen", "bitflags 2.13.0", @@ -648,7 +648,7 @@ dependencies = [ [[package]] name = "aprender-data" -version = "0.71.3" +version = "0.72.0" dependencies = [ "aes-gcm", "aprender-build-sha", @@ -701,7 +701,7 @@ dependencies = [ [[package]] name = "aprender-db" -version = "0.71.3" +version = "0.72.0" dependencies = [ "anyhow", "aprender-build-sha", @@ -747,7 +747,7 @@ dependencies = [ [[package]] name = "aprender-decide" -version = "0.71.3" +version = "0.72.0" dependencies = [ "aprender-contracts-macros", "aprender-core", @@ -766,7 +766,7 @@ dependencies = [ [[package]] name = "aprender-distribute" -version = "0.71.3" +version = "0.72.0" dependencies = [ "aprender-compute", "aprender-db", @@ -796,7 +796,7 @@ dependencies = [ [[package]] name = "aprender-explain" -version = "0.71.3" +version = "0.72.0" dependencies = [ "aprender-build-sha", "aprender-gpu", @@ -815,7 +815,7 @@ dependencies = [ [[package]] name = "aprender-fft" -version = "0.71.3" +version = "0.72.0" dependencies = [ "criterion 0.7.0", "proptest", @@ -824,7 +824,7 @@ dependencies = [ [[package]] name = "aprender-gemm-codegen" -version = "0.71.3" +version = "0.72.0" dependencies = [ "proc-macro2", "quote", @@ -833,7 +833,7 @@ dependencies = [ [[package]] name = "aprender-gpu" -version = "0.71.3" +version = "0.72.0" dependencies = [ "aprender-common", "aprender-simulate", @@ -851,7 +851,7 @@ dependencies = [ [[package]] name = "aprender-graph" -version = "0.71.3" +version = "0.72.0" dependencies = [ "anyhow", "aprender-compute", @@ -872,7 +872,7 @@ dependencies = [ [[package]] name = "aprender-image" -version = "0.71.3" +version = "0.72.0" dependencies = [ "criterion 0.7.0", "proptest", @@ -881,7 +881,7 @@ dependencies = [ [[package]] name = "aprender-mcp" -version = "0.71.3" +version = "0.72.0" dependencies = [ "inventory", "jsonschema", @@ -894,7 +894,7 @@ dependencies = [ [[package]] name = "aprender-monte-carlo" -version = "0.71.3" +version = "0.72.0" dependencies = [ "aprender", "assert_cmd", @@ -913,7 +913,7 @@ dependencies = [ [[package]] name = "aprender-orchestrate" -version = "0.71.3" +version = "0.72.0" dependencies = [ "anyhow", "aprender-build-sha", @@ -983,7 +983,7 @@ dependencies = [ [[package]] name = "aprender-present-cli" -version = "0.71.3" +version = "0.72.0" dependencies = [ "aprender-build-sha", "aprender-contracts", @@ -998,7 +998,7 @@ dependencies = [ [[package]] name = "aprender-present-core" -version = "0.71.3" +version = "0.72.0" dependencies = [ "aprender-contracts", "aprender-contracts-macros", @@ -1011,7 +1011,7 @@ dependencies = [ [[package]] name = "aprender-present-layout" -version = "0.71.3" +version = "0.72.0" dependencies = [ "aprender-present-core", "criterion 0.7.0", @@ -1022,7 +1022,7 @@ dependencies = [ [[package]] name = "aprender-present-lib" -version = "0.71.3" +version = "0.72.0" dependencies = [ "aprender-contracts", "aprender-present-core", @@ -1051,7 +1051,7 @@ dependencies = [ [[package]] name = "aprender-present-terminal" -version = "0.71.3" +version = "0.72.0" dependencies = [ "aprender-build-sha", "aprender-contracts", @@ -1073,7 +1073,7 @@ dependencies = [ [[package]] name = "aprender-present-test" -version = "0.71.3" +version = "0.72.0" dependencies = [ "aprender-present-core", "aprender-present-test-macros", @@ -1084,7 +1084,7 @@ dependencies = [ [[package]] name = "aprender-present-test-macros" -version = "0.71.3" +version = "0.72.0" dependencies = [ "proc-macro2", "quote", @@ -1093,7 +1093,7 @@ dependencies = [ [[package]] name = "aprender-present-widgets" -version = "0.71.3" +version = "0.72.0" dependencies = [ "aprender-present-core", "aprender-present-yaml", @@ -1105,7 +1105,7 @@ dependencies = [ [[package]] name = "aprender-present-yaml" -version = "0.71.3" +version = "0.72.0" dependencies = [ "aprender-present-core", "proptest", @@ -1116,7 +1116,7 @@ dependencies = [ [[package]] name = "aprender-profile" -version = "0.71.3" +version = "0.72.0" dependencies = [ "addr2line 0.25.1", "anyhow", @@ -1169,7 +1169,7 @@ dependencies = [ [[package]] name = "aprender-profile-core" -version = "0.71.3" +version = "0.72.0" dependencies = [ "hex", "serde", @@ -1179,7 +1179,7 @@ dependencies = [ [[package]] name = "aprender-ptx-debug" -version = "0.71.3" +version = "0.72.0" dependencies = [ "aprender-build-sha", "clap", @@ -1189,7 +1189,7 @@ dependencies = [ [[package]] name = "aprender-qa-certify" -version = "0.71.3" +version = "0.72.0" dependencies = [ "aprender-build-sha", "chrono", @@ -1201,7 +1201,7 @@ dependencies = [ [[package]] name = "aprender-qa-cli" -version = "0.71.3" +version = "0.72.0" dependencies = [ "aprender-build-sha", "aprender-qa-certify", @@ -1221,7 +1221,7 @@ dependencies = [ [[package]] name = "aprender-qa-gen" -version = "0.71.3" +version = "0.72.0" dependencies = [ "anyhow", "chrono", @@ -1238,7 +1238,7 @@ dependencies = [ [[package]] name = "aprender-qa-report" -version = "0.71.3" +version = "0.72.0" dependencies = [ "anyhow", "aprender-qa-gen", @@ -1256,7 +1256,7 @@ dependencies = [ [[package]] name = "aprender-qa-runner" -version = "0.71.3" +version = "0.72.0" dependencies = [ "anyhow", "aprender-qa-gen", @@ -1281,7 +1281,7 @@ dependencies = [ [[package]] name = "aprender-quant" -version = "0.71.3" +version = "0.72.0" dependencies = [ "half", "serde_json", @@ -1289,7 +1289,7 @@ dependencies = [ [[package]] name = "aprender-rag" -version = "0.71.3" +version = "0.72.0" dependencies = [ "aprender-common", "aprender-compute", @@ -1318,7 +1318,7 @@ dependencies = [ [[package]] name = "aprender-rag-cli" -version = "0.71.3" +version = "0.72.0" dependencies = [ "anyhow", "aprender-build-sha", @@ -1338,7 +1338,7 @@ dependencies = [ [[package]] name = "aprender-rand" -version = "0.71.3" +version = "0.72.0" dependencies = [ "criterion 0.7.0", "proptest", @@ -1347,7 +1347,7 @@ dependencies = [ [[package]] name = "aprender-registry" -version = "0.71.3" +version = "0.72.0" dependencies = [ "anyhow", "aprender-core", @@ -1377,7 +1377,7 @@ dependencies = [ [[package]] name = "aprender-review-experiment" -version = "0.71.3" +version = "0.72.0" dependencies = [ "half", "regex", @@ -1391,14 +1391,14 @@ dependencies = [ [[package]] name = "aprender-rollup" -version = "0.71.3" +version = "0.72.0" dependencies = [ "serde_json", ] [[package]] name = "aprender-serve" -version = "0.71.3" +version = "0.72.0" dependencies = [ "anyhow", "approx", @@ -1470,7 +1470,7 @@ dependencies = [ [[package]] name = "aprender-shell" -version = "0.71.3" +version = "0.72.0" dependencies = [ "aprender-core", "clap", @@ -1485,7 +1485,7 @@ dependencies = [ [[package]] name = "aprender-simulate" -version = "0.71.3" +version = "0.72.0" dependencies = [ "aprender-build-sha", "aprender-contracts", @@ -1528,7 +1528,7 @@ dependencies = [ [[package]] name = "aprender-solve" -version = "0.71.3" +version = "0.72.0" dependencies = [ "criterion 0.7.0", "half", @@ -1538,7 +1538,7 @@ dependencies = [ [[package]] name = "aprender-sparse" -version = "0.71.3" +version = "0.72.0" dependencies = [ "criterion 0.7.0", "proptest", @@ -1547,7 +1547,7 @@ dependencies = [ [[package]] name = "aprender-tensor" -version = "0.71.3" +version = "0.72.0" dependencies = [ "criterion 0.7.0", "proptest", @@ -1556,7 +1556,7 @@ dependencies = [ [[package]] name = "aprender-test-cli" -version = "0.71.3" +version = "0.72.0" dependencies = [ "aprender-build-sha", "aprender-test-lib", @@ -1585,7 +1585,7 @@ dependencies = [ [[package]] name = "aprender-test-derive" -version = "0.71.3" +version = "0.72.0" dependencies = [ "proc-macro2", "quote", @@ -1595,7 +1595,7 @@ dependencies = [ [[package]] name = "aprender-test-js-gen" -version = "0.71.3" +version = "0.72.0" dependencies = [ "blake3", "chrono", @@ -1611,7 +1611,7 @@ dependencies = [ [[package]] name = "aprender-test-lib" -version = "0.71.3" +version = "0.72.0" dependencies = [ "aprender-compute", "aprender-present-core", @@ -1657,7 +1657,7 @@ dependencies = [ [[package]] name = "aprender-test-showcase" -version = "0.71.3" +version = "0.72.0" dependencies = [ "aprender-present-terminal", "aprender-test-lib", @@ -1674,7 +1674,7 @@ dependencies = [ [[package]] name = "aprender-train" -version = "0.71.3" +version = "0.72.0" dependencies = [ "approx", "aprender-common", @@ -1743,7 +1743,7 @@ dependencies = [ [[package]] name = "aprender-train-bench" -version = "0.71.3" +version = "0.72.0" dependencies = [ "aprender-build-sha", "aprender-train", @@ -1761,7 +1761,7 @@ dependencies = [ [[package]] name = "aprender-train-common" -version = "0.71.3" +version = "0.72.0" dependencies = [ "aprender-viz", "clap", @@ -1773,7 +1773,7 @@ dependencies = [ [[package]] name = "aprender-train-distill" -version = "0.71.3" +version = "0.72.0" dependencies = [ "aprender-build-sha", "aprender-train", @@ -1795,7 +1795,7 @@ dependencies = [ [[package]] name = "aprender-train-inspect" -version = "0.71.3" +version = "0.72.0" dependencies = [ "aprender-build-sha", "aprender-train", @@ -1812,7 +1812,7 @@ dependencies = [ [[package]] name = "aprender-train-lora" -version = "0.71.3" +version = "0.72.0" dependencies = [ "aprender-build-sha", "aprender-train", @@ -1829,7 +1829,7 @@ dependencies = [ [[package]] name = "aprender-train-shell" -version = "0.71.3" +version = "0.72.0" dependencies = [ "aprender-build-sha", "aprender-train", @@ -1846,7 +1846,7 @@ dependencies = [ [[package]] name = "aprender-train-wasm" -version = "0.71.3" +version = "0.72.0" dependencies = [ "proptest", "serde", @@ -1857,7 +1857,7 @@ dependencies = [ [[package]] name = "aprender-tsp" -version = "0.71.3" +version = "0.72.0" dependencies = [ "aprender", "assert_cmd", @@ -1872,7 +1872,7 @@ dependencies = [ [[package]] name = "aprender-update" -version = "0.71.3" +version = "0.72.0" dependencies = [ "flate2", "semver", @@ -1886,7 +1886,7 @@ dependencies = [ [[package]] name = "aprender-verify" -version = "0.71.3" +version = "0.72.0" dependencies = [ "chrono", "criterion 0.5.1", @@ -1898,7 +1898,7 @@ dependencies = [ [[package]] name = "aprender-verify-ml" -version = "0.71.3" +version = "0.72.0" dependencies = [ "aprender-build-sha", "aprender-compute", @@ -1929,7 +1929,7 @@ dependencies = [ [[package]] name = "aprender-viz" -version = "0.71.3" +version = "0.72.0" dependencies = [ "approx", "aprender-common", @@ -1958,7 +1958,7 @@ dependencies = [ [[package]] name = "aprender-zram" -version = "0.71.3" +version = "0.72.0" dependencies = [ "aprender-zram-adaptive", "aprender-zram-core", @@ -1968,7 +1968,7 @@ dependencies = [ [[package]] name = "aprender-zram-adaptive" -version = "0.71.3" +version = "0.72.0" dependencies = [ "aprender-zram-core", "proptest", @@ -1977,7 +1977,7 @@ dependencies = [ [[package]] name = "aprender-zram-cli" -version = "0.71.3" +version = "0.72.0" dependencies = [ "aprender-build-sha", "aprender-zram-adaptive", @@ -1992,7 +1992,7 @@ dependencies = [ [[package]] name = "aprender-zram-core" -version = "0.71.3" +version = "0.72.0" dependencies = [ "aprender-gpu", "criterion 0.7.0", @@ -2004,7 +2004,7 @@ dependencies = [ [[package]] name = "aprender-zram-generator" -version = "0.71.3" +version = "0.72.0" dependencies = [ "aprender-build-sha", "aprender-zram-core", diff --git a/Cargo.toml b/Cargo.toml index 71b08df5c4..7d4dad3b4d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -163,7 +163,7 @@ exclude = [ resolver = "2" [workspace.package] -version = "0.71.3" +version = "0.72.0" edition = "2021" license = "MIT" repository = "https://github.com/paiml/aprender" @@ -251,11 +251,11 @@ tokenizers = { version = "0.23.1", default-features = false, features = ["fancy- # aprender-rand (lib name `trueno_rand`) is a workspace MEMBER but had no # [workspace.dependencies] entry, so every consumer had to re-declare the path. # It is the counter-based Philox RNG behind the SetFit encoder's dropout masks. -aprender-rand = { path = "crates/aprender-rand", version = "0.71.3" } +aprender-rand = { path = "crates/aprender-rand", version = "0.72.0" } # aprender-core (lib name `aprender`): a consumer that hand-pins it falls behind the # workspace version, and the rc stamp then cannot resolve it (aprender-decide pinned # 0.70.2 under a 0.70.3 workspace). Inherit this entry instead. -aprender-core = { path = "crates/aprender-core", version = "0.71.3" } +aprender-core = { path = "crates/aprender-core", version = "0.72.0" } # Chat templates minijinja = { version = "2.14", features = ["loader", "serde"] } @@ -265,9 +265,9 @@ minijinja = { version = "2.14", features = ["loader", "serde"] } # `crates/aprender-contracts` (lib name `provable_contracts`). Declaring it from # crates.io compiled a SECOND copy of the contract engine alongside the in-tree # one — exactly the duplicate-crate + publish-cycle the consolidation removed. -provable-contracts = { path = "crates/aprender-contracts", version = "0.71.3", package = "aprender-contracts" } -provable-contracts-macros = { path = "crates/aprender-contracts-macros", version = "0.71.3", package = "aprender-contracts-macros" } -aprender-contracts-macros = { path = "crates/aprender-contracts-macros", version = "0.71.3" } +provable-contracts = { path = "crates/aprender-contracts", version = "0.72.0", package = "aprender-contracts" } +provable-contracts-macros = { path = "crates/aprender-contracts-macros", version = "0.72.0", package = "aprender-contracts-macros" } +aprender-contracts-macros = { path = "crates/aprender-contracts-macros", version = "0.72.0" } # YAML serde_yaml = "0.9" @@ -281,70 +281,70 @@ hf-hub = { version = "0.5", default-features = false, features = ["ureq"] } hf-xet = "1.5.1" # Workspace-internal crates (paths within the monorepo) -aprender-compute = { path = "crates/aprender-compute", version = "0.71.3" } -aprender-gpu = { path = "crates/aprender-gpu", version = "0.71.3" } -aprender-quant = { path = "crates/aprender-quant", version = "0.71.3" } -aprender-serve = { path = "crates/aprender-serve", version = "0.71.3" } -realizar = { path = "crates/aprender-serve", version = "0.71.3", package = "aprender-serve" } -alimentar = { path = "crates/aprender-data", version = "0.71.3", package = "aprender-data" } -pacha = { path = "crates/aprender-registry", version = "0.71.3", package = "aprender-registry" } +aprender-compute = { path = "crates/aprender-compute", version = "0.72.0" } +aprender-gpu = { path = "crates/aprender-gpu", version = "0.72.0" } +aprender-quant = { path = "crates/aprender-quant", version = "0.72.0" } +aprender-serve = { path = "crates/aprender-serve", version = "0.72.0" } +realizar = { path = "crates/aprender-serve", version = "0.72.0", package = "aprender-serve" } +alimentar = { path = "crates/aprender-data", version = "0.72.0", package = "aprender-data" } +pacha = { path = "crates/aprender-registry", version = "0.72.0", package = "aprender-registry" } # APR-MONO self-containment: legacy sibling lib-name aliases -> in-monorepo crates (2026-06-12) -trueno-quant = { path = "crates/aprender-quant", version = "0.71.3", package = "aprender-quant" } -trueno-db = { path = "crates/aprender-db", version = "0.71.3", package = "aprender-db" } -trueno-viz = { path = "crates/aprender-viz", version = "0.71.3", package = "aprender-viz" } -trueno-rag = { path = "crates/aprender-rag", version = "0.71.3", package = "aprender-rag" } -trueno-cuda-edge = { path = "crates/aprender-cuda-edge", version = "0.71.3", package = "aprender-cuda-edge" } -trueno-sparse = { path = "crates/aprender-sparse", version = "0.71.3", package = "aprender-sparse" } -trueno-solve = { path = "crates/aprender-solve", version = "0.71.3", package = "aprender-solve" } -renacer-core = { path = "crates/aprender-profile-core", version = "0.71.3", package = "aprender-profile-core" } -repartir = { path = "crates/aprender-distribute", version = "0.71.3", package = "aprender-distribute" } -simular = { path = "crates/aprender-simulate", version = "0.71.3", package = "aprender-simulate" } -probar = { path = "crates/aprender-test-lib", version = "0.71.3", package = "aprender-test-lib" } +trueno-quant = { path = "crates/aprender-quant", version = "0.72.0", package = "aprender-quant" } +trueno-db = { path = "crates/aprender-db", version = "0.72.0", package = "aprender-db" } +trueno-viz = { path = "crates/aprender-viz", version = "0.72.0", package = "aprender-viz" } +trueno-rag = { path = "crates/aprender-rag", version = "0.72.0", package = "aprender-rag" } +trueno-cuda-edge = { path = "crates/aprender-cuda-edge", version = "0.72.0", package = "aprender-cuda-edge" } +trueno-sparse = { path = "crates/aprender-sparse", version = "0.72.0", package = "aprender-sparse" } +trueno-solve = { path = "crates/aprender-solve", version = "0.72.0", package = "aprender-solve" } +renacer-core = { path = "crates/aprender-profile-core", version = "0.72.0", package = "aprender-profile-core" } +repartir = { path = "crates/aprender-distribute", version = "0.72.0", package = "aprender-distribute" } +simular = { path = "crates/aprender-simulate", version = "0.72.0", package = "aprender-simulate" } +probar = { path = "crates/aprender-test-lib", version = "0.72.0", package = "aprender-test-lib" } # English key for the same crate, named after the command it serves (`apr test`). # The dep KEY, not the package name, is what becomes the extern crate name when # the two differ — the lib target is still `jugar_probar`, so keying it by its # package name would have imported Spanish anyway. Renaming that lib target is # 196 call sites in 17 crates and gets its own PR; this key is how new code # reads English today. -apr-test = { path = "crates/aprender-test-lib", version = "0.71.3", package = "aprender-test-lib", default-features = false, features = ["llm"] } +apr-test = { path = "crates/aprender-test-lib", version = "0.72.0", package = "aprender-test-lib", default-features = false, features = ["llm"] } # `jugar-probar` is the crates.io/lib name of the same in-tree crate as `probar` # above. Both aliases resolve to ONE package; consumers that spelled it # `jugar-probar = "0.4"|"0.5"|"1.0"` were building three extra copies. -jugar-probar = { path = "crates/aprender-test-lib", version = "0.71.3", package = "aprender-test-lib" } +jugar-probar = { path = "crates/aprender-test-lib", version = "0.72.0", package = "aprender-test-lib" } # NOTE: the matching `jugar-probar-derive` alias already exists further down this # same table (it points at crates/aprender-test-derive). crates/aprender-test-lib # was pinning `jugar-probar-derive = "=1.0.3"` from crates.io regardless — the # alias was sitting there unused. -aprender-train = { path = "crates/aprender-train", version = "0.71.3" } -entrenar = { path = "crates/aprender-train", version = "0.71.3", package = "aprender-train" } -aprender-train-common = { path = "crates/aprender-train-common", version = "0.71.3" } -aprender-train-distill = { path = "crates/aprender-train-distill", version = "0.71.3" } -aprender-orchestrate = { path = "crates/aprender-orchestrate", version = "0.71.3" } -aprender-contracts = { path = "crates/aprender-contracts", version = "0.71.3" } -aprender-build-sha = { path = "crates/aprender-build-sha", version = "0.71.3" } -aprender-rollup = { path = "crates/aprender-rollup", version = "0.71.3" } -aprender-profile = { path = "crates/aprender-profile", version = "0.71.3" } -aprender-profile-core = { path = "crates/aprender-profile-core", version = "0.71.3" } -aprender-zram-core = { path = "crates/aprender-zram-core", version = "0.71.3" } -aprender-zram-adaptive = { path = "crates/aprender-zram-adaptive", version = "0.71.3" } -aprender-zram-cli = { path = "crates/aprender-zram-cli", version = "0.71.3" } -aprender-present-core = { path = "crates/aprender-present-core", version = "0.71.3" } -aprender-present-terminal = { path = "crates/aprender-present-terminal", version = "0.71.3" } -aprender-present-widgets = { path = "crates/aprender-present-widgets", version = "0.71.3" } -aprender-present-layout = { path = "crates/aprender-present-layout", version = "0.71.3" } -aprender-present-yaml = { path = "crates/aprender-present-yaml", version = "0.71.3" } -aprender-present-test = { path = "crates/aprender-present-test", version = "0.71.3" } -aprender-db = { path = "crates/aprender-db", version = "0.71.3" } -aprender-graph = { path = "crates/aprender-graph", version = "0.71.3" } -aprender-rag = { path = "crates/aprender-rag", version = "0.71.3" } -aprender-rag-cli = { path = "crates/aprender-rag-cli", version = "0.71.3" } -aprender-qa-cli = { path = "crates/aprender-qa-cli", version = "0.71.3" } -aprender-cgp = { path = "crates/aprender-cgp", version = "0.71.3" } -aprender-contrastive-data = { path = "crates/aprender-contrastive-data", version = "0.71.3" } -aprender-contracts-cli = { path = "crates/aprender-contracts-cli", version = "0.71.3" } -aprender-update = { path = "crates/aprender-update", version = "0.71.3" } -aprender-data = { path = "crates/aprender-data", version = "0.71.3" } -aprender-ptx-debug = { path = "crates/aprender-ptx-debug", version = "0.71.3" } +aprender-train = { path = "crates/aprender-train", version = "0.72.0" } +entrenar = { path = "crates/aprender-train", version = "0.72.0", package = "aprender-train" } +aprender-train-common = { path = "crates/aprender-train-common", version = "0.72.0" } +aprender-train-distill = { path = "crates/aprender-train-distill", version = "0.72.0" } +aprender-orchestrate = { path = "crates/aprender-orchestrate", version = "0.72.0" } +aprender-contracts = { path = "crates/aprender-contracts", version = "0.72.0" } +aprender-build-sha = { path = "crates/aprender-build-sha", version = "0.72.0" } +aprender-rollup = { path = "crates/aprender-rollup", version = "0.72.0" } +aprender-profile = { path = "crates/aprender-profile", version = "0.72.0" } +aprender-profile-core = { path = "crates/aprender-profile-core", version = "0.72.0" } +aprender-zram-core = { path = "crates/aprender-zram-core", version = "0.72.0" } +aprender-zram-adaptive = { path = "crates/aprender-zram-adaptive", version = "0.72.0" } +aprender-zram-cli = { path = "crates/aprender-zram-cli", version = "0.72.0" } +aprender-present-core = { path = "crates/aprender-present-core", version = "0.72.0" } +aprender-present-terminal = { path = "crates/aprender-present-terminal", version = "0.72.0" } +aprender-present-widgets = { path = "crates/aprender-present-widgets", version = "0.72.0" } +aprender-present-layout = { path = "crates/aprender-present-layout", version = "0.72.0" } +aprender-present-yaml = { path = "crates/aprender-present-yaml", version = "0.72.0" } +aprender-present-test = { path = "crates/aprender-present-test", version = "0.72.0" } +aprender-db = { path = "crates/aprender-db", version = "0.72.0" } +aprender-graph = { path = "crates/aprender-graph", version = "0.72.0" } +aprender-rag = { path = "crates/aprender-rag", version = "0.72.0" } +aprender-rag-cli = { path = "crates/aprender-rag-cli", version = "0.72.0" } +aprender-qa-cli = { path = "crates/aprender-qa-cli", version = "0.72.0" } +aprender-cgp = { path = "crates/aprender-cgp", version = "0.72.0" } +aprender-contrastive-data = { path = "crates/aprender-contrastive-data", version = "0.72.0" } +aprender-contracts-cli = { path = "crates/aprender-contracts-cli", version = "0.72.0" } +aprender-update = { path = "crates/aprender-update", version = "0.72.0" } +aprender-data = { path = "crates/aprender-data", version = "0.72.0" } +aprender-ptx-debug = { path = "crates/aprender-ptx-debug", version = "0.72.0" } # Additional external deps needed by sub-crates serde_yaml_ng = "0.10" @@ -403,25 +403,25 @@ wasmtime = "48.0.3" bollard = "0.17" # Old crate name aliases (workspace = true refs in migrated sub-crates) -trueno-zram-core = { path = "crates/aprender-zram-core", version = "0.71.3", package = "aprender-zram-core" } -trueno-zram-adaptive = { path = "crates/aprender-zram-adaptive", version = "0.71.3", package = "aprender-zram-adaptive" } -trueno = { path = "crates/aprender-compute", version = "0.71.3", package = "aprender-compute" } -presentar-core = { path = "crates/aprender-present-core", version = "0.71.3", package = "aprender-present-core" } -presentar-terminal = { path = "crates/aprender-present-terminal", version = "0.71.3", package = "aprender-present-terminal" } -presentar-widgets = { path = "crates/aprender-present-widgets", version = "0.71.3", package = "aprender-present-widgets" } -presentar-layout = { path = "crates/aprender-present-layout", version = "0.71.3", package = "aprender-present-layout" } -presentar-yaml = { path = "crates/aprender-present-yaml", version = "0.71.3", package = "aprender-present-yaml" } -presentar-test = { path = "crates/aprender-present-test", version = "0.71.3", package = "aprender-present-test" } -presentar-test-macros = { path = "crates/aprender-present-test-macros", version = "0.71.3", package = "aprender-present-test-macros" } -jugar-probar-derive = { path = "crates/aprender-test-derive", version = "0.71.3", package = "aprender-test-derive" } -batuta-common = { path = "crates/aprender-common", version = "0.71.3", package = "aprender-common" } +trueno-zram-core = { path = "crates/aprender-zram-core", version = "0.72.0", package = "aprender-zram-core" } +trueno-zram-adaptive = { path = "crates/aprender-zram-adaptive", version = "0.72.0", package = "aprender-zram-adaptive" } +trueno = { path = "crates/aprender-compute", version = "0.72.0", package = "aprender-compute" } +presentar-core = { path = "crates/aprender-present-core", version = "0.72.0", package = "aprender-present-core" } +presentar-terminal = { path = "crates/aprender-present-terminal", version = "0.72.0", package = "aprender-present-terminal" } +presentar-widgets = { path = "crates/aprender-present-widgets", version = "0.72.0", package = "aprender-present-widgets" } +presentar-layout = { path = "crates/aprender-present-layout", version = "0.72.0", package = "aprender-present-layout" } +presentar-yaml = { path = "crates/aprender-present-yaml", version = "0.72.0", package = "aprender-present-yaml" } +presentar-test = { path = "crates/aprender-present-test", version = "0.72.0", package = "aprender-present-test" } +presentar-test-macros = { path = "crates/aprender-present-test-macros", version = "0.72.0", package = "aprender-present-test-macros" } +jugar-probar-derive = { path = "crates/aprender-test-derive", version = "0.72.0", package = "aprender-test-derive" } +batuta-common = { path = "crates/aprender-common", version = "0.72.0", package = "aprender-common" } # APR-MONO sibling aliases — MUST use workspace refs, NEVER crates.io versions. # Hard-pinning these to crates.io creates a diamond: sub-crates pull older # aprender versions → multiple `aprender` crates in Cargo.lock = MUDA. -renacer = { path = "crates/aprender-profile", version = "0.71.3", package = "aprender-profile" } -entrenar-lora = { path = "crates/aprender-train-lora", version = "0.71.3", package = "aprender-train-lora" } -batuta = { path = "crates/aprender-orchestrate", version = "0.71.3", package = "aprender-orchestrate" } -trueno-graph = { path = "crates/aprender-graph", version = "0.71.3", package = "aprender-graph" } +renacer = { path = "crates/aprender-profile", version = "0.72.0", package = "aprender-profile" } +entrenar-lora = { path = "crates/aprender-train-lora", version = "0.72.0", package = "aprender-train-lora" } +batuta = { path = "crates/aprender-orchestrate", version = "0.72.0", package = "aprender-orchestrate" } +trueno-graph = { path = "crates/aprender-graph", version = "0.72.0", package = "aprender-graph" } [workspace.lints.rust] # Safety @@ -603,7 +603,7 @@ semicolon_if_nothing_returned = "allow" # Style preference [package] name = "aprender" -version = "0.71.3" +version = "0.72.0" edition = "2021" rust-version = "1.91" authors = ["Noah Gift "] @@ -767,9 +767,9 @@ required-features = ["cli"] # (e.g. apr-cookbook, downstream apps) don't pull apr-cli's heavy transitive # dep graph (renacer/profile/inference/zram/visualization). `cargo install # aprender` still works because `cli` is in default features. See GH-1599. -apr-cli = { path = "crates/apr-cli", version = "0.71.3", default-features = true, optional = true } +apr-cli = { path = "crates/apr-cli", version = "0.72.0", default-features = true, optional = true } # Core ML library (lib name = "aprender") -aprender_ml = { path = "crates/aprender-core", version = "0.71.3", package = "aprender-core" } +aprender_ml = { path = "crates/aprender-core", version = "0.72.0", package = "aprender-core" } [features] default = ["cli"] diff --git a/crates/apr-cli/Cargo.toml b/crates/apr-cli/Cargo.toml index 7d9fe56c3d..3c628d79e7 100644 --- a/crates/apr-cli/Cargo.toml +++ b/crates/apr-cli/Cargo.toml @@ -130,12 +130,12 @@ libc = "0.2" # performed. A surface and its capability must not be able to drift. apr-test = { workspace = true } # Core aprender library -aprender = { path = "../aprender-core", version = "0.71.3", package = "aprender-core", default-features = true, features = ["format-compression"] } +aprender = { path = "../aprender-core", version = "0.72.0", package = "aprender-core", default-features = true, features = ["format-compression"] } # Provable-contracts engine — `apr beat-run` parses + evaluates BeatBenchmark contracts (PMAT-741) aprender-contracts = { workspace = true } # MCP (Model Context Protocol) server — `apr mcp` subcommand -aprender-mcp = { path = "../aprender-mcp", version = "0.71.3", features = ["apr-tools"] } # apr mcp serves apr's own tools (PMAT-3954) +aprender-mcp = { path = "../aprender-mcp", version = "0.72.0", features = ["apr-tools"] } # apr mcp serves apr's own tools (PMAT-3954) async-stream = "0.3" provable-contracts-macros = { workspace = true } @@ -146,7 +146,7 @@ batuta-common = { workspace = true, features = ["cli-roles"] } # Non-optional since claude-code-parity-apr M150 (outcome-parity Phase 3 # requires apr code in default builds). GH-344: path dep via # .cargo/config.toml.dev-overrides; CI uses checkout+symlink. -batuta = { version = "0.71.3", path = "../aprender-orchestrate", package = "aprender-orchestrate", default-features = false, features = ["agents", "agents-inference", "agents-mcp", "rag"] } +batuta = { version = "0.72.0", path = "../aprender-orchestrate", package = "aprender-orchestrate", default-features = false, features = ["agents", "agents-inference", "agents-mcp", "rag"] } # Inference engine (workspace path dep — enables cuda/gpu feature forwarding) # GH-573: shim (0.9.1) didn't forward cuda feature → 0.7 tok/s instead of 273 tok/s @@ -242,7 +242,7 @@ pacha = { workspace = true } # published-crate dependency cycle the APR-MONO consolidation removed, and # (b) was never actually linked, so every `trueno_explain::` path in # commands/ptx_explain.rs failed to resolve. -trueno-explain = { path = "../aprender-explain", version = "0.71.3", package = "aprender-explain", optional = true } +trueno-explain = { path = "../aprender-explain", version = "0.72.0", package = "aprender-explain", optional = true } # ML tuning: LoRA/QLoRA configuration and memory planning (GH-176, PMAT-184) entrenar-lora = { workspace = true, optional = true } diff --git a/crates/aprender-bench-compute/Cargo.toml b/crates/aprender-bench-compute/Cargo.toml index 909b2f31ff..12804aadc5 100644 --- a/crates/aprender-bench-compute/Cargo.toml +++ b/crates/aprender-bench-compute/Cargo.toml @@ -11,7 +11,7 @@ disallowed_methods = "allow" [dependencies] # The implementation under test -aprender = { path = "../aprender-core", version = "0.71.3", package = "aprender-core", features = ["format-quantize"] } +aprender = { path = "../aprender-core", version = "0.72.0", package = "aprender-core", features = ["format-quantize"] } # Reference implementation for A/B comparison # Pure Rust (no BLAS) to keep comparison fair — both are pure Rust SIMD diff --git a/crates/aprender-bench-tokenizer/Cargo.toml b/crates/aprender-bench-tokenizer/Cargo.toml index 46a5933d76..75908a5747 100644 --- a/crates/aprender-bench-tokenizer/Cargo.toml +++ b/crates/aprender-bench-tokenizer/Cargo.toml @@ -11,7 +11,7 @@ disallowed_methods = "allow" [dependencies] # The implementation under test -aprender = { path = "../aprender-core", version = "0.71.3", package = "aprender-core" } +aprender = { path = "../aprender-core", version = "0.72.0", package = "aprender-core" } # HuggingFace reference for A/B comparison # Regular dep (not dev-dep) because [[bench]] harness=false binaries can't see dev-deps diff --git a/crates/aprender-cbtop/Cargo.toml b/crates/aprender-cbtop/Cargo.toml index 6dcaa8aa31..bfab703b3c 100644 --- a/crates/aprender-cbtop/Cargo.toml +++ b/crates/aprender-cbtop/Cargo.toml @@ -32,7 +32,7 @@ presentar-terminal = { workspace = true } # Compute backends trueno = { workspace = true } -trueno-gpu = { version = "0.71.3", path = "../aprender-gpu", package = "aprender-gpu", optional = true } +trueno-gpu = { version = "0.72.0", path = "../aprender-gpu", package = "aprender-gpu", optional = true } # Terminal handling crossterm = "0.28" diff --git a/crates/aprender-cgp/Cargo.toml b/crates/aprender-cgp/Cargo.toml index 8d24663a49..de0f249f6e 100644 --- a/crates/aprender-cgp/Cargo.toml +++ b/crates/aprender-cgp/Cargo.toml @@ -36,7 +36,7 @@ num_cpus = "1.17" comfy-table = "7" colored = "3" # GPU profiling: direct cuBLAS measurement (OWN THE STACK) -trueno-gpu = { path = "../aprender-gpu", version = "0.71.3", features = ["cuda"], optional = true, package = "aprender-gpu" } +trueno-gpu = { path = "../aprender-gpu", version = "0.72.0", features = ["cuda"], optional = true, package = "aprender-gpu" } [features] default = [] diff --git a/crates/aprender-compute/Cargo.toml b/crates/aprender-compute/Cargo.toml index 1aa9e9f4ab..1146ce220a 100644 --- a/crates/aprender-compute/Cargo.toml +++ b/crates/aprender-compute/Cargo.toml @@ -41,11 +41,11 @@ futures-intrusive = { version = "0.5", optional = true } wasm-bindgen-futures = { version = "0.4", optional = true } wasm-bindgen = { version = "0.2", optional = true } # Native CUDA monitoring via trueno-gpu (TRUENO-SPEC-010) -trueno-gpu = { version = "0.71.3", path = "../aprender-gpu", optional = true, package = "aprender-gpu" } +trueno-gpu = { version = "0.72.0", path = "../aprender-gpu", optional = true, package = "aprender-gpu" } # PMAT-336: Provable contracts compile-time enforcement provable-contracts-macros = { workspace = true } # P1a: Sovereign GEMM microkernel codegen (compile-time shape specialization) -trueno-gemm-codegen = { version = "0.71.3", path = "../aprender-gemm-codegen", package = "aprender-gemm-codegen" } +trueno-gemm-codegen = { version = "0.72.0", path = "../aprender-gemm-codegen", package = "aprender-gemm-codegen" } # Tracing/profiling support (renacer integration) tracing = { version = "0.1", optional = true } tracing-subscriber = { version = "0.3", features = ["env-filter", "fmt"], optional = true } diff --git a/crates/aprender-contracts-cli/Cargo.toml b/crates/aprender-contracts-cli/Cargo.toml index 623642e661..ff5e2bd4a1 100644 --- a/crates/aprender-contracts-cli/Cargo.toml +++ b/crates/aprender-contracts-cli/Cargo.toml @@ -18,7 +18,7 @@ name = "pv-sat" path = "src/bin/pv-sat/main.rs" [dependencies] -aprender-contracts = { path = "../aprender-contracts", version = "0.71.3" } +aprender-contracts = { path = "../aprender-contracts", version = "0.72.0" } # #3745 S1: `--model` is declared through the shared role type `ModelPath`, so # `apr surface` reads `apr pv verify-structure --model` as a model argument. batuta-common = { workspace = true, features = ["cli-roles"] } diff --git a/crates/aprender-contracts/Cargo.toml b/crates/aprender-contracts/Cargo.toml index 7be48d9776..456b9be315 100644 --- a/crates/aprender-contracts/Cargo.toml +++ b/crates/aprender-contracts/Cargo.toml @@ -26,7 +26,7 @@ kani = [] name = "provable_contracts" [dependencies] -provable-contracts-macros = { path = "../aprender-contracts-macros", version = "0.71.3", package = "aprender-contracts-macros" } +provable-contracts-macros = { path = "../aprender-contracts-macros", version = "0.72.0", package = "aprender-contracts-macros" } regex = "1" sha2 = { workspace = true } blake3 = "1" diff --git a/crates/aprender-core/Cargo.toml b/crates/aprender-core/Cargo.toml index 2c6a34777c..0f4d89e172 100644 --- a/crates/aprender-core/Cargo.toml +++ b/crates/aprender-core/Cargo.toml @@ -135,7 +135,7 @@ trueno-quant = { workspace = true } # APR-2231: sovereign `.apr` container leaf. aprender-core From-wraps its # AprFormatError into AprenderError and re-exports it (no API break). Leaf has # no dependency back on core, so this introduces no publish cycle. -apr-format = { path = "../apr-format", version = "0.71" } +apr-format = { path = "../apr-format", version = "0.72" } # NOTE: trueno-rag (=aprender-rag) removed (APR-MONO self-containment): aprender-rag # depends on aprender-core + aprender-serve, so core→rag closed a core→rag→serve→core diff --git a/crates/aprender-cuda-edge/Cargo.toml b/crates/aprender-cuda-edge/Cargo.toml index e31d3b5a9e..463488db78 100644 --- a/crates/aprender-cuda-edge/Cargo.toml +++ b/crates/aprender-cuda-edge/Cargo.toml @@ -15,7 +15,7 @@ name = "trueno_cuda_edge" [dependencies] thiserror = "2.0" serde = { version = "1.0", features = ["derive"] } -trueno-gpu = { version = "0.71.3", path = "../aprender-gpu", package = "aprender-gpu", optional = true } +trueno-gpu = { version = "0.72.0", path = "../aprender-gpu", package = "aprender-gpu", optional = true } [dev-dependencies] proptest = "1.6" diff --git a/crates/aprender-explain/Cargo.toml b/crates/aprender-explain/Cargo.toml index a93edcd1a8..58e15bf28f 100644 --- a/crates/aprender-explain/Cargo.toml +++ b/crates/aprender-explain/Cargo.toml @@ -25,7 +25,7 @@ presentar-terminal = { workspace = true } crossterm = "0.28" # Internal dependency for PTX generation -trueno-gpu = { version = "0.71.3", path = "../aprender-gpu", package = "aprender-gpu", features = ["cuda"] } +trueno-gpu = { version = "0.72.0", path = "../aprender-gpu", package = "aprender-gpu", features = ["cuda"] } [build-dependencies] aprender-build-sha = { workspace = true } # #4219: APR_GIT_SHA for --version diff --git a/crates/aprender-graph/Cargo.toml b/crates/aprender-graph/Cargo.toml index a425c2d747..5c8738d6a1 100644 --- a/crates/aprender-graph/Cargo.toml +++ b/crates/aprender-graph/Cargo.toml @@ -28,7 +28,7 @@ anyhow = "1" thiserror = "2" # Phase 2+ dependencies -aprender = { path = "../aprender-core", version = "0.71.3", package = "aprender-core" } # Sparse matrix ops, PageRank +aprender = { path = "../aprender-core", version = "0.72.0", package = "aprender-core" } # Sparse matrix ops, PageRank trueno = { workspace = true } # Phase 3: GPU acceleration (optional - requires wgpu-capable hardware) diff --git a/crates/aprender-monte-carlo/Cargo.toml b/crates/aprender-monte-carlo/Cargo.toml index 28189ec767..0097820293 100644 --- a/crates/aprender-monte-carlo/Cargo.toml +++ b/crates/aprender-monte-carlo/Cargo.toml @@ -18,7 +18,7 @@ path = "src/lib.rs" name = "aprender-monte-carlo" [dependencies] -aprender = { path = "../../", version = "0.71.3" } +aprender = { path = "../../", version = "0.72.0" } clap = { version = "4", features = ["derive"] } rand = "0.9" rand_chacha = "0.9" diff --git a/crates/aprender-orchestrate/Cargo.toml b/crates/aprender-orchestrate/Cargo.toml index 0d2a522d2d..5f6b60f0ce 100644 --- a/crates/aprender-orchestrate/Cargo.toml +++ b/crates/aprender-orchestrate/Cargo.toml @@ -132,7 +132,7 @@ pacha = { workspace = true, optional = true } realizar = { workspace = true, optional = true } # ML algorithms and training (native-only) -aprender = { path = "../aprender-core", version = "0.71.3", package = "aprender-core", optional = true } +aprender = { path = "../aprender-core", version = "0.72.0", package = "aprender-core", optional = true } entrenar = { workspace = true, optional = true } alimentar = { workspace = true, optional = true } diff --git a/crates/aprender-profile/Cargo.toml b/crates/aprender-profile/Cargo.toml index 793d800d12..894a93801c 100644 --- a/crates/aprender-profile/Cargo.toml +++ b/crates/aprender-profile/Cargo.toml @@ -74,7 +74,7 @@ trueno = { workspace = true } # Machine learning for anomaly detection (Sprint 23) # GH-581: Use path dep to avoid trueno version skew with local workspace -aprender = { path = "../aprender-core", version = "0.71.3", package = "aprender-core" } +aprender = { path = "../aprender-core", version = "0.72.0", package = "aprender-core" } # Lock-free data structures for ring buffer (Sprint 40) crossbeam = { version = "0.8", default-features = false, features = ["crossbeam-channel", "crossbeam-queue", "std"] } @@ -112,7 +112,7 @@ crossterm = "0.28" # SIMD-accelerated visualization primitives (sister project) # NOTE: `monitor` feature removed upstream (#1979); this crate uses only viz primitives. -trueno-viz = { version = "0.71.3", path = "../aprender-viz", package = "aprender-viz", default-features = false } +trueno-viz = { version = "0.72.0", path = "../aprender-viz", package = "aprender-viz", default-features = false } # OpenTelemetry for distributed tracing (Sprint 30) opentelemetry = { version = "0.31.0", optional = true } diff --git a/crates/aprender-qa-cli/Cargo.toml b/crates/aprender-qa-cli/Cargo.toml index a55025c1b6..cfce7190e6 100644 --- a/crates/aprender-qa-cli/Cargo.toml +++ b/crates/aprender-qa-cli/Cargo.toml @@ -23,10 +23,10 @@ serde_json = { workspace = true } serde-saphyr = { workspace = true } clap = { workspace = true } chrono = { workspace = true } -aprender-qa-gen = { version = "0.71.3", path = "../aprender-qa-gen" } -aprender-qa-runner = { version = "0.71.3", path = "../aprender-qa-runner" } -aprender-qa-report = { version = "0.71.3", path = "../aprender-qa-report" } -aprender-qa-certify = { version = "0.71.3", path = "../aprender-qa-certify" } +aprender-qa-gen = { version = "0.72.0", path = "../aprender-qa-gen" } +aprender-qa-runner = { version = "0.72.0", path = "../aprender-qa-runner" } +aprender-qa-report = { version = "0.72.0", path = "../aprender-qa-report" } +aprender-qa-certify = { version = "0.72.0", path = "../aprender-qa-certify" } ctrlc = "3.4" safetensors = { workspace = true } colored = "2.2" diff --git a/crates/aprender-qa-report/Cargo.toml b/crates/aprender-qa-report/Cargo.toml index ba7a1ea355..d1baa7cb00 100644 --- a/crates/aprender-qa-report/Cargo.toml +++ b/crates/aprender-qa-report/Cargo.toml @@ -18,8 +18,8 @@ anyhow = { workspace = true } chrono = { workspace = true } colored = "2.2" csv = "1.3" -aprender-qa-gen = { version = "0.71.3", path = "../aprender-qa-gen" } -aprender-qa-runner = { version = "0.71.3", path = "../aprender-qa-runner" } +aprender-qa-gen = { version = "0.72.0", path = "../aprender-qa-gen" } +aprender-qa-runner = { version = "0.72.0", path = "../aprender-qa-runner" } [dev-dependencies] proptest = { workspace = true } diff --git a/crates/aprender-qa-runner/Cargo.toml b/crates/aprender-qa-runner/Cargo.toml index 1f2efff033..38e4540744 100644 --- a/crates/aprender-qa-runner/Cargo.toml +++ b/crates/aprender-qa-runner/Cargo.toml @@ -22,14 +22,14 @@ anyhow = { workspace = true } tokio = { workspace = true } chrono = { workspace = true, features = ["serde"] } tracing = { workspace = true } -aprender-qa-gen = { version = "0.71.3", path = "../aprender-qa-gen" } +aprender-qa-gen = { version = "0.72.0", path = "../aprender-qa-gen" } hostname = "0.4" rayon = "1.10" sha2 = "0.10" regex = { workspace = true } num_cpus = "1.16" safetensors = { workspace = true } -jugar-probar = { version = "0.71.3", path = "../aprender-test-lib", package = "aprender-test-lib", features = ["llm-types"] } +jugar-probar = { version = "0.72.0", path = "../aprender-test-lib", package = "aprender-test-lib", features = ["llm-types"] } # Test fixtures dependencies (optional) tempfile = { version = "3", optional = true } half = { version = "2.4", optional = true } diff --git a/crates/aprender-rag-cli/Cargo.toml b/crates/aprender-rag-cli/Cargo.toml index dbcef96aae..838f8b0139 100644 --- a/crates/aprender-rag-cli/Cargo.toml +++ b/crates/aprender-rag-cli/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "aprender-rag-cli" -version = "0.71.3" +version = "0.72.0" edition = "2021" authors = ["Pragmatic AI Labs"] description = "CLI for Trueno-RAG pipeline" diff --git a/crates/aprender-rag/Cargo.toml b/crates/aprender-rag/Cargo.toml index f2f4ff3900..93ea394dc8 100644 --- a/crates/aprender-rag/Cargo.toml +++ b/crates/aprender-rag/Cargo.toml @@ -63,7 +63,7 @@ fastembed = { version = "5", optional = true } # NVIDIA Embed Nemotron 8B via realizar GGUF infrastructure (GH-3) # Large model (~4-32GB depending on quantization), requires GGUF file -realizar = { version = "0.71.3", path = "../aprender-serve", package = "aprender-serve", default-features = false, optional = true } +realizar = { version = "0.72.0", path = "../aprender-serve", package = "aprender-serve", default-features = false, optional = true } # Eval: LLM-as-judge via Anthropic API (PMAT-015) diff --git a/crates/aprender-registry/Cargo.toml b/crates/aprender-registry/Cargo.toml index b4971eb795..1d60058b37 100644 --- a/crates/aprender-registry/Cargo.toml +++ b/crates/aprender-registry/Cargo.toml @@ -42,7 +42,7 @@ zstd = { version = "0.13", optional = true } clap = { version = "4", features = ["derive"], optional = true } # Ecosystem integration -aprender = { path = "../aprender-core", version = "0.71.3", package = "aprender-core", optional = true } +aprender = { path = "../aprender-core", version = "0.72.0", package = "aprender-core", optional = true } alimentar = { workspace = true, optional = true } # HTTP client (optional) diff --git a/crates/aprender-serve/Cargo.toml b/crates/aprender-serve/Cargo.toml index 64d48d0846..3ce5b802a5 100644 --- a/crates/aprender-serve/Cargo.toml +++ b/crates/aprender-serve/Cargo.toml @@ -51,7 +51,7 @@ trueno = { workspace = true, features = ["gpu"] } unicode-properties = { workspace = true } # CUDA PTX generation and runtime for NVIDIA GPUs (optional, pure Rust, no LLVM/nvcc) # v0.4.12: BatchedQ6K, MultiWarp, RopeNeox, FlashDecoding kernels -trueno-gpu = { version = "0.71.3", path = "../aprender-gpu", package = "aprender-gpu", optional = true, features = ["cuda"] } # APR-MONO single-source-of-truth +trueno-gpu = { version = "0.72.0", path = "../aprender-gpu", package = "aprender-gpu", optional = true, features = ["cuda"] } # APR-MONO single-source-of-truth # KV cache storage (for attention caching) trueno-db = { workspace = true, optional = true } # K-quantization formats (Q4_K, Q5_K, Q6_K) - Toyota Way: ONE source of truth @@ -65,7 +65,7 @@ renacer-core = { workspace = true } # ML algorithms and .apr model format (optional for aprender model serving) # v0.24.1: chat templates, GGUF tokenizer preservation (local may be newer) -aprender = { path = "../aprender-core", version = "0.71.3", package = "aprender-core", optional = true } +aprender = { path = "../aprender-core", version = "0.72.0", package = "aprender-core", optional = true } # Data loading library (optional for data pipeline examples) alimentar = { workspace = true, optional = true } diff --git a/crates/aprender-shell/Cargo.toml b/crates/aprender-shell/Cargo.toml index fc7282bd35..fbaeee9ff9 100644 --- a/crates/aprender-shell/Cargo.toml +++ b/crates/aprender-shell/Cargo.toml @@ -24,7 +24,7 @@ harness = false [dependencies] # Use path for local dev, crates.io for release -aprender = { path = "../aprender-core", version = "0.71.3", package = "aprender-core", features = ["format-encryption", "format-compression", "format-homomorphic"] } +aprender = { path = "../aprender-core", version = "0.72.0", package = "aprender-core", features = ["format-encryption", "format-compression", "format-homomorphic"] } clap = { version = "4", features = ["derive"] } dirs = "5" serde = { version = "1", features = ["derive"] } diff --git a/crates/aprender-simulate/Cargo.toml b/crates/aprender-simulate/Cargo.toml index e5ff4de6bf..849b3c899d 100644 --- a/crates/aprender-simulate/Cargo.toml +++ b/crates/aprender-simulate/Cargo.toml @@ -59,7 +59,7 @@ crossterm = { version = "0.28", optional = true } # ComputeBlocks from presentar-terminal (SIMD-optimized TUI widgets) # Path deps to avoid jugar-probar prod dep chain (CB-081) presentar-terminal = { workspace = true, optional = true } -presentar-core = { version = "0.71.3", path = "../aprender-present-core", package = "aprender-present-core", default-features = false, optional = true } +presentar-core = { version = "0.72.0", path = "../aprender-present-core", package = "aprender-present-core", default-features = false, optional = true } # WASM (optional) wasm-bindgen = { version = "0.2", optional = true } diff --git a/crates/aprender-test-cli/Cargo.toml b/crates/aprender-test-cli/Cargo.toml index 7a9d2ed89c..e87fa6b3e6 100644 --- a/crates/aprender-test-cli/Cargo.toml +++ b/crates/aprender-test-cli/Cargo.toml @@ -35,7 +35,7 @@ path = "src/main.rs" [dependencies] # Core library (was: jugar-probar) — dep key preserves Rust identifier -jugar-probar = { path = "../aprender-test-lib", version = "0.71.3", package = "aprender-test-lib" } +jugar-probar = { path = "../aprender-test-lib", version = "0.72.0", package = "aprender-test-lib" } # CLI framework clap = { workspace = true } diff --git a/crates/aprender-train-bench/Cargo.toml b/crates/aprender-train-bench/Cargo.toml index e84ad4199b..c4b33b1156 100644 --- a/crates/aprender-train-bench/Cargo.toml +++ b/crates/aprender-train-bench/Cargo.toml @@ -20,7 +20,7 @@ workspace = true [dependencies] entrenar = { workspace = true } -entrenar-common = { version = "0.71.3", path = "../aprender-train-common", package = "aprender-train-common" } +entrenar-common = { version = "0.72.0", path = "../aprender-train-common", package = "aprender-train-common" } clap = { version = "4.5", features = ["derive"] } serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" diff --git a/crates/aprender-train-distill/Cargo.toml b/crates/aprender-train-distill/Cargo.toml index 8414bd60fb..86dd6de48b 100644 --- a/crates/aprender-train-distill/Cargo.toml +++ b/crates/aprender-train-distill/Cargo.toml @@ -33,7 +33,7 @@ shard-batch-source = [] [dependencies] entrenar = { workspace = true } -entrenar-common = { version = "0.71.3", path = "../aprender-train-common", package = "aprender-train-common" } +entrenar-common = { version = "0.72.0", path = "../aprender-train-common", package = "aprender-train-common" } clap = { version = "4.5", features = ["derive"] } serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" diff --git a/crates/aprender-train-inspect/Cargo.toml b/crates/aprender-train-inspect/Cargo.toml index c86eb7ab8e..790a45deb0 100644 --- a/crates/aprender-train-inspect/Cargo.toml +++ b/crates/aprender-train-inspect/Cargo.toml @@ -20,7 +20,7 @@ workspace = true [dependencies] entrenar = { workspace = true } -entrenar-common = { version = "0.71.3", path = "../aprender-train-common", package = "aprender-train-common" } +entrenar-common = { version = "0.72.0", path = "../aprender-train-common", package = "aprender-train-common" } clap = { version = "4.5", features = ["derive"] } serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" diff --git a/crates/aprender-train-lora/Cargo.toml b/crates/aprender-train-lora/Cargo.toml index a0ac8118ca..ffb3be7845 100644 --- a/crates/aprender-train-lora/Cargo.toml +++ b/crates/aprender-train-lora/Cargo.toml @@ -20,7 +20,7 @@ workspace = true [dependencies] entrenar = { workspace = true } -entrenar-common = { version = "0.71.3", path = "../aprender-train-common", package = "aprender-train-common" } +entrenar-common = { version = "0.72.0", path = "../aprender-train-common", package = "aprender-train-common" } clap = { version = "4.5", features = ["derive"] } serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" diff --git a/crates/aprender-train-shell/Cargo.toml b/crates/aprender-train-shell/Cargo.toml index e59339b719..25b8369004 100644 --- a/crates/aprender-train-shell/Cargo.toml +++ b/crates/aprender-train-shell/Cargo.toml @@ -20,7 +20,7 @@ workspace = true [dependencies] entrenar = { workspace = true } -entrenar-common = { version = "0.71.3", path = "../aprender-train-common", package = "aprender-train-common" } +entrenar-common = { version = "0.72.0", path = "../aprender-train-common", package = "aprender-train-common" } clap = { version = "4.5", features = ["derive"] } rustyline = "14" serde = { version = "1.0", features = ["derive"] } diff --git a/crates/aprender-train/Cargo.toml b/crates/aprender-train/Cargo.toml index ac52f7d28d..aa62825cba 100644 --- a/crates/aprender-train/Cargo.toml +++ b/crates/aprender-train/Cargo.toml @@ -80,9 +80,9 @@ provable-contracts-macros = { workspace = true } # Core PAIML stack ndarray = "0.16" trueno = { workspace = true, features = ["parallel"] } -trueno-gpu = { version = "0.71.3", path = "../aprender-gpu", package = "aprender-gpu", optional = true } # CUDA compute (APR-MONO single-source-of-truth) +trueno-gpu = { version = "0.72.0", path = "../aprender-gpu", package = "aprender-gpu", optional = true } # CUDA compute (APR-MONO single-source-of-truth) realizar = { workspace = true, optional = true } -aprender = { path = "../aprender-core", version = "0.71.3", package = "aprender-core" } # ML interpret module + .apr format + tokenizers +aprender = { path = "../aprender-core", version = "0.72.0", package = "aprender-core" } # ML interpret module + .apr format + tokenizers trueno-viz = { workspace = true, features = ["terminal"] } batuta-common = { workspace = true, optional = true } # WithDimensions trait for `viz` feature (APR-MONO §S #1978) presentar-terminal = { workspace = true, optional = true } diff --git a/crates/aprender-tsp/Cargo.toml b/crates/aprender-tsp/Cargo.toml index c40eeb0377..75b9f21dd0 100644 --- a/crates/aprender-tsp/Cargo.toml +++ b/crates/aprender-tsp/Cargo.toml @@ -18,7 +18,7 @@ path = "src/lib.rs" name = "aprender-tsp" [dependencies] -aprender = { path = "../../", version = "0.71.3" } +aprender = { path = "../../", version = "0.72.0" } clap = { version = "4", features = ["derive"] } rand = "0.9" crc32fast = "1.4" diff --git a/crates/aprender-verify-ml/Cargo.toml b/crates/aprender-verify-ml/Cargo.toml index 7c04113a63..ca64c8cf94 100644 --- a/crates/aprender-verify-ml/Cargo.toml +++ b/crates/aprender-verify-ml/Cargo.toml @@ -56,7 +56,7 @@ uuid = { version = "1.11", features = ["v4", "serde"] } trueno = { workspace = true } # Machine learning for bug prediction (optional) -aprender = { path = "../aprender-core", version = "0.71.3", package = "aprender-core", optional = true } +aprender = { path = "../aprender-core", version = "0.72.0", package = "aprender-core", optional = true } # LLM fine-tuning integration (optional) entrenar = { workspace = true, optional = true } diff --git a/crates/aprender-viz/Cargo.toml b/crates/aprender-viz/Cargo.toml index af6276b15e..617c30095f 100644 --- a/crates/aprender-viz/Cargo.toml +++ b/crates/aprender-viz/Cargo.toml @@ -14,7 +14,7 @@ categories = ["visualization", "graphics", "science", "wasm"] [dependencies] # Core SIMD/GPU acceleration -trueno = { version = "0.71.3", path = "../aprender-compute", package = "aprender-compute", default-features = false } +trueno = { version = "0.72.0", path = "../aprender-compute", package = "aprender-compute", default-features = false } # Shared Batuta stack utilities (display traits, formatting) batuta-common = { workspace = true } @@ -52,10 +52,10 @@ kurbo = { version = "0.13", optional = true } resvg = { version = "0.45", optional = true, default-features = false } # Optional: ML integration -aprender = { path = "../aprender-core", version = "0.71.3", package = "aprender-core", optional = true } +aprender = { path = "../aprender-core", version = "0.72.0", package = "aprender-core", optional = true } # Optional: Graph integration -trueno-graph = { version = "0.71.3", path = "../aprender-graph", package = "aprender-graph", default-features = false, optional = true } +trueno-graph = { version = "0.72.0", path = "../aprender-graph", package = "aprender-graph", default-features = false, optional = true } # Optional: Database integration trueno-db = { workspace = true, optional = true } diff --git a/crates/facades/Cargo.lock b/crates/facades/Cargo.lock index 4dfeab8f7c..c020ddfb75 100644 --- a/crates/facades/Cargo.lock +++ b/crates/facades/Cargo.lock @@ -13,7 +13,7 @@ dependencies = [ [[package]] name = "aprender-contracts" -version = "0.71.3" +version = "0.72.0" dependencies = [ "aprender-contracts-macros", "blake3", @@ -29,7 +29,7 @@ dependencies = [ [[package]] name = "aprender-contracts-macros" -version = "0.71.3" +version = "0.72.0" dependencies = [ "proc-macro2", "quote", diff --git a/crates/facades/provable-contracts-macros/Cargo.toml b/crates/facades/provable-contracts-macros/Cargo.toml index a8617f0acf..88a79c0205 100644 --- a/crates/facades/provable-contracts-macros/Cargo.toml +++ b/crates/facades/provable-contracts-macros/Cargo.toml @@ -30,7 +30,7 @@ path = "src/lib.rs" # facade resolves. `scripts/check_facade_compat.sh` fails if this version ever # drifts from the aprender workspace version — these manifests are excluded # from the root workspace, so `cargo set-version` does NOT reach them. -upstream = { path = "../../aprender-contracts-macros", version = "0.71.3", package = "aprender-contracts-macros" } +upstream = { path = "../../aprender-contracts-macros", version = "0.72.0", package = "aprender-contracts-macros" } [features] default = [] diff --git a/crates/facades/provable-contracts/Cargo.toml b/crates/facades/provable-contracts/Cargo.toml index b772957245..c0bdad09f5 100644 --- a/crates/facades/provable-contracts/Cargo.toml +++ b/crates/facades/provable-contracts/Cargo.toml @@ -31,7 +31,7 @@ path = "src/lib.rs" # facade resolves. `scripts/check_facade_compat.sh` fails if this version ever # drifts from the aprender workspace version — these manifests are excluded # from the root workspace, so `cargo set-version` does NOT reach them. -upstream = { path = "../../aprender-contracts", version = "0.71.3", package = "aprender-contracts" } +upstream = { path = "../../aprender-contracts", version = "0.72.0", package = "aprender-contracts" } [dev-dependencies] # Used by three of the vendored 0.3.1 example programs. They were regular diff --git a/evidence/crux/0.72.0/prompt-certification-inventory.json b/evidence/crux/0.72.0/prompt-certification-inventory.json new file mode 100644 index 0000000000..93705629cf --- /dev/null +++ b/evidence/crux/0.72.0/prompt-certification-inventory.json @@ -0,0 +1,13 @@ +[ + {"model": "Qwen3.5-2B", + "source": {"repo": "Qwen/Qwen3.5-2B", "revision": "15852e8c16360a2fea060d615a32b45270f8a8fc"}, + "bf16_gguf": "dc11be2ca4519954a3e4b013ef59497b5522209987d67891851113eda89619f1", + "quants": {"Q4_K_M": "aaf42c8b7c3cab2bf3d69c355048d4a0ee9973d48f16c731c0520ee914699223"}, + "thinking": ["on", "off"]}, + {"model": "Qwen3.5-4B", + "source": {"repo": "Qwen/Qwen3.5-4B", "revision": "851bf6e806efd8d0a36b00ddf55e13ccb7b8cd0a"}, + "bf16_gguf": "9e6e2841a75f503ccb330831832fd7861266e187e0dbf149a954219ccb8c197a", + "quants": {"Q4_K_M": "00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4", + "UD-Q4_K_XL": "b252c5610a42ca82d20fe2a12813e9d069eed89292907e26c783eeb0bc961bc7"}, + "thinking": ["on", "off"]} +] diff --git a/evidence/crux/0.72.0/prompt-certification.json b/evidence/crux/0.72.0/prompt-certification.json new file mode 100644 index 0000000000..1323d4134b --- /dev/null +++ b/evidence/crux/0.72.0/prompt-certification.json @@ -0,0 +1,8906 @@ +{ + "schema": "crux-prompt-certification/v1", + "prompts": "scripts/crux_inference_prompts.v2.json", + "prompts_sha256": "cea80df5e785213667e27697fe3d139b99dcba95cfc6e970d76ac005d50fc806", + "apr_commit": "a781138719bc6d147f585ed6d803190456663fec", + "inventory_sha256": "b8e4c4626b20b7764041f5b5fe6d8a98166a952a64b3a844207b796b388778e0", + "manifests": { + "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl": "f56747953b33bb2e4adac4bac12be902224a202b2a07c35ede3e47121d178728", + "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl": "6a5657345a6a61ee95ca7a8d7237d7f9748ec8e6d652bdd95be042a586a4e45f" + }, + "admitted": { + "Qwen3.5-2B/Q4_K_M": [], + "Qwen3.5-4B/Q4_K_M": [ + "ctl-2plus2", + "ctl-code-add", + "fact-capital-japan", + "fact-capital-france", + "golden-paris", + "nonascii-water-kanji", + "arith-17x23", + "arith-precedence", + "arith-word-apples", + "hard-not-prime", + "hard-reverse-word", + "hard-count-letters", + "recall-arith-chain", + "code-count-vowels", + "long-primes-10", + "long-countdown-15" + ], + "Qwen3.5-4B/UD-Q4_K_XL": [ + "ctl-2plus2", + "ctl-code-add", + "fact-capital-japan", + "fact-capital-france", + "golden-paris", + "nonascii-water-kanji", + "arith-17x23", + "arith-precedence", + "arith-word-apples", + "hard-not-prime", + "hard-count-letters", + "recall-arith-chain", + "long-primes-10", + "long-countdown-15" + ] + }, + "admitted_by_sha": { + "aaf42c8b7c3cab2bf3d69c355048d4a0ee9973d48f16c731c0520ee914699223": [], + "00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4": [ + "ctl-2plus2", + "ctl-code-add", + "fact-capital-japan", + "fact-capital-france", + "golden-paris", + "nonascii-water-kanji", + "arith-17x23", + "arith-precedence", + "arith-word-apples", + "hard-not-prime", + "hard-reverse-word", + "hard-count-letters", + "recall-arith-chain", + "code-count-vowels", + "long-primes-10", + "long-countdown-15" + ], + "b252c5610a42ca82d20fe2a12813e9d069eed89292907e26c783eeb0bc961bc7": [ + "ctl-2plus2", + "ctl-code-add", + "fact-capital-japan", + "fact-capital-france", + "golden-paris", + "nonascii-water-kanji", + "arith-17x23", + "arith-precedence", + "arith-word-apples", + "hard-not-prime", + "hard-count-letters", + "recall-arith-chain", + "long-primes-10", + "long-countdown-15" + ] + }, + "admitted_by_sha_thinking": { + "aaf42c8b7c3cab2bf3d69c355048d4a0ee9973d48f16c731c0520ee914699223": { + "on": [], + "off": [ + "ctl-2plus2", + "ctl-recall-7", + "ctl-code-add", + "fact-capital-japan", + "fact-capital-france", + "golden-paris", + "arith-17x23", + "arith-precedence", + "arith-word-apples", + "hard-add-4digit", + "hard-not-prime", + "hard-count-letters", + "recall-cats", + "recall-name", + "recall-arith-chain", + "code-is-palindrome", + "code-fib", + "code-count-vowels", + "long-primes-10", + "long-countdown-15" + ] + }, + "00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4": { + "on": [ + "ctl-2plus2", + "ctl-code-add", + "fact-capital-japan", + "fact-capital-france", + "golden-paris", + "nonascii-water-kanji", + "arith-17x23", + "arith-precedence", + "arith-word-apples", + "hard-not-prime", + "hard-reverse-word", + "hard-count-letters", + "recall-arith-chain", + "code-count-vowels", + "long-primes-10", + "long-countdown-15" + ], + "off": [ + "ctl-2plus2", + "ctl-recall-7", + "ctl-code-add", + "fact-capital-japan", + "fact-capital-france", + "golden-paris", + "nonascii-water-kanji", + "arith-17x23", + "arith-precedence", + "arith-word-apples", + "hard-add-4digit", + "hard-not-prime", + "hard-reverse-word", + "hard-count-letters", + "recall-cats", + "recall-name", + "recall-arith-chain", + "code-is-palindrome", + "code-fib", + "code-count-vowels", + "long-primes-10", + "long-countdown-15" + ] + }, + "b252c5610a42ca82d20fe2a12813e9d069eed89292907e26c783eeb0bc961bc7": { + "on": [ + "ctl-2plus2", + "ctl-code-add", + "fact-capital-japan", + "fact-capital-france", + "golden-paris", + "nonascii-water-kanji", + "arith-17x23", + "arith-precedence", + "arith-word-apples", + "hard-not-prime", + "hard-count-letters", + "recall-arith-chain", + "long-primes-10", + "long-countdown-15" + ], + "off": [ + "ctl-2plus2", + "ctl-recall-7", + "ctl-code-add", + "fact-capital-japan", + "fact-capital-france", + "golden-paris", + "nonascii-water-kanji", + "arith-17x23", + "arith-precedence", + "arith-word-apples", + "hard-add-4digit", + "hard-not-prime", + "hard-reverse-word", + "hard-count-letters", + "recall-cats", + "recall-name", + "recall-arith-chain", + "code-is-palindrome", + "code-fib", + "code-count-vowels", + "long-primes-10", + "long-countdown-15" + ] + } + }, + "rejected": { + "Qwen3.5-2B/Q4_K_M": { + "ctl-2plus2": "ggml@bf16 llama.cpp serve run thinking=on on gx10: unclosed think (budget exhausted)", + "ctl-recall-7": "ggml@bf16 llama.cpp serve run thinking=on on gx10: unclosed think (budget exhausted)", + "ctl-code-add": "hf@bf16 thinking=on: no row", + "fact-capital-japan": "ggml@bf16 llama.cpp serve run thinking=on on gx10: unclosed think (budget exhausted)", + "fact-capital-france": "hf@bf16 thinking=on: no row", + "golden-paris": "ggml@bf16 thinking=on: no row", + "fact-water-formula": "ggml@bf16 llama.cpp serve run thinking=on on gx10: unclosed think (budget exhausted)", + "nonascii-water-kanji": "ggml@bf16 thinking=on: no row", + "arith-17x23": "ggml@bf16 llama.cpp serve run thinking=on on gx10: unclosed think (budget exhausted)", + "arith-precedence": "hf@bf16 thinking=on: no row", + "arith-word-apples": "hf@bf16 thinking=on: no row", + "hard-add-4digit": "hf@bf16 thinking=on: no row", + "hard-not-prime": "ggml@bf16 thinking=on: no row", + "hard-reverse-word": "ggml@bf16 thinking=on: no row", + "hard-count-letters": "ggml@bf16 thinking=on: no row", + "recall-cats": "ggml@bf16 thinking=on: no row", + "recall-name": "ggml@bf16 thinking=on: no row", + "recall-arith-chain": "ggml@bf16 thinking=on: no row", + "code-is-palindrome": "ggml@bf16 thinking=on: no row", + "code-fib": "ggml@bf16 thinking=on: no row", + "code-count-vowels": "ggml@bf16 thinking=on: no row", + "long-primes-10": "ggml@bf16 thinking=on: no row", + "long-countdown-15": "ggml@bf16 thinking=on: no row" + }, + "Qwen3.5-4B/Q4_K_M": { + "ctl-recall-7": "ggml@quant llama.cpp serve run thinking=on on gx10: unclosed think (budget exhausted)", + "fact-water-formula": "ggml@bf16 llama.cpp serve run thinking=on on gx10: unclosed think (budget exhausted)", + "hard-add-4digit": "ggml@quant llama.cpp serve run thinking=on on gx10: unclosed think (budget exhausted)", + "recall-cats": "ggml@bf16 llama.cpp serve run thinking=on on gx10: unclosed think (budget exhausted)", + "recall-name": "ggml@quant llama.cpp serve run thinking=on on gx10: unclosed think (budget exhausted)", + "code-is-palindrome": "vllm@bf16 vllm chat thinking=on on gx10: tests_failed: AssertionError", + "code-fib": "ggml@bf16 llama.cpp serve run thinking=on on gx10: unclosed think (budget exhausted)" + }, + "Qwen3.5-4B/UD-Q4_K_XL": { + "ctl-recall-7": "ggml@quant llama.cpp serve run thinking=on on gx10: unclosed think (budget exhausted)", + "fact-water-formula": "ggml@bf16 llama.cpp serve run thinking=on on gx10: unclosed think (budget exhausted)", + "hard-add-4digit": "ggml@quant llama.cpp serve run thinking=on on gx10: unclosed think (budget exhausted)", + "hard-reverse-word": "ggml@quant llama.cpp serve run thinking=on on gx10: mismatch", + "recall-cats": "ggml@bf16 llama.cpp serve run thinking=on on gx10: unclosed think (budget exhausted)", + "recall-name": "ggml@quant llama.cpp serve run thinking=on on gx10: unclosed think (budget exhausted)", + "code-is-palindrome": "vllm@bf16 vllm chat thinking=on on gx10: tests_failed: AssertionError", + "code-fib": "ggml@bf16 llama.cpp serve run thinking=on on gx10: unclosed think (budget exhausted)", + "code-count-vowels": "ggml@quant llama.cpp serve run thinking=on on gx10: unclosed think (budget exhausted)" + } + }, + "uncontrolled": [ + "Qwen3.5-2B/Q4_K_M" + ], + "uncontrolled_detail": [ + { + "model": "Qwen3.5-2B/Q4_K_M", + "sha256": "aaf42c8b7c3cab2bf3d69c355048d4a0ee9973d48f16c731c0520ee914699223", + "thinking": "on", + "verbs": [ + "chat", + "code", + "run", + "serve run", + "serve stream" + ] + } + ], + "think_closure": { + "Qwen3.5-2B/Q4_K_M|ctl-2plus2": { + "ggml@bf16:llama.cpp": "unclosed" + }, + "Qwen3.5-2B/Q4_K_M|ctl-recall-7": { + "ggml@bf16:llama.cpp": "unclosed" + }, + "Qwen3.5-2B/Q4_K_M|ctl-code-add": { + "ggml@bf16:llama.cpp": "closed" + }, + "Qwen3.5-2B/Q4_K_M|fact-capital-japan": { + "ggml@bf16:llama.cpp": "unclosed" + }, + "Qwen3.5-2B/Q4_K_M|fact-capital-france": { + "ggml@bf16:llama.cpp": "closed" + }, + "Qwen3.5-2B/Q4_K_M|fact-water-formula": { + "ggml@bf16:llama.cpp": "unclosed" + }, + "Qwen3.5-2B/Q4_K_M|arith-17x23": { + "ggml@bf16:llama.cpp": "unclosed" + }, + "Qwen3.5-2B/Q4_K_M|arith-precedence": { + "ggml@bf16:llama.cpp": "closed" + }, + "Qwen3.5-2B/Q4_K_M|arith-word-apples": { + "ggml@bf16:llama.cpp": "closed" + }, + "Qwen3.5-2B/Q4_K_M|hard-add-4digit": { + "ggml@bf16:llama.cpp": "closed" + }, + "Qwen3.5-4B/Q4_K_M|ctl-2plus2": { + "ggml@bf16:llama.cpp": "closed", + "hf@bf16:hf": "closed", + "vllm@bf16:vllm": "closed", + "ggml@quant:llama.cpp": "closed" + }, + "Qwen3.5-4B/Q4_K_M|ctl-recall-7": { + "ggml@bf16:llama.cpp": "closed", + "hf@bf16:hf": "closed", + "vllm@bf16:vllm": "closed", + "ggml@quant:llama.cpp": "unclosed" + }, + "Qwen3.5-4B/Q4_K_M|ctl-code-add": { + "ggml@bf16:llama.cpp": "closed", + "hf@bf16:hf": "closed", + "vllm@bf16:vllm": "closed", + "ggml@quant:llama.cpp": "closed" + }, + "Qwen3.5-4B/Q4_K_M|fact-capital-japan": { + "ggml@bf16:llama.cpp": "closed", + "hf@bf16:hf": "closed", + "vllm@bf16:vllm": "closed", + "ggml@quant:llama.cpp": "closed" + }, + "Qwen3.5-4B/Q4_K_M|fact-capital-france": { + "ggml@bf16:llama.cpp": "closed", + "hf@bf16:hf": "closed", + "vllm@bf16:vllm": "closed", + "ggml@quant:llama.cpp": "closed" + }, + "Qwen3.5-4B/Q4_K_M|golden-paris": { + "ggml@bf16:llama.cpp": "closed", + "hf@bf16:hf": "closed", + "vllm@bf16:vllm": "closed", + "ggml@quant:llama.cpp": "closed" + }, + "Qwen3.5-4B/Q4_K_M|fact-water-formula": { + "ggml@bf16:llama.cpp": "unclosed", + "hf@bf16:hf": "unclosed", + "vllm@bf16:vllm": "unclosed", + "ggml@quant:llama.cpp": "closed" + }, + "Qwen3.5-4B/Q4_K_M|nonascii-water-kanji": { + "ggml@bf16:llama.cpp": "closed", + "hf@bf16:hf": "closed", + "vllm@bf16:vllm": "closed", + "ggml@quant:llama.cpp": "closed" + }, + "Qwen3.5-4B/Q4_K_M|arith-17x23": { + "ggml@bf16:llama.cpp": "closed", + "hf@bf16:hf": "closed", + "vllm@bf16:vllm": "closed", + "ggml@quant:llama.cpp": "closed" + }, + "Qwen3.5-4B/Q4_K_M|arith-precedence": { + "ggml@bf16:llama.cpp": "closed", + "hf@bf16:hf": "closed", + "vllm@bf16:vllm": "closed", + "ggml@quant:llama.cpp": "closed" + }, + "Qwen3.5-4B/Q4_K_M|arith-word-apples": { + "ggml@bf16:llama.cpp": "closed", + "hf@bf16:hf": "closed", + "vllm@bf16:vllm": "closed", + "ggml@quant:llama.cpp": "closed" + }, + "Qwen3.5-4B/Q4_K_M|hard-add-4digit": { + "ggml@bf16:llama.cpp": "closed", + "hf@bf16:hf": "closed", + "vllm@bf16:vllm": "closed", + "ggml@quant:llama.cpp": "unclosed" + }, + "Qwen3.5-4B/Q4_K_M|hard-not-prime": { + "ggml@bf16:llama.cpp": "closed", + "hf@bf16:hf": "closed", + "vllm@bf16:vllm": "closed", + "ggml@quant:llama.cpp": "closed" + }, + "Qwen3.5-4B/Q4_K_M|hard-reverse-word": { + "ggml@bf16:llama.cpp": "closed", + "hf@bf16:hf": "closed", + "vllm@bf16:vllm": "closed", + "ggml@quant:llama.cpp": "closed" + }, + "Qwen3.5-4B/Q4_K_M|hard-count-letters": { + "ggml@bf16:llama.cpp": "closed", + "hf@bf16:hf": "closed", + "vllm@bf16:vllm": "closed", + "ggml@quant:llama.cpp": "closed" + }, + "Qwen3.5-4B/Q4_K_M|recall-cats": { + "ggml@bf16:llama.cpp": "unclosed", + "hf@bf16:hf": "closed", + "vllm@bf16:vllm": "closed", + "ggml@quant:llama.cpp": "unclosed" + }, + "Qwen3.5-4B/Q4_K_M|recall-name": { + "ggml@bf16:llama.cpp": "closed", + "hf@bf16:hf": "closed", + "vllm@bf16:vllm": "closed", + "ggml@quant:llama.cpp": "unclosed" + }, + "Qwen3.5-4B/Q4_K_M|recall-arith-chain": { + "ggml@bf16:llama.cpp": "closed", + "hf@bf16:hf": "closed", + "vllm@bf16:vllm": "closed", + "ggml@quant:llama.cpp": "closed" + }, + "Qwen3.5-4B/Q4_K_M|code-is-palindrome": { + "ggml@bf16:llama.cpp": "closed", + "hf@bf16:hf": "closed", + "vllm@bf16:vllm": "closed", + "ggml@quant:llama.cpp": "unclosed" + }, + "Qwen3.5-4B/Q4_K_M|code-fib": { + "ggml@bf16:llama.cpp": "unclosed", + "hf@bf16:hf": "unclosed", + "vllm@bf16:vllm": "unclosed", + "ggml@quant:llama.cpp": "unclosed" + }, + "Qwen3.5-4B/Q4_K_M|code-count-vowels": { + "ggml@bf16:llama.cpp": "closed", + "hf@bf16:hf": "closed", + "vllm@bf16:vllm": "closed", + "ggml@quant:llama.cpp": "closed" + }, + "Qwen3.5-4B/Q4_K_M|long-primes-10": { + "ggml@bf16:llama.cpp": "closed", + "hf@bf16:hf": "closed", + "vllm@bf16:vllm": "closed", + "ggml@quant:llama.cpp": "closed" + }, + "Qwen3.5-4B/Q4_K_M|long-countdown-15": { + "ggml@bf16:llama.cpp": "closed", + "hf@bf16:hf": "closed", + "vllm@bf16:vllm": "closed", + "ggml@quant:llama.cpp": "closed" + }, + "Qwen3.5-4B/UD-Q4_K_XL|ctl-2plus2": { + "ggml@bf16:llama.cpp": "closed", + "hf@bf16:hf": "closed", + "vllm@bf16:vllm": "closed", + "ggml@quant:llama.cpp": "closed" + }, + "Qwen3.5-4B/UD-Q4_K_XL|ctl-recall-7": { + "ggml@bf16:llama.cpp": "closed", + "hf@bf16:hf": "closed", + "vllm@bf16:vllm": "closed", + "ggml@quant:llama.cpp": "unclosed" + }, + "Qwen3.5-4B/UD-Q4_K_XL|ctl-code-add": { + "ggml@bf16:llama.cpp": "closed", + "hf@bf16:hf": "closed", + "vllm@bf16:vllm": "closed", + "ggml@quant:llama.cpp": "closed" + }, + "Qwen3.5-4B/UD-Q4_K_XL|fact-capital-japan": { + "ggml@bf16:llama.cpp": "closed", + "hf@bf16:hf": "closed", + "vllm@bf16:vllm": "closed", + "ggml@quant:llama.cpp": "closed" + }, + "Qwen3.5-4B/UD-Q4_K_XL|fact-capital-france": { + "ggml@bf16:llama.cpp": "closed", + "hf@bf16:hf": "closed", + "vllm@bf16:vllm": "closed", + "ggml@quant:llama.cpp": "closed" + }, + "Qwen3.5-4B/UD-Q4_K_XL|golden-paris": { + "ggml@bf16:llama.cpp": "closed", + "hf@bf16:hf": "closed", + "vllm@bf16:vllm": "closed", + "ggml@quant:llama.cpp": "closed" + }, + "Qwen3.5-4B/UD-Q4_K_XL|fact-water-formula": { + "ggml@bf16:llama.cpp": "unclosed", + "hf@bf16:hf": "unclosed", + "vllm@bf16:vllm": "unclosed", + "ggml@quant:llama.cpp": "unclosed" + }, + "Qwen3.5-4B/UD-Q4_K_XL|nonascii-water-kanji": { + "ggml@bf16:llama.cpp": "closed", + "hf@bf16:hf": "closed", + "vllm@bf16:vllm": "closed", + "ggml@quant:llama.cpp": "closed" + }, + "Qwen3.5-4B/UD-Q4_K_XL|arith-17x23": { + "ggml@bf16:llama.cpp": "closed", + "hf@bf16:hf": "closed", + "vllm@bf16:vllm": "closed", + "ggml@quant:llama.cpp": "closed" + }, + "Qwen3.5-4B/UD-Q4_K_XL|arith-precedence": { + "ggml@bf16:llama.cpp": "closed", + "hf@bf16:hf": "closed", + "vllm@bf16:vllm": "closed", + "ggml@quant:llama.cpp": "closed" + }, + "Qwen3.5-4B/UD-Q4_K_XL|arith-word-apples": { + "ggml@bf16:llama.cpp": "closed", + "hf@bf16:hf": "closed", + "vllm@bf16:vllm": "closed", + "ggml@quant:llama.cpp": "closed" + }, + "Qwen3.5-4B/UD-Q4_K_XL|hard-add-4digit": { + "ggml@bf16:llama.cpp": "closed", + "hf@bf16:hf": "closed", + "vllm@bf16:vllm": "closed", + "ggml@quant:llama.cpp": "unclosed" + }, + "Qwen3.5-4B/UD-Q4_K_XL|hard-not-prime": { + "ggml@bf16:llama.cpp": "closed", + "hf@bf16:hf": "closed", + "vllm@bf16:vllm": "closed", + "ggml@quant:llama.cpp": "closed" + }, + "Qwen3.5-4B/UD-Q4_K_XL|hard-reverse-word": { + "ggml@bf16:llama.cpp": "closed", + "hf@bf16:hf": "closed", + "vllm@bf16:vllm": "closed", + "ggml@quant:llama.cpp": "closed" + }, + "Qwen3.5-4B/UD-Q4_K_XL|hard-count-letters": { + "ggml@bf16:llama.cpp": "closed", + "hf@bf16:hf": "closed", + "vllm@bf16:vllm": "closed", + "ggml@quant:llama.cpp": "closed" + }, + "Qwen3.5-4B/UD-Q4_K_XL|recall-cats": { + "ggml@bf16:llama.cpp": "unclosed", + "hf@bf16:hf": "closed", + "vllm@bf16:vllm": "closed", + "ggml@quant:llama.cpp": "unclosed" + }, + "Qwen3.5-4B/UD-Q4_K_XL|recall-name": { + "ggml@bf16:llama.cpp": "closed", + "hf@bf16:hf": "closed", + "vllm@bf16:vllm": "closed", + "ggml@quant:llama.cpp": "unclosed" + }, + "Qwen3.5-4B/UD-Q4_K_XL|recall-arith-chain": { + "ggml@bf16:llama.cpp": "closed", + "hf@bf16:hf": "closed", + "vllm@bf16:vllm": "closed", + "ggml@quant:llama.cpp": "closed" + }, + "Qwen3.5-4B/UD-Q4_K_XL|code-is-palindrome": { + "ggml@bf16:llama.cpp": "closed", + "hf@bf16:hf": "closed", + "vllm@bf16:vllm": "closed", + "ggml@quant:llama.cpp": "unclosed" + }, + "Qwen3.5-4B/UD-Q4_K_XL|code-fib": { + "ggml@bf16:llama.cpp": "unclosed", + "hf@bf16:hf": "unclosed", + "vllm@bf16:vllm": "unclosed", + "ggml@quant:llama.cpp": "unclosed" + }, + "Qwen3.5-4B/UD-Q4_K_XL|code-count-vowels": { + "ggml@bf16:llama.cpp": "closed", + "hf@bf16:hf": "closed", + "vllm@bf16:vllm": "closed", + "ggml@quant:llama.cpp": "unclosed" + }, + "Qwen3.5-4B/UD-Q4_K_XL|long-primes-10": { + "ggml@bf16:llama.cpp": "closed", + "hf@bf16:hf": "closed", + "vllm@bf16:vllm": "closed", + "ggml@quant:llama.cpp": "closed" + }, + "Qwen3.5-4B/UD-Q4_K_XL|long-countdown-15": { + "ggml@bf16:llama.cpp": "closed", + "hf@bf16:hf": "closed", + "vllm@bf16:vllm": "closed", + "ggml@quant:llama.cpp": "closed" + } + }, + "cells": [ + { + "leg": "ggml@bf16", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": false, + "why": "unclosed think (budget exhausted)", + "extracted": null, + "think": "unclosed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L1", + "prompt_id": "ctl-2plus2", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": false, + "why": "unclosed think (budget exhausted)", + "extracted": null, + "think": "unclosed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L263", + "prompt_id": "ctl-2plus2", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "ctl-2plus2", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "ctl-2plus2", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "ctl-2plus2", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "4", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L11", + "prompt_id": "ctl-2plus2", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "4", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L1", + "prompt_id": "ctl-2plus2", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "4", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L23", + "prompt_id": "ctl-2plus2", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "4", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L33", + "prompt_id": "ctl-2plus2", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": false, + "why": "unclosed think (budget exhausted)", + "extracted": null, + "think": "unclosed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L2", + "prompt_id": "ctl-recall-7", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": false, + "why": "unclosed think (budget exhausted)", + "extracted": null, + "think": "unclosed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L264", + "prompt_id": "ctl-recall-7", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "ctl-recall-7", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "ctl-recall-7", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "ctl-recall-7", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "7", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L12", + "prompt_id": "ctl-recall-7", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "7", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L2", + "prompt_id": "ctl-recall-7", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "7", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L24", + "prompt_id": "ctl-recall-7", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "7", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L34", + "prompt_id": "ctl-recall-7", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "def add(a, b):\n return a + b\n", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L3", + "prompt_id": "ctl-code-add", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "ctl-code-add", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "ctl-code-add", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "ctl-code-add", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "def add(a, b):\n return a + b\n", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L13", + "prompt_id": "ctl-code-add", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "def add(a, b):\n return a + b\n", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L3", + "prompt_id": "ctl-code-add", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "def add(a, b):\n return a + b\n", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L25", + "prompt_id": "ctl-code-add", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "def add(a, b):\n return a + b\n", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L35", + "prompt_id": "ctl-code-add", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": false, + "why": "unclosed think (budget exhausted)", + "extracted": null, + "think": "unclosed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L4", + "prompt_id": "fact-capital-japan", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "fact-capital-japan", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "fact-capital-japan", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "fact-capital-japan", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "Tokyo", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L14", + "prompt_id": "fact-capital-japan", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "Tokyo", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L4", + "prompt_id": "fact-capital-japan", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "Tokyo", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L26", + "prompt_id": "fact-capital-japan", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "Tokyo", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L36", + "prompt_id": "fact-capital-japan", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "Paris", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L5", + "prompt_id": "fact-capital-france", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "fact-capital-france", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "fact-capital-france", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "fact-capital-france", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "Paris", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L15", + "prompt_id": "fact-capital-france", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "Paris", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L5", + "prompt_id": "fact-capital-france", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "Paris", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L27", + "prompt_id": "fact-capital-france", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "Paris", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L37", + "prompt_id": "fact-capital-france", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "golden-paris", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "golden-paris", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "golden-paris", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "golden-paris", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "Paris", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L16", + "prompt_id": "golden-paris", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "Paris", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L6", + "prompt_id": "golden-paris", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "Paris", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L28", + "prompt_id": "golden-paris", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "Paris", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L38", + "prompt_id": "golden-paris", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": false, + "why": "unclosed think (budget exhausted)", + "extracted": null, + "think": "unclosed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L6", + "prompt_id": "fact-water-formula", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "fact-water-formula", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "fact-water-formula", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "fact-water-formula", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": false, + "why": "mismatch", + "extracted": "H_2O", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L17", + "prompt_id": "fact-water-formula", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": false, + "why": "mismatch", + "extracted": "H_2O", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L7", + "prompt_id": "fact-water-formula", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "lambda", + "correct": false, + "why": "mismatch", + "extracted": "H_2O", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L29", + "prompt_id": "fact-water-formula", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": false, + "why": "mismatch", + "extracted": "H₂O", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L39", + "prompt_id": "fact-water-formula", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "nonascii-water-kanji", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "nonascii-water-kanji", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "nonascii-water-kanji", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "nonascii-water-kanji", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "nonascii-water-kanji", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "nonascii-water-kanji", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "nonascii-water-kanji", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "nonascii-water-kanji", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": false, + "why": "unclosed think (budget exhausted)", + "extracted": null, + "think": "unclosed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L7", + "prompt_id": "arith-17x23", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "arith-17x23", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "arith-17x23", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "arith-17x23", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "391", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L18", + "prompt_id": "arith-17x23", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "391", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L8", + "prompt_id": "arith-17x23", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "391", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L30", + "prompt_id": "arith-17x23", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "391", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L40", + "prompt_id": "arith-17x23", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "23", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L8", + "prompt_id": "arith-precedence", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "arith-precedence", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "arith-precedence", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "arith-precedence", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "23", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L19", + "prompt_id": "arith-precedence", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "23", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L9", + "prompt_id": "arith-precedence", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "23", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L31", + "prompt_id": "arith-precedence", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "23", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L41", + "prompt_id": "arith-precedence", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "56", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L9", + "prompt_id": "arith-word-apples", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "arith-word-apples", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "arith-word-apples", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "arith-word-apples", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "56", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L20", + "prompt_id": "arith-word-apples", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "56", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L10", + "prompt_id": "arith-word-apples", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "56", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L32", + "prompt_id": "arith-word-apples", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "56", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L42", + "prompt_id": "arith-word-apples", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "8613", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L10", + "prompt_id": "hard-add-4digit", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "hard-add-4digit", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "hard-add-4digit", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "hard-add-4digit", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "8613", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L21", + "prompt_id": "hard-add-4digit", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "8613", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L11", + "prompt_id": "hard-add-4digit", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "8613", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L33", + "prompt_id": "hard-add-4digit", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "8613", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L43", + "prompt_id": "hard-add-4digit", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "hard-not-prime", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "hard-not-prime", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "hard-not-prime", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "hard-not-prime", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "15", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L22", + "prompt_id": "hard-not-prime", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "15", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L12", + "prompt_id": "hard-not-prime", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "15", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L34", + "prompt_id": "hard-not-prime", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "15", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L44", + "prompt_id": "hard-not-prime", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "hard-reverse-word", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "hard-reverse-word", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "hard-reverse-word", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "hard-reverse-word", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": false, + "why": "mismatch", + "extracted": "tnipal", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L23", + "prompt_id": "hard-reverse-word", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": false, + "why": "mismatch", + "extracted": "tnipal", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L13", + "prompt_id": "hard-reverse-word", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "lambda", + "correct": false, + "why": "mismatch", + "extracted": "tnipal", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L35", + "prompt_id": "hard-reverse-word", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": false, + "why": "mismatch", + "extracted": "tenap", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L45", + "prompt_id": "hard-reverse-word", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "hard-count-letters", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "hard-count-letters", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "hard-count-letters", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "hard-count-letters", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "3", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L24", + "prompt_id": "hard-count-letters", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "3", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L14", + "prompt_id": "hard-count-letters", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "3", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L36", + "prompt_id": "hard-count-letters", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "3", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L46", + "prompt_id": "hard-count-letters", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "recall-cats", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "recall-cats", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "recall-cats", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "recall-cats", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "5", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L25", + "prompt_id": "recall-cats", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "5", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L15", + "prompt_id": "recall-cats", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "5", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L37", + "prompt_id": "recall-cats", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "5", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L47", + "prompt_id": "recall-cats", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "recall-name", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "recall-name", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "recall-name", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "recall-name", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "Ada", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L26", + "prompt_id": "recall-name", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "Ada", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L16", + "prompt_id": "recall-name", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "Ada", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L38", + "prompt_id": "recall-name", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "Ada", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L48", + "prompt_id": "recall-name", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "recall-arith-chain", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "recall-arith-chain", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "recall-arith-chain", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "recall-arith-chain", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "7", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L27", + "prompt_id": "recall-arith-chain", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "7", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L17", + "prompt_id": "recall-arith-chain", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "7", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L39", + "prompt_id": "recall-arith-chain", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "7", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L49", + "prompt_id": "recall-arith-chain", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "code-is-palindrome", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "code-is-palindrome", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "code-is-palindrome", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "code-is-palindrome", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "def is_palindrome(s):\n s = s.lower()\n return s == s[::-1]\n", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L28", + "prompt_id": "code-is-palindrome", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "def is_palindrome(s):\n s = s.lower()\n return s == s[::-1]\n", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L18", + "prompt_id": "code-is-palindrome", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "def is_palindrome(s):\n s = s.lower()\n return s == s[::-1]\n", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L40", + "prompt_id": "code-is-palindrome", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "def is_palindrome(s):\n s = s.lower()\n return s == s[::-1]\n", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L50", + "prompt_id": "code-is-palindrome", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "code-fib", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "code-fib", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "code-fib", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "code-fib", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "def fib(n):\n if n < 0:\n raise ValueError(\"n must be a non-negative integer\")\n if n == 0:\n return 0\n if n == 1:\n return 1\n a, b = 0, 1\n for _ in range(2, n + 1):\n a, b = b, a + b\n return b\n", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L29", + "prompt_id": "code-fib", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "def fib(n):\n if n < 0:\n raise ValueError(\"n must be a non-negative integer\")\n if n == 0:\n return 0\n if n == 1:\n return 1\n a, b = 0, 1\n for _ in range(2, n + 1):\n a, b = b, a + b\n return b\n", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L19", + "prompt_id": "code-fib", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "def fib(n):\n if n < 0:\n raise ValueError(\"n must be a non-negative integer\")\n if n == 0:\n return 0\n if n == 1:\n return 1\n a, b = 0, 1\n for _ in range(2, n + 1):\n a, b = b, a + b\n return b\n", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L41", + "prompt_id": "code-fib", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "def fib(n):\n if n < 0:\n raise ValueError(\"n must be a non-negative integer\")\n if n == 0:\n return 0\n if n == 1:\n return 1\n a, b = 0, 1\n for _ in range(2, n + 1):\n a, b = b, a + b\n return b\n", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L51", + "prompt_id": "code-fib", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "code-count-vowels", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "code-count-vowels", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "code-count-vowels", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "code-count-vowels", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "def count_vowels(s):\n vowels = set('aeiouAEIOU')\n return sum(1 for char in s if char.lower() in vowels)\n", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L30", + "prompt_id": "code-count-vowels", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "def count_vowels(s):\n vowels = set('aeiouAEIOU')\n return sum(1 for char in s if char.lower() in vowels)\n", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L20", + "prompt_id": "code-count-vowels", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "def count_vowels(s):\n vowels = set('aeiouAEIOU')\n return sum(1 for char in s if char.lower() in vowels)\n", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L42", + "prompt_id": "code-count-vowels", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "def count_vowels(s):\n vowels = set('aeiouAEIOU')\n return sum(1 for char in s if char in vowels)\n", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L52", + "prompt_id": "code-count-vowels", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "long-primes-10", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "long-primes-10", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "long-primes-10", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "long-primes-10", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": [ + "2", + "3", + "5", + "7", + "11", + "13", + "17", + "19", + "23", + "29" + ], + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L31", + "prompt_id": "long-primes-10", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": [ + "2", + "3", + "5", + "7", + "11", + "13", + "17", + "19", + "23", + "29" + ], + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L21", + "prompt_id": "long-primes-10", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": [ + "2", + "3", + "5", + "7", + "11", + "13", + "17", + "19", + "23", + "29" + ], + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L43", + "prompt_id": "long-primes-10", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": [ + "2", + "3", + "5", + "7", + "11", + "13", + "17", + "19", + "23", + "29" + ], + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L53", + "prompt_id": "long-primes-10", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "long-countdown-15", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "long-countdown-15", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "long-countdown-15", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "correct": false, + "why": "no row", + "row": null, + "prompt_id": "long-countdown-15", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": [ + "15", + "14", + "13", + "12", + "11", + "10", + "9", + "8", + "7", + "6", + "5", + "4", + "3", + "2", + "1" + ], + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L32", + "prompt_id": "long-countdown-15", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": [ + "15", + "14", + "13", + "12", + "11", + "10", + "9", + "8", + "7", + "6", + "5", + "4", + "3", + "2", + "1" + ], + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L22", + "prompt_id": "long-countdown-15", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": [ + "15", + "14", + "13", + "12", + "11", + "10", + "9", + "8", + "7", + "6", + "5", + "4", + "3", + "2", + "1" + ], + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L44", + "prompt_id": "long-countdown-15", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": [ + "15", + "14", + "13", + "12", + "11", + "10", + "9", + "8", + "7", + "6", + "5", + "4", + "3", + "2", + "1" + ], + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L54", + "prompt_id": "long-countdown-15", + "model": "Qwen3.5-2B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "4", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L121", + "prompt_id": "ctl-2plus2", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "engine": "hf", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "4", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L187", + "prompt_id": "ctl-2plus2", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "4", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L235", + "prompt_id": "ctl-2plus2", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "4", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L143", + "prompt_id": "ctl-2plus2", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "4", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L55", + "prompt_id": "ctl-2plus2", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "4", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L45", + "prompt_id": "ctl-2plus2", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "4", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L210", + "prompt_id": "ctl-2plus2", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "4", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L77", + "prompt_id": "ctl-2plus2", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "7", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L122", + "prompt_id": "ctl-recall-7", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "engine": "hf", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "7", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L188", + "prompt_id": "ctl-recall-7", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "7", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L236", + "prompt_id": "ctl-recall-7", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": false, + "why": "unclosed think (budget exhausted)", + "extracted": null, + "think": "unclosed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L144", + "prompt_id": "ctl-recall-7", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "7", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L56", + "prompt_id": "ctl-recall-7", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "7", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L46", + "prompt_id": "ctl-recall-7", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "7", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L211", + "prompt_id": "ctl-recall-7", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "7", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L78", + "prompt_id": "ctl-recall-7", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "def add(a, b):\n return a + b\n", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L127", + "prompt_id": "ctl-code-add", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "engine": "hf", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "def add(a, b):\n return a + b\n", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L189", + "prompt_id": "ctl-code-add", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "def add(a, b):\n return a + b\n", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L237", + "prompt_id": "ctl-code-add", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "def add(a, b):\n return a + b\n", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L145", + "prompt_id": "ctl-code-add", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "def add(a, b):\n return a + b\n", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L57", + "prompt_id": "ctl-code-add", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "def add(a, b):\n return a + b\n", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L47", + "prompt_id": "ctl-code-add", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "def add(a, b):\n return a + b\n", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L212", + "prompt_id": "ctl-code-add", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "def add(a, b):\n return a + b\n", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L79", + "prompt_id": "ctl-code-add", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "Tokyo", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L123", + "prompt_id": "fact-capital-japan", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "engine": "hf", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "Tokyo", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L190", + "prompt_id": "fact-capital-japan", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "Tokyo", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L238", + "prompt_id": "fact-capital-japan", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "Tokyo", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L146", + "prompt_id": "fact-capital-japan", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "Tokyo", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L58", + "prompt_id": "fact-capital-japan", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "Tokyo", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L48", + "prompt_id": "fact-capital-japan", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "Tokyo", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L213", + "prompt_id": "fact-capital-japan", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "Tokyo", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L80", + "prompt_id": "fact-capital-japan", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "Paris", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L124", + "prompt_id": "fact-capital-france", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "engine": "hf", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "Paris", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L191", + "prompt_id": "fact-capital-france", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "Paris", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L239", + "prompt_id": "fact-capital-france", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "Paris", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L147", + "prompt_id": "fact-capital-france", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "Paris", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L59", + "prompt_id": "fact-capital-france", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "Paris", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L49", + "prompt_id": "fact-capital-france", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "Paris", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L214", + "prompt_id": "fact-capital-france", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "Paris", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L81", + "prompt_id": "fact-capital-france", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "Paris", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L128", + "prompt_id": "golden-paris", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "engine": "hf", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "Paris", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L192", + "prompt_id": "golden-paris", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "Paris", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L240", + "prompt_id": "golden-paris", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "Paris", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L148", + "prompt_id": "golden-paris", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "Paris", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L60", + "prompt_id": "golden-paris", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "Paris", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L50", + "prompt_id": "golden-paris", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "Paris", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L215", + "prompt_id": "golden-paris", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "Paris", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L82", + "prompt_id": "golden-paris", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": false, + "why": "unclosed think (budget exhausted)", + "extracted": null, + "think": "unclosed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L125", + "prompt_id": "fact-water-formula", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "engine": "hf", + "verb": "chat", + "host": "gx10", + "correct": false, + "why": "unclosed think (budget exhausted)", + "extracted": null, + "think": "unclosed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L193", + "prompt_id": "fact-water-formula", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": false, + "why": "unclosed think (budget exhausted)", + "extracted": null, + "think": "unclosed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L241", + "prompt_id": "fact-water-formula", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": false, + "why": "mismatch", + "extracted": "H_2O", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L149", + "prompt_id": "fact-water-formula", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": false, + "why": "mismatch", + "extracted": "H₂O", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L61", + "prompt_id": "fact-water-formula", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": false, + "why": "mismatch", + "extracted": "H₂O", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L51", + "prompt_id": "fact-water-formula", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": false, + "why": "mismatch", + "extracted": "H₂O", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L216", + "prompt_id": "fact-water-formula", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": false, + "why": "mismatch", + "extracted": "H2O", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L83", + "prompt_id": "fact-water-formula", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "水", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L233", + "prompt_id": "nonascii-water-kanji", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "engine": "hf", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "水", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L194", + "prompt_id": "nonascii-water-kanji", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "水", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L242", + "prompt_id": "nonascii-water-kanji", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "水", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L258", + "prompt_id": "nonascii-water-kanji", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "水", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L234", + "prompt_id": "nonascii-water-kanji", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "水", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L262", + "prompt_id": "nonascii-water-kanji", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "水", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L217", + "prompt_id": "nonascii-water-kanji", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "水", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L259", + "prompt_id": "nonascii-water-kanji", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "391", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L126", + "prompt_id": "arith-17x23", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "engine": "hf", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "391", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L195", + "prompt_id": "arith-17x23", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "391", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L243", + "prompt_id": "arith-17x23", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "391", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L150", + "prompt_id": "arith-17x23", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "391", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L62", + "prompt_id": "arith-17x23", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "391", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L52", + "prompt_id": "arith-17x23", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "391", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L218", + "prompt_id": "arith-17x23", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "391", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L84", + "prompt_id": "arith-17x23", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "23", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L129", + "prompt_id": "arith-precedence", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "engine": "hf", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "23", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L196", + "prompt_id": "arith-precedence", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "23", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L244", + "prompt_id": "arith-precedence", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "23", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L151", + "prompt_id": "arith-precedence", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "23", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L63", + "prompt_id": "arith-precedence", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "23", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L53", + "prompt_id": "arith-precedence", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "23", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L219", + "prompt_id": "arith-precedence", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "23", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L85", + "prompt_id": "arith-precedence", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "56", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L130", + "prompt_id": "arith-word-apples", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "engine": "hf", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "56", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L197", + "prompt_id": "arith-word-apples", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "56", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L245", + "prompt_id": "arith-word-apples", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "56", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L152", + "prompt_id": "arith-word-apples", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "56", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L64", + "prompt_id": "arith-word-apples", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "56", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L54", + "prompt_id": "arith-word-apples", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "56", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L220", + "prompt_id": "arith-word-apples", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "56", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L86", + "prompt_id": "arith-word-apples", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "8613", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L131", + "prompt_id": "hard-add-4digit", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "engine": "hf", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "8613", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L198", + "prompt_id": "hard-add-4digit", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "8613", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L246", + "prompt_id": "hard-add-4digit", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": false, + "why": "unclosed think (budget exhausted)", + "extracted": null, + "think": "unclosed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L153", + "prompt_id": "hard-add-4digit", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "8613", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L65", + "prompt_id": "hard-add-4digit", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "8613", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L55", + "prompt_id": "hard-add-4digit", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "8613", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L221", + "prompt_id": "hard-add-4digit", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "8613", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L87", + "prompt_id": "hard-add-4digit", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "15", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L132", + "prompt_id": "hard-not-prime", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "engine": "hf", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "15", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L199", + "prompt_id": "hard-not-prime", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "15", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L247", + "prompt_id": "hard-not-prime", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "15", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L154", + "prompt_id": "hard-not-prime", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "15", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L66", + "prompt_id": "hard-not-prime", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "15", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L56", + "prompt_id": "hard-not-prime", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "15", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L222", + "prompt_id": "hard-not-prime", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "15", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L88", + "prompt_id": "hard-not-prime", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "tenalp", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L133", + "prompt_id": "hard-reverse-word", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "engine": "hf", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "tenalp", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L200", + "prompt_id": "hard-reverse-word", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "tenalp", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L248", + "prompt_id": "hard-reverse-word", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "tenalp", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L155", + "prompt_id": "hard-reverse-word", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "tenalp", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L67", + "prompt_id": "hard-reverse-word", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "tenalp", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L57", + "prompt_id": "hard-reverse-word", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "tenalp", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L223", + "prompt_id": "hard-reverse-word", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "tenalp", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L89", + "prompt_id": "hard-reverse-word", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "3", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L134", + "prompt_id": "hard-count-letters", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "engine": "hf", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "3", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L201", + "prompt_id": "hard-count-letters", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "3", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L249", + "prompt_id": "hard-count-letters", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "3", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L156", + "prompt_id": "hard-count-letters", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "3", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L68", + "prompt_id": "hard-count-letters", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "3", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L58", + "prompt_id": "hard-count-letters", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "3", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L224", + "prompt_id": "hard-count-letters", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "3", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L90", + "prompt_id": "hard-count-letters", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": false, + "why": "unclosed think (budget exhausted)", + "extracted": null, + "think": "unclosed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L135", + "prompt_id": "recall-cats", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "engine": "hf", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "5", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L202", + "prompt_id": "recall-cats", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "5", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L250", + "prompt_id": "recall-cats", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": false, + "why": "unclosed think (budget exhausted)", + "extracted": null, + "think": "unclosed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L157", + "prompt_id": "recall-cats", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "5", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L69", + "prompt_id": "recall-cats", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "5", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L59", + "prompt_id": "recall-cats", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "5", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L225", + "prompt_id": "recall-cats", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "5", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L91", + "prompt_id": "recall-cats", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "Ada", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L136", + "prompt_id": "recall-name", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "engine": "hf", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "Ada", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L203", + "prompt_id": "recall-name", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "Ada", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L251", + "prompt_id": "recall-name", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": false, + "why": "unclosed think (budget exhausted)", + "extracted": null, + "think": "unclosed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L158", + "prompt_id": "recall-name", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "Ada", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L70", + "prompt_id": "recall-name", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "Ada", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L60", + "prompt_id": "recall-name", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "Ada", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L226", + "prompt_id": "recall-name", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "Ada", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L92", + "prompt_id": "recall-name", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "7", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L137", + "prompt_id": "recall-arith-chain", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "engine": "hf", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "7", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L204", + "prompt_id": "recall-arith-chain", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "7", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L252", + "prompt_id": "recall-arith-chain", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "7", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L159", + "prompt_id": "recall-arith-chain", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "7", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L71", + "prompt_id": "recall-arith-chain", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "7", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L61", + "prompt_id": "recall-arith-chain", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "7", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L227", + "prompt_id": "recall-arith-chain", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "7", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L93", + "prompt_id": "recall-arith-chain", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "def is_palindrome(s):\n s = s.lower()\n return s == s[::-1]\n", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L138", + "prompt_id": "code-is-palindrome", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "engine": "hf", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "def is_palindrome(s):\n return s.lower() == s[::-1].lower()\n", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L205", + "prompt_id": "code-is-palindrome", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": false, + "why": "tests_failed: AssertionError", + "extracted": "def is_palindrome(s):\n return s.lower() == s[::-1]\n", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L253", + "prompt_id": "code-is-palindrome", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": false, + "why": "unclosed think (budget exhausted)", + "extracted": null, + "think": "unclosed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L160", + "prompt_id": "code-is-palindrome", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "def is_palindrome(s):\n cleaned = ''.join(char.lower() for char in s if char.isalnum())\n return cleaned == cleaned[::-1]\n", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L72", + "prompt_id": "code-is-palindrome", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "def is_palindrome(s):\n cleaned = ''.join(char.lower() for char in s if char.isalnum())\n return cleaned == cleaned[::-1]\n", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L62", + "prompt_id": "code-is-palindrome", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "def is_palindrome(s):\n cleaned = ''.join(char.lower() for char in s if char.isalnum())\n return cleaned == cleaned[::-1]\n", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L228", + "prompt_id": "code-is-palindrome", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "def is_palindrome(s):\n cleaned = ''.join(char.lower() for char in s if char.isalnum())\n return cleaned == cleaned[::-1]\n", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L94", + "prompt_id": "code-is-palindrome", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": false, + "why": "unclosed think (budget exhausted)", + "extracted": null, + "think": "unclosed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L139", + "prompt_id": "code-fib", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "engine": "hf", + "verb": "chat", + "host": "gx10", + "correct": false, + "why": "unclosed think (budget exhausted)", + "extracted": null, + "think": "unclosed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L206", + "prompt_id": "code-fib", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": false, + "why": "unclosed think (budget exhausted)", + "extracted": null, + "think": "unclosed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L254", + "prompt_id": "code-fib", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": false, + "why": "unclosed think (budget exhausted)", + "extracted": null, + "think": "unclosed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L161", + "prompt_id": "code-fib", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "def fib(n):\n if n == 0:\n return 0\n elif n == 1:\n return 1\n else:\n a, b = 0, 1\n for _ in range(2, n + 1):\n a, b = b, a + b\n return b\n", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L73", + "prompt_id": "code-fib", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "def fib(n):\n if n == 0:\n return 0\n elif n == 1:\n return 1\n a, b = 0, 1\n for _ in range(2, n + 1):\n a, b = b, a + b\n return b\n", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L63", + "prompt_id": "code-fib", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "def fib(n):\n if n == 0:\n return 0\n elif n == 1:\n return 1\n a, b = 0, 1\n for _ in range(2, n + 1):\n a, b = b, a + b\n return b\n", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L229", + "prompt_id": "code-fib", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "def fib(n):\n if n == 0:\n return 0\n elif n == 1:\n return 1\n else:\n a, b = 0, 1\n for _ in range(2, n + 1):\n a, b = b, a + b\n return b\n", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L95", + "prompt_id": "code-fib", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "def count_vowels(s):\n count = 0\n for char in s:\n if char.lower() in 'aeiou':\n count += 1\n return count\n", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L140", + "prompt_id": "code-count-vowels", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "engine": "hf", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "def count_vowels(s):\n count = 0\n vowels = \"aeiou\"\n for char in s:\n if char.lower() in vowels:\n count += 1\n return count\n", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L207", + "prompt_id": "code-count-vowels", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "def count_vowels(s):\n return sum(1 for char in s if char.lower() in 'aeiou')\n", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L255", + "prompt_id": "code-count-vowels", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "def count_vowels(s):\n vowels = \"aeiou\"\n count = 0\n for char in s:\n if char.lower() in vowels:\n count += 1\n return count\n", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L162", + "prompt_id": "code-count-vowels", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "def count_vowels(s):\n vowels = set('aeiouAEIOU')\n return sum(1 for char in s if char in vowels)\n", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L74", + "prompt_id": "code-count-vowels", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "def count_vowels(s):\n vowels = set('aeiouAEIOU')\n return sum(1 for char in s if char in vowels)\n", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L64", + "prompt_id": "code-count-vowels", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "def count_vowels(s):\n vowels = set('aeiouAEIOU')\n return sum(1 for char in s if char in vowels)\n", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L230", + "prompt_id": "code-count-vowels", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "def count_vowels(s):\n vowels = \"aeiouAEIOU\"\n count = 0\n for char in s:\n if char in vowels:\n count += 1\n return count\n", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L96", + "prompt_id": "code-count-vowels", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": [ + "2", + "3", + "5", + "7", + "11", + "13", + "17", + "19", + "23", + "29" + ], + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L141", + "prompt_id": "long-primes-10", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "engine": "hf", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": [ + "2", + "3", + "5", + "7", + "11", + "13", + "17", + "19", + "23", + "29" + ], + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L208", + "prompt_id": "long-primes-10", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": [ + "2", + "3", + "5", + "7", + "11", + "13", + "17", + "19", + "23", + "29" + ], + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L256", + "prompt_id": "long-primes-10", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": [ + "2", + "3", + "5", + "7", + "11", + "13", + "17", + "19", + "23", + "29" + ], + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L163", + "prompt_id": "long-primes-10", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": [ + "2", + "3", + "5", + "7", + "11", + "13", + "17", + "19", + "23", + "29" + ], + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L75", + "prompt_id": "long-primes-10", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": [ + "2", + "3", + "5", + "7", + "11", + "13", + "17", + "19", + "23", + "29" + ], + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L65", + "prompt_id": "long-primes-10", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": [ + "2", + "3", + "5", + "7", + "11", + "13", + "17", + "19", + "23", + "29" + ], + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L231", + "prompt_id": "long-primes-10", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": [ + "2", + "3", + "5", + "7", + "11", + "13", + "17", + "19", + "23", + "29" + ], + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L97", + "prompt_id": "long-primes-10", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": [ + "15", + "14", + "13", + "12", + "11", + "10", + "9", + "8", + "7", + "6", + "5", + "4", + "3", + "2", + "1" + ], + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L142", + "prompt_id": "long-countdown-15", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "engine": "hf", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": [ + "15", + "14", + "13", + "12", + "11", + "10", + "9", + "8", + "7", + "6", + "5", + "4", + "3", + "2", + "1" + ], + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L209", + "prompt_id": "long-countdown-15", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": [ + "15", + "14", + "13", + "12", + "11", + "10", + "9", + "8", + "7", + "6", + "5", + "4", + "3", + "2", + "1" + ], + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L257", + "prompt_id": "long-countdown-15", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": [ + "15", + "14", + "13", + "12", + "11", + "10", + "9", + "8", + "7", + "6", + "5", + "4", + "3", + "2", + "1" + ], + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L164", + "prompt_id": "long-countdown-15", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": [ + "15", + "14", + "13", + "12", + "11", + "10", + "9", + "8", + "7", + "6", + "5", + "4", + "3", + "2", + "1" + ], + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L76", + "prompt_id": "long-countdown-15", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": [ + "15", + "14", + "13", + "12", + "11", + "10", + "9", + "8", + "7", + "6", + "5", + "4", + "3", + "2", + "1" + ], + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L66", + "prompt_id": "long-countdown-15", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": [ + "15", + "14", + "13", + "12", + "11", + "10", + "9", + "8", + "7", + "6", + "5", + "4", + "3", + "2", + "1" + ], + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L232", + "prompt_id": "long-countdown-15", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": [ + "15", + "14", + "13", + "12", + "11", + "10", + "9", + "8", + "7", + "6", + "5", + "4", + "3", + "2", + "1" + ], + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L98", + "prompt_id": "long-countdown-15", + "model": "Qwen3.5-4B/Q4_K_M" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "4", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L121", + "prompt_id": "ctl-2plus2", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "engine": "hf", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "4", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L187", + "prompt_id": "ctl-2plus2", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "4", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L235", + "prompt_id": "ctl-2plus2", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "4", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L165", + "prompt_id": "ctl-2plus2", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "4", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L55", + "prompt_id": "ctl-2plus2", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "4", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L45", + "prompt_id": "ctl-2plus2", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "4", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L210", + "prompt_id": "ctl-2plus2", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "4", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L99", + "prompt_id": "ctl-2plus2", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "7", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L122", + "prompt_id": "ctl-recall-7", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "engine": "hf", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "7", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L188", + "prompt_id": "ctl-recall-7", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "7", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L236", + "prompt_id": "ctl-recall-7", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": false, + "why": "unclosed think (budget exhausted)", + "extracted": null, + "think": "unclosed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L166", + "prompt_id": "ctl-recall-7", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "7", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L56", + "prompt_id": "ctl-recall-7", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "7", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L46", + "prompt_id": "ctl-recall-7", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "7", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L211", + "prompt_id": "ctl-recall-7", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "7", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L100", + "prompt_id": "ctl-recall-7", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "def add(a, b):\n return a + b\n", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L127", + "prompt_id": "ctl-code-add", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "engine": "hf", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "def add(a, b):\n return a + b\n", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L189", + "prompt_id": "ctl-code-add", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "def add(a, b):\n return a + b\n", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L237", + "prompt_id": "ctl-code-add", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "def add(a, b):\n return a + b\n", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L167", + "prompt_id": "ctl-code-add", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "def add(a, b):\n return a + b\n", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L57", + "prompt_id": "ctl-code-add", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "def add(a, b):\n return a + b\n", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L47", + "prompt_id": "ctl-code-add", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "def add(a, b):\n return a + b\n", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L212", + "prompt_id": "ctl-code-add", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "def add(a, b):\n return a + b\n", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L101", + "prompt_id": "ctl-code-add", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "Tokyo", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L123", + "prompt_id": "fact-capital-japan", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "engine": "hf", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "Tokyo", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L190", + "prompt_id": "fact-capital-japan", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "Tokyo", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L238", + "prompt_id": "fact-capital-japan", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "Tokyo", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L168", + "prompt_id": "fact-capital-japan", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "Tokyo", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L58", + "prompt_id": "fact-capital-japan", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "Tokyo", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L48", + "prompt_id": "fact-capital-japan", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "Tokyo", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L213", + "prompt_id": "fact-capital-japan", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "Tokyo", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L102", + "prompt_id": "fact-capital-japan", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "Paris", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L124", + "prompt_id": "fact-capital-france", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "engine": "hf", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "Paris", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L191", + "prompt_id": "fact-capital-france", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "Paris", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L239", + "prompt_id": "fact-capital-france", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "Paris", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L169", + "prompt_id": "fact-capital-france", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "Paris", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L59", + "prompt_id": "fact-capital-france", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "Paris", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L49", + "prompt_id": "fact-capital-france", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "Paris", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L214", + "prompt_id": "fact-capital-france", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "Paris", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L103", + "prompt_id": "fact-capital-france", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "Paris", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L128", + "prompt_id": "golden-paris", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "engine": "hf", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "Paris", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L192", + "prompt_id": "golden-paris", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "Paris", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L240", + "prompt_id": "golden-paris", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "Paris", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L170", + "prompt_id": "golden-paris", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "Paris", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L60", + "prompt_id": "golden-paris", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "Paris", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L50", + "prompt_id": "golden-paris", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "Paris", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L215", + "prompt_id": "golden-paris", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "Paris", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L104", + "prompt_id": "golden-paris", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": false, + "why": "unclosed think (budget exhausted)", + "extracted": null, + "think": "unclosed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L125", + "prompt_id": "fact-water-formula", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "engine": "hf", + "verb": "chat", + "host": "gx10", + "correct": false, + "why": "unclosed think (budget exhausted)", + "extracted": null, + "think": "unclosed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L193", + "prompt_id": "fact-water-formula", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": false, + "why": "unclosed think (budget exhausted)", + "extracted": null, + "think": "unclosed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L241", + "prompt_id": "fact-water-formula", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": false, + "why": "unclosed think (budget exhausted)", + "extracted": null, + "think": "unclosed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L171", + "prompt_id": "fact-water-formula", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": false, + "why": "mismatch", + "extracted": "H₂O", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L61", + "prompt_id": "fact-water-formula", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": false, + "why": "mismatch", + "extracted": "H₂O", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L51", + "prompt_id": "fact-water-formula", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": false, + "why": "mismatch", + "extracted": "H₂O", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L216", + "prompt_id": "fact-water-formula", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": false, + "why": "mismatch", + "extracted": "H₂O", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L105", + "prompt_id": "fact-water-formula", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "水", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L233", + "prompt_id": "nonascii-water-kanji", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "engine": "hf", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "水", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L194", + "prompt_id": "nonascii-water-kanji", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "水", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L242", + "prompt_id": "nonascii-water-kanji", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "水", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L260", + "prompt_id": "nonascii-water-kanji", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "水", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L234", + "prompt_id": "nonascii-water-kanji", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "水", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L262", + "prompt_id": "nonascii-water-kanji", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "水", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L217", + "prompt_id": "nonascii-water-kanji", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "水", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L261", + "prompt_id": "nonascii-water-kanji", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "391", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L126", + "prompt_id": "arith-17x23", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "engine": "hf", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "391", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L195", + "prompt_id": "arith-17x23", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "391", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L243", + "prompt_id": "arith-17x23", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "391", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L172", + "prompt_id": "arith-17x23", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "391", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L62", + "prompt_id": "arith-17x23", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "391", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L52", + "prompt_id": "arith-17x23", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "391", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L218", + "prompt_id": "arith-17x23", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "391", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L106", + "prompt_id": "arith-17x23", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "23", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L129", + "prompt_id": "arith-precedence", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "engine": "hf", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "23", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L196", + "prompt_id": "arith-precedence", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "23", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L244", + "prompt_id": "arith-precedence", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "23", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L173", + "prompt_id": "arith-precedence", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "23", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L63", + "prompt_id": "arith-precedence", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "23", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L53", + "prompt_id": "arith-precedence", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "23", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L219", + "prompt_id": "arith-precedence", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "23", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L107", + "prompt_id": "arith-precedence", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "56", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L130", + "prompt_id": "arith-word-apples", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "engine": "hf", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "56", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L197", + "prompt_id": "arith-word-apples", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "56", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L245", + "prompt_id": "arith-word-apples", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "56", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L174", + "prompt_id": "arith-word-apples", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "56", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L64", + "prompt_id": "arith-word-apples", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "56", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L54", + "prompt_id": "arith-word-apples", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "56", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L220", + "prompt_id": "arith-word-apples", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "56", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L108", + "prompt_id": "arith-word-apples", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "8613", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L131", + "prompt_id": "hard-add-4digit", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "engine": "hf", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "8613", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L198", + "prompt_id": "hard-add-4digit", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "8613", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L246", + "prompt_id": "hard-add-4digit", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": false, + "why": "unclosed think (budget exhausted)", + "extracted": null, + "think": "unclosed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L175", + "prompt_id": "hard-add-4digit", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "8613", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L65", + "prompt_id": "hard-add-4digit", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "8613", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L55", + "prompt_id": "hard-add-4digit", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "8613", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L221", + "prompt_id": "hard-add-4digit", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "8613", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L109", + "prompt_id": "hard-add-4digit", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "15", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L132", + "prompt_id": "hard-not-prime", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "engine": "hf", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "15", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L199", + "prompt_id": "hard-not-prime", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "15", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L247", + "prompt_id": "hard-not-prime", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "15", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L176", + "prompt_id": "hard-not-prime", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "15", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L66", + "prompt_id": "hard-not-prime", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "15", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L56", + "prompt_id": "hard-not-prime", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "15", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L222", + "prompt_id": "hard-not-prime", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "15", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L110", + "prompt_id": "hard-not-prime", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "tenalp", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L133", + "prompt_id": "hard-reverse-word", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "engine": "hf", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "tenalp", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L200", + "prompt_id": "hard-reverse-word", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "tenalp", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L248", + "prompt_id": "hard-reverse-word", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": false, + "why": "mismatch", + "extracted": "teanlap", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L177", + "prompt_id": "hard-reverse-word", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "tenalp", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L67", + "prompt_id": "hard-reverse-word", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "tenalp", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L57", + "prompt_id": "hard-reverse-word", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "tenalp", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L223", + "prompt_id": "hard-reverse-word", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "tenalp", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L111", + "prompt_id": "hard-reverse-word", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "3", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L134", + "prompt_id": "hard-count-letters", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "engine": "hf", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "3", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L201", + "prompt_id": "hard-count-letters", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "3", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L249", + "prompt_id": "hard-count-letters", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "3", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L178", + "prompt_id": "hard-count-letters", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "3", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L68", + "prompt_id": "hard-count-letters", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "3", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L58", + "prompt_id": "hard-count-letters", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "3", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L224", + "prompt_id": "hard-count-letters", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "3", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L112", + "prompt_id": "hard-count-letters", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": false, + "why": "unclosed think (budget exhausted)", + "extracted": null, + "think": "unclosed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L135", + "prompt_id": "recall-cats", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "engine": "hf", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "5", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L202", + "prompt_id": "recall-cats", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "5", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L250", + "prompt_id": "recall-cats", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": false, + "why": "unclosed think (budget exhausted)", + "extracted": null, + "think": "unclosed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L179", + "prompt_id": "recall-cats", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "5", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L69", + "prompt_id": "recall-cats", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "5", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L59", + "prompt_id": "recall-cats", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "5", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L225", + "prompt_id": "recall-cats", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "5", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L113", + "prompt_id": "recall-cats", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "Ada", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L136", + "prompt_id": "recall-name", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "engine": "hf", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "Ada", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L203", + "prompt_id": "recall-name", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "Ada", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L251", + "prompt_id": "recall-name", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": false, + "why": "unclosed think (budget exhausted)", + "extracted": null, + "think": "unclosed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L180", + "prompt_id": "recall-name", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "Ada", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L70", + "prompt_id": "recall-name", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "Ada", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L60", + "prompt_id": "recall-name", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "Ada", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L226", + "prompt_id": "recall-name", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "Ada", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L114", + "prompt_id": "recall-name", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "7", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L137", + "prompt_id": "recall-arith-chain", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "engine": "hf", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "7", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L204", + "prompt_id": "recall-arith-chain", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "7", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L252", + "prompt_id": "recall-arith-chain", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "7", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L181", + "prompt_id": "recall-arith-chain", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "7", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L71", + "prompt_id": "recall-arith-chain", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "7", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L61", + "prompt_id": "recall-arith-chain", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "7", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L227", + "prompt_id": "recall-arith-chain", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "7", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L115", + "prompt_id": "recall-arith-chain", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "def is_palindrome(s):\n s = s.lower()\n return s == s[::-1]\n", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L138", + "prompt_id": "code-is-palindrome", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "engine": "hf", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "def is_palindrome(s):\n return s.lower() == s[::-1].lower()\n", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L205", + "prompt_id": "code-is-palindrome", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": false, + "why": "tests_failed: AssertionError", + "extracted": "def is_palindrome(s):\n return s.lower() == s[::-1]\n", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L253", + "prompt_id": "code-is-palindrome", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": false, + "why": "unclosed think (budget exhausted)", + "extracted": null, + "think": "unclosed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L182", + "prompt_id": "code-is-palindrome", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "def is_palindrome(s):\n cleaned = ''.join(char.lower() for char in s if char.isalnum())\n return cleaned == cleaned[::-1]\n", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L72", + "prompt_id": "code-is-palindrome", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "def is_palindrome(s):\n cleaned = ''.join(char.lower() for char in s if char.isalnum())\n return cleaned == cleaned[::-1]\n", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L62", + "prompt_id": "code-is-palindrome", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "def is_palindrome(s):\n cleaned = ''.join(char.lower() for char in s if char.isalnum())\n return cleaned == cleaned[::-1]\n", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L228", + "prompt_id": "code-is-palindrome", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "def is_palindrome(s):\n cleaned = ''.join(char.lower() for char in s if char.isalnum())\n return cleaned == cleaned[::-1]\n", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L116", + "prompt_id": "code-is-palindrome", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": false, + "why": "unclosed think (budget exhausted)", + "extracted": null, + "think": "unclosed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L139", + "prompt_id": "code-fib", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "engine": "hf", + "verb": "chat", + "host": "gx10", + "correct": false, + "why": "unclosed think (budget exhausted)", + "extracted": null, + "think": "unclosed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L206", + "prompt_id": "code-fib", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": false, + "why": "unclosed think (budget exhausted)", + "extracted": null, + "think": "unclosed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L254", + "prompt_id": "code-fib", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": false, + "why": "unclosed think (budget exhausted)", + "extracted": null, + "think": "unclosed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L183", + "prompt_id": "code-fib", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "def fib(n):\n if n == 0:\n return 0\n elif n == 1:\n return 1\n else:\n a, b = 0, 1\n for _ in range(2, n + 1):\n a, b = b, a + b\n return b\n", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L73", + "prompt_id": "code-fib", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "def fib(n):\n if n == 0:\n return 0\n elif n == 1:\n return 1\n a, b = 0, 1\n for _ in range(2, n + 1):\n a, b = b, a + b\n return b\n", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L63", + "prompt_id": "code-fib", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "def fib(n):\n if n == 0:\n return 0\n elif n == 1:\n return 1\n a, b = 0, 1\n for _ in range(2, n + 1):\n a, b = b, a + b\n return b\n", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L229", + "prompt_id": "code-fib", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "def fib(n):\n if n == 0:\n return 0\n elif n == 1:\n return 1\n else:\n a, b = 0, 1\n for _ in range(2, n + 1):\n a, b = b, a + b\n return b\n", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L117", + "prompt_id": "code-fib", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "def count_vowels(s):\n count = 0\n for char in s:\n if char.lower() in 'aeiou':\n count += 1\n return count\n", + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L140", + "prompt_id": "code-count-vowels", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "engine": "hf", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "def count_vowels(s):\n count = 0\n vowels = \"aeiou\"\n for char in s:\n if char.lower() in vowels:\n count += 1\n return count\n", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L207", + "prompt_id": "code-count-vowels", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "def count_vowels(s):\n return sum(1 for char in s if char.lower() in 'aeiou')\n", + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L255", + "prompt_id": "code-count-vowels", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": false, + "why": "unclosed think (budget exhausted)", + "extracted": null, + "think": "unclosed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L184", + "prompt_id": "code-count-vowels", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "def count_vowels(s):\n vowels = set('aeiouAEIOU')\n return sum(1 for char in s if char in vowels)\n", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L74", + "prompt_id": "code-count-vowels", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": "def count_vowels(s):\n vowels = set('aeiouAEIOU')\n return sum(1 for char in s if char in vowels)\n", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L64", + "prompt_id": "code-count-vowels", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "def count_vowels(s):\n vowels = set('aeiouAEIOU')\n return sum(1 for char in s if char in vowels)\n", + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L230", + "prompt_id": "code-count-vowels", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": "def count_vowels(s):\n vowels = \"aeiouAEIOU\"\n count = 0\n for char in s:\n if char in vowels:\n count += 1\n return count\n", + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L118", + "prompt_id": "code-count-vowels", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": [ + "2", + "3", + "5", + "7", + "11", + "13", + "17", + "19", + "23", + "29" + ], + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L141", + "prompt_id": "long-primes-10", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "engine": "hf", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": [ + "2", + "3", + "5", + "7", + "11", + "13", + "17", + "19", + "23", + "29" + ], + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L208", + "prompt_id": "long-primes-10", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": [ + "2", + "3", + "5", + "7", + "11", + "13", + "17", + "19", + "23", + "29" + ], + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L256", + "prompt_id": "long-primes-10", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": [ + "2", + "3", + "5", + "7", + "11", + "13", + "17", + "19", + "23", + "29" + ], + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L185", + "prompt_id": "long-primes-10", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": [ + "2", + "3", + "5", + "7", + "11", + "13", + "17", + "19", + "23", + "29" + ], + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L75", + "prompt_id": "long-primes-10", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": [ + "2", + "3", + "5", + "7", + "11", + "13", + "17", + "19", + "23", + "29" + ], + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L65", + "prompt_id": "long-primes-10", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": [ + "2", + "3", + "5", + "7", + "11", + "13", + "17", + "19", + "23", + "29" + ], + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L231", + "prompt_id": "long-primes-10", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": [ + "2", + "3", + "5", + "7", + "11", + "13", + "17", + "19", + "23", + "29" + ], + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L119", + "prompt_id": "long-primes-10", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@bf16", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": [ + "15", + "14", + "13", + "12", + "11", + "10", + "9", + "8", + "7", + "6", + "5", + "4", + "3", + "2", + "1" + ], + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L142", + "prompt_id": "long-countdown-15", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "hf@bf16", + "thinking": "on", + "engine": "hf", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": [ + "15", + "14", + "13", + "12", + "11", + "10", + "9", + "8", + "7", + "6", + "5", + "4", + "3", + "2", + "1" + ], + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L209", + "prompt_id": "long-countdown-15", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "vllm@bf16", + "thinking": "on", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": [ + "15", + "14", + "13", + "12", + "11", + "10", + "9", + "8", + "7", + "6", + "5", + "4", + "3", + "2", + "1" + ], + "think": "closed", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L257", + "prompt_id": "long-countdown-15", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@quant", + "thinking": "on", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": [ + "15", + "14", + "13", + "12", + "11", + "10", + "9", + "8", + "7", + "6", + "5", + "4", + "3", + "2", + "1" + ], + "think": "closed", + "max_tokens": 4096, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L186", + "prompt_id": "long-countdown-15", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@bf16", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": [ + "15", + "14", + "13", + "12", + "11", + "10", + "9", + "8", + "7", + "6", + "5", + "4", + "3", + "2", + "1" + ], + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L76", + "prompt_id": "long-countdown-15", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "hf@bf16", + "thinking": "off", + "engine": "hf", + "verb": "chat", + "host": "lambda", + "correct": true, + "why": null, + "extracted": [ + "15", + "14", + "13", + "12", + "11", + "10", + "9", + "8", + "7", + "6", + "5", + "4", + "3", + "2", + "1" + ], + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-lambda.jsonl#L66", + "prompt_id": "long-countdown-15", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "vllm@bf16", + "thinking": "off", + "engine": "vllm", + "verb": "chat", + "host": "gx10", + "correct": true, + "why": null, + "extracted": [ + "15", + "14", + "13", + "12", + "11", + "10", + "9", + "8", + "7", + "6", + "5", + "4", + "3", + "2", + "1" + ], + "think": "none", + "max_tokens": null, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L232", + "prompt_id": "long-countdown-15", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + }, + { + "leg": "ggml@quant", + "thinking": "off", + "engine": "llama.cpp", + "verb": "serve run", + "host": "gx10", + "correct": true, + "why": null, + "extracted": [ + "15", + "14", + "13", + "12", + "11", + "10", + "9", + "8", + "7", + "6", + "5", + "4", + "3", + "2", + "1" + ], + "think": "closed", + "max_tokens": 1024, + "row": "evidence/crux/0.69.1/prompt-certification/manifest-gx10.jsonl#L120", + "prompt_id": "long-countdown-15", + "model": "Qwen3.5-4B/UD-Q4_K_XL" + } + ] +} From 4bf8583277cd6a1643cad8ab8f3a528a9e55be03 Mon Sep 17 00:00:00 2001 From: t Date: Mon, 26 Oct 2026 13:42:15 +0100 Subject: [PATCH 2/2] docs(roadmap): PMAT-6390 entry for the 0.72.0 release train epic (#6390) Metadata only: the release commit's ticket trailer had no roadmap entry. Refs #6390 PMAT-Ticket: PMAT-6390 Co-Authored-By: Claude Sonnet 5.5 --- docs/roadmaps/entries/PMAT-6390.yaml | 18 ++++++++++++++++++ docs/roadmaps/roadmap.yaml | 18 ++++++++++++++++++ 2 files changed, 36 insertions(+) create mode 100644 docs/roadmaps/entries/PMAT-6390.yaml diff --git a/docs/roadmaps/entries/PMAT-6390.yaml b/docs/roadmaps/entries/PMAT-6390.yaml new file mode 100644 index 0000000000..24803f1cd2 --- /dev/null +++ b/docs/roadmaps/entries/PMAT-6390.yaml @@ -0,0 +1,18 @@ +- id: PMAT-6390 + github_issue: 6390 + item_type: epic + title: 'release train 0.72.0' + status: planned + priority: high + assigned_to: null + created: 2026-10-26T08:00:00Z + updated: 2026-10-26T12:50:00Z + spec: null + acceptance_criteria: [] + phases: [] + subtasks: [] + estimated_effort: null + labels: + - kind:release + notes: null + epic: null diff --git a/docs/roadmaps/roadmap.yaml b/docs/roadmaps/roadmap.yaml index d0c6302855..eaa191a156 100644 --- a/docs/roadmaps/roadmap.yaml +++ b/docs/roadmaps/roadmap.yaml @@ -26356,3 +26356,21 @@ Not done, pending the ruling: moving the 31 fixture files to an obviously synthe notes: |- release-rehearsal-nightly pick: not_measured every night since 10-15; the reason was swallowed (gh stderr to /dev/null). MT-2: row measured_cpu_a_refused_read_names_its_status, mutant m53. +- id: PMAT-6390 + github_issue: 6390 + item_type: epic + title: 'release train 0.72.0' + status: planned + priority: high + assigned_to: null + created: 2026-10-26T08:00:00Z + updated: 2026-10-26T12:50:00Z + spec: null + acceptance_criteria: [] + phases: [] + subtasks: [] + estimated_effort: null + labels: + - kind:release + notes: null + epic: null